From 5eb5012dad6dee8be9ae03f9a5f0bdc6b8a00d32 Mon Sep 17 00:00:00 2001 From: Ant Somers Date: Mon, 5 Oct 2026 15:04:29 +0300 Subject: [PATCH 1/4] fix(playbooks): resolve a co-located host's firewall holes in every play baseline_extra_inbound came from playbooks/group_vars/decdn_nodes.yml and read decdn_bind_port, a decdn_node role default. On a host in both decdn_nodes and sponsord_hosts, the sponsord play runs baseline without that role, so the nftables render hit an undefined variable. all.yml now builds the list of public holes from the host's groups (_baseline_service_inbound, with the 4433 fallback), and decdn_nodes.yml and the new sponsord_hosts.yml both point baseline_extra_inbound at it. Every play on a host renders the same firewall. tests/firewall-holes pins the result per host shape and runs in `make test-scripts`; CI's scripts job installs ansible-core for it. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 3 ++ AGENTS.md | 12 +++++--- ansible/README.md | 10 ++++--- ansible/inventory/group_vars/decdn_nodes.yml | 4 +-- ansible/playbooks/group_vars/all.yml | 12 ++++++++ ansible/playbooks/group_vars/decdn_nodes.yml | 11 ++++---- .../playbooks/group_vars/sponsord_hosts.yml | 8 ++++++ ansible/tests/firewall-holes/check.sh | 28 +++++++++++++++++++ ansible/tests/firewall-holes/expected.json | 6 ++++ ansible/tests/firewall-holes/inventory.yml | 17 +++++++++++ tests/scripts-test.sh | 16 +++++++++-- 11 files changed, 109 insertions(+), 18 deletions(-) create mode 100644 ansible/playbooks/group_vars/sponsord_hosts.yml create mode 100755 ansible/tests/firewall-holes/check.sh create mode 100644 ansible/tests/firewall-holes/expected.json create mode 100644 ansible/tests/firewall-holes/inventory.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 42feb081..76599fb1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -381,6 +381,9 @@ jobs: # Without cloud-init on PATH the harness SKIPS the lint-cloud-init cases. - name: Install cloud-init run: command -v cloud-init || { sudo apt-get update -qq && sudo apt-get install -y -qq cloud-init; } + # Without ansible-core on PATH the harness FAILS the firewall-holes case in CI. + - name: Install ansible-core + run: command -v ansible || pipx install ansible-core - name: Script and guard tests run: make test-scripts diff --git a/AGENTS.md b/AGENTS.md index 3c19e15c..2df5fbce 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -231,10 +231,14 @@ make build / galaxy-check # the decdn.node collection **Inventory is private; the firewall hole is not.** This repo is public, so `ansible/inventory/hosts.yml` is git-ignored and a real fleet lives in a private overlay. -Inventory-adjacent group_vars do not load for an overlay, so anything every node needs regardless of inventory (today the udp/4433 -`baseline_extra_inbound` hole) lives in `ansible/playbooks/group_vars/decdn_nodes.yml`, and -the Alloy per-daemon toggles every host needs in `ansible/playbooks/group_vars/all.yml`. -Don't move it back under `inventory/`. +Inventory-adjacent group_vars do not load for an overlay, so anything every host needs +regardless of inventory lives in `ansible/playbooks/group_vars/`. The public firewall holes +(`baseline_extra_inbound`, today the node's udp/4433) are built from a host's groups in +`all.yml` (`_baseline_service_inbound`). `decdn_nodes.yml` and `sponsord_hosts.yml` both set +`baseline_extra_inbound` to that list, so a co-located host renders one firewall in every play. +`ansible/tests/firewall-holes/` (run by `make test-scripts`) pins the result per host shape. +The Alloy per-daemon toggles also live in `all.yml`. Don't move any of it back under +`inventory/`. **Galaxy collection (`decdn.node`).** The four roles (`baseline` + `decdn_node` + `grafana_alloy` + `sponsord`) ship as a diff --git a/ansible/README.md b/ansible/README.md index 9c4057bc..a9d7e18f 100644 --- a/ansible/README.md +++ b/ansible/README.md @@ -107,10 +107,12 @@ the overlay carries its own: create them next to its `hosts.yml`, starting from `host_vars/*/secret.*` in the private repo, so it never tracks a per-node secret. The public `inventory/group_vars/` and `inventory/host_vars/` are **not** loaded for it. Add child groups of your own when hosts differ by group (an origin, an egress cap on metered bandwidth), each -with a `group_vars/.yml`. Settings every node needs regardless of -inventory, currently only the udp/4433 QUIC firewall hole, live in -`playbooks/group_vars/decdn_nodes.yml`, so an overlay can't drop them. Override that per node -in `host_vars` if you have to. Playbook group_vars beat inventory group_vars. +with a `group_vars/.yml`. Settings every host needs regardless of +inventory, currently the public firewall holes (the node's udp/4433), live in +`playbooks/group_vars/`, so an overlay can't drop them: `all.yml` builds the list from the +host's groups and `decdn_nodes.yml` / `sponsord_hosts.yml` apply it. Override +`baseline_extra_inbound` per host in `host_vars` if you have to. Playbook group_vars beat +inventory group_vars. By default baseline **deploys you as yourself**: the runner (your control-machine `$USER` + its autodetected `~/.ssh` key, `id_ed25519` > `ecdsa` > `rsa`) is prepended as the head of diff --git a/ansible/inventory/group_vars/decdn_nodes.yml b/ansible/inventory/group_vars/decdn_nodes.yml index 2b3fc5d3..bf1a7c68 100644 --- a/ansible/inventory/group_vars/decdn_nodes.yml +++ b/ansible/inventory/group_vars/decdn_nodes.yml @@ -5,8 +5,8 @@ # you would rather the role authored that file) in a git-ignored secret.yml. # The node's public QUIC firewall hole (udp/4433) is NOT set here. It lives in -# playbooks/group_vars/decdn_nodes.yml so it applies to every inventory, including a -# private fleet overlay passed with INVENTORY=. +# playbooks/group_vars/ (all.yml builds it) so it applies to every inventory, +# including a private fleet overlay passed with INVENTORY=. # # Settings every node shares (decdn_network, decdn_node_install_method, …) can move # here from host_vars//main.yml once you run more than one node. host_vars diff --git a/ansible/playbooks/group_vars/all.yml b/ansible/playbooks/group_vars/all.yml index e003d873..a35a2cec 100644 --- a/ansible/playbooks/group_vars/all.yml +++ b/ansible/playbooks/group_vars/all.yml @@ -15,3 +15,15 @@ # host_vars, so override them there. grafana_alloy_node_enabled: "{{ 'decdn_nodes' in group_names }}" grafana_alloy_sponsord_enabled: "{{ 'sponsord_hosts' in group_names }}" + +# The public holes in the baseline firewall, from the groups a host is in. Each +# group that runs baseline (decdn_nodes.yml, sponsord_hosts.yml) sets +# baseline_extra_inbound to this, so every play on a host renders the same rules. +# Only genuinely public listeners belong here: metrics, admin RPC and the sponsord +# API stay on loopback and get no hole. +# decdn_nodes: the node's QUIC listener. decdn_bind_port is a decdn_node role +# default, which a play without that role (sponsord.yml) does not load; the +# fallback must match roles/decdn_node/defaults/main.yml. +_baseline_service_inbound: >- + {{ ([{'proto': 'udp', 'port': decdn_bind_port | default(4433), 'comment': 'decdn QUIC'}] + if 'decdn_nodes' in group_names else []) }} diff --git a/ansible/playbooks/group_vars/decdn_nodes.yml b/ansible/playbooks/group_vars/decdn_nodes.yml index 68b95e86..591a7b9b 100644 --- a/ansible/playbooks/group_vars/decdn_nodes.yml +++ b/ansible/playbooks/group_vars/decdn_nodes.yml @@ -9,9 +9,8 @@ # Precedence: playbook group_vars beat inventory group_vars, so to change this # for a node, override it in that node's host_vars. -# Open the node's public QUIC listener in the baseline firewall. Metrics (9090) -# and admin RPC (9191) stay loopback and get NO hole. -baseline_extra_inbound: - - proto: udp - port: "{{ decdn_bind_port }}" - comment: "decdn QUIC" +# The public holes in the baseline firewall: the node's QUIC listener, plus those of +# any other service the host also runs (all.yml builds the list from the host's +# groups). sponsord_hosts.yml sets the same value, so a host in both groups renders +# one firewall in both plays. +baseline_extra_inbound: "{{ _baseline_service_inbound }}" diff --git a/ansible/playbooks/group_vars/sponsord_hosts.yml b/ansible/playbooks/group_vars/sponsord_hosts.yml new file mode 100644 index 00000000..737c9f7b --- /dev/null +++ b/ansible/playbooks/group_vars/sponsord_hosts.yml @@ -0,0 +1,8 @@ +--- +# Playbook-adjacent group_vars for sponsord_hosts (see decdn_nodes.yml beside this +# for why playbook-adjacent, and for the precedence). +# +# The same list as decdn_nodes.yml, built in all.yml from the host's groups. It +# must be identical in both files: a host in both groups runs baseline once per +# play, and two different lists would flip its firewall on every run. +baseline_extra_inbound: "{{ _baseline_service_inbound }}" diff --git a/ansible/tests/firewall-holes/check.sh b/ansible/tests/firewall-holes/check.sh new file mode 100755 index 00000000..8f760ccd --- /dev/null +++ b/ansible/tests/firewall-holes/check.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# The baseline firewall's public holes are derived from a host's groups in +# playbooks/group_vars/ (all.yml builds the list; decdn_nodes.yml and +# sponsord_hosts.yml both point baseline_extra_inbound at it). Resolve them for +# every host shape in inventory.yml, outside any role, which is how the sponsord +# play sees a co-located node, and compare with expected.json. +# Needs ansible-core and jq. tests/scripts-test.sh runs it. +set -euo pipefail + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +playbooks="$here/../../playbooks" + +got="$( + # From this directory, so ansible/ansible.cfg (inventory, become) is not read. + cd "$here" + ANSIBLE_LOAD_CALLBACK_PLUGINS=1 ANSIBLE_STDOUT_CALLBACK=json ANSIBLE_DEPRECATION_WARNINGS=0 \ + ansible all -i "$here/inventory.yml" --playbook-dir "$playbooks" \ + -m ansible.builtin.debug -a var=baseline_extra_inbound &2 + diff <(echo "$want") <(echo "$got") >&2 || true + exit 1 +fi +echo "baseline_extra_inbound matches for $(jq -r 'keys | join(", ")' <<<"$want")" diff --git a/ansible/tests/firewall-holes/expected.json b/ansible/tests/firewall-holes/expected.json new file mode 100644 index 00000000..f9927011 --- /dev/null +++ b/ansible/tests/firewall-holes/expected.json @@ -0,0 +1,6 @@ +{ + "node": [{"proto": "udp", "port": 4433, "comment": "decdn QUIC"}], + "node-port": [{"proto": "udp", "port": 5555, "comment": "decdn QUIC"}], + "sponsord": [], + "co-located": [{"proto": "udp", "port": 4433, "comment": "decdn QUIC"}] +} diff --git a/ansible/tests/firewall-holes/inventory.yml b/ansible/tests/firewall-holes/inventory.yml new file mode 100644 index 00000000..312e2339 --- /dev/null +++ b/ansible/tests/firewall-holes/inventory.yml @@ -0,0 +1,17 @@ +--- +# Fixture for check.sh: one host per group shape. Nothing here is ever contacted; +# check.sh only asks Ansible to resolve each host's variables. +all: + vars: + ansible_connection: local + children: + decdn_nodes: + hosts: + node: {} + node-port: + decdn_bind_port: 5555 + co-located: {} + sponsord_hosts: + hosts: + sponsord: {} + co-located: {} diff --git a/tests/scripts-test.sh b/tests/scripts-test.sh index 47c05336..1a966f7d 100755 --- a/tests/scripts-test.sh +++ b/tests/scripts-test.sh @@ -1,10 +1,12 @@ #!/usr/bin/env bash # Tests for the repo's own guard rails that no molecule scenario or chart render # exercises: the ansible/ Makefile's scoping guards, the release gate, the -# lint-compose and lint-cloud-init invariants (negative cases), and — with +# lint-compose and lint-cloud-init invariants (negative cases), the firewall holes +# playbooks/group_vars/ derives per host, and — with # UPSTREAM= — the upstream-mirror generators' exit codes. # `make test-scripts` runs it; CI's `scripts` job does too. Needs make, docker -# (compose v2), jq, flock; the cloud-init cases also need cloud-init, shellcheck and yq. +# (compose v2), jq, flock; the cloud-init cases also need cloud-init, shellcheck and yq, +# the firewall-holes case ansible-core. set -euo pipefail repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" @@ -161,6 +163,16 @@ else skipped+=("lint-cloud-init negatives (needs cloud-init on PATH; CI installs it)") fi +# --- baseline firewall holes per host shape (playbooks/group_vars) ------------------ +if command -v ansible >/dev/null; then + expect 0 "firewall holes resolve per host shape, co-located included" \ + "$repo/ansible/tests/firewall-holes/check.sh" +elif [[ -n ${CI:-} ]]; then + fail "ansible is not on PATH in CI; the firewall-holes check would be skipped" +else + skipped+=("firewall holes (needs ansible-core on PATH)") +fi + # --- upstream-mirror generators (optional: needs a decdn/decdn checkout) -------------- if [[ -n "${UPSTREAM:-}" ]]; then expect 0 "network profiles current" "$repo/scripts/sync-network-profiles.py" "$UPSTREAM" --check From 1d9a4d427fa2279b07ffb5c74c5b046b08875aa6 Mon Sep 17 00:00:00 2001 From: Ant Somers Date: Mon, 5 Oct 2026 15:50:55 +0300 Subject: [PATCH 2/4] feat(sponsord_onramp): deploy sponsord-onramp behind Caddy beside sponsord Upstream decdn/sponsord ships two server-side services; only the daemon was deployed. The new sponsord_onramp role deploys the public onramp (Turnstile gate, decdn.sh/decdn.ps1 installers, the decdn-sponsored CLI API) as a second play of playbooks/sponsord.yml on sponsord_onramp_hosts, which must also be in sponsord_hosts: it reads the daemon's own API token and calls it on loopback. - Install, unit, gates: the sponsord role's patterns. Manual binary or a GPG-verified sponsord-onramp-v* release (KEYS from the sponsord role), DynamicUser with the token and the host-provisioned Turnstile secret as LoadCredential=, PartOf=sponsord.service, a restart-inputs record, and a fatal /healthz gate on a loopback listener. - Public inputs: the RPC URL is served to every user, so preflight refuses userinfo, a query and anything upstream's installers cannot quote. Release pins and the sitekey follow upstream's shapes. CapacityBond/SlashJudge come from a third generated network mirror. - TLS front: Caddy from the distro by default (role-owned, validated Caddyfile, admin API off, ACME or tls internal; a foreign Caddyfile is refused without an explicit overwrite), or proxy "none". tcp/80 + 443 come from _baseline_service_inbound. - Alloy: grafana_alloy_sponsord_onramp_enabled labels and re-levels the onramp's journal stream (no /metrics, no scrape); the systemd collector also watches sponsord-onramp.service and caddy.service. - Tests: molecule/sponsord-onramp (Debian 12 + Ubuntu 24.04, real Caddy with tls internal: https chain, a forged X-Forwarded-For, Turnstile rotation, fatal gate, foreign Caddyfile), 25 validation cases, a new lint-alloy render + level-harness case, the firewall-holes shapes. - The role ships in the decdn.node collection (now five roles). Co-Authored-By: Claude Opus 5.5 --- .github/workflows/molecule.yml | 3 +- .github/workflows/upstream-drift.yml | 5 +- AGENTS.md | 38 +- README.md | 5 +- ansible/Makefile | 10 +- ansible/README.md | 35 +- ansible/galaxy/CHANGELOG.md | 45 ++ ansible/galaxy/README.md | 10 +- ansible/galaxy/build.sh | 5 +- ansible/galaxy/galaxy.yml | 8 +- .../group_vars/sponsord_onramp_hosts.yml | 35 + ansible/inventory/hosts.yml.example | 9 + ansible/molecule/sponsord-onramp/converge.yml | 7 + .../files/sponsord-onramp-stub | 114 ++++ ansible/molecule/sponsord-onramp/molecule.yml | 109 +++ ansible/molecule/sponsord-onramp/prepare.yml | 51 ++ .../molecule/sponsord-onramp/side_effect.yml | 148 +++++ ansible/molecule/sponsord-onramp/verify.yml | 169 +++++ ansible/molecule/validation/converge.yml | 408 +++++++++++- ansible/molecule/validation/molecule.yml | 5 + ansible/playbooks/group_vars/all.yml | 18 +- ansible/playbooks/sponsord.yml | 32 +- ansible/roles/grafana_alloy/README.md | 14 +- ansible/roles/grafana_alloy/defaults/main.yml | 8 +- .../roles/grafana_alloy/tasks/preflight.yml | 8 +- .../grafana_alloy/templates/config.alloy.j2 | 34 +- ansible/roles/sponsord/README.md | 12 +- ansible/roles/sponsord_onramp/README.md | 182 +++++ .../roles/sponsord_onramp/defaults/main.yml | 137 ++++ .../roles/sponsord_onramp/handlers/main.yml | 19 + ansible/roles/sponsord_onramp/meta/main.yml | 17 + ansible/roles/sponsord_onramp/tasks/caddy.yml | 85 +++ .../roles/sponsord_onramp/tasks/install.yml | 257 +++++++ ansible/roles/sponsord_onramp/tasks/main.yml | 626 ++++++++++++++++++ .../sponsord_onramp/templates/Caddyfile.j2 | 22 + .../templates/sponsord-onramp.env.j2 | 34 + .../templates/sponsord-onramp.service.j2 | 66 ++ .../roles/sponsord_onramp/vars/main/arch.yml | 16 + .../sponsord_onramp/vars/main/networks.yml | 15 + ansible/tests/alloy-config/render.yml | 20 + ansible/tests/alloy-config/validate.sh | 37 +- ansible/tests/firewall-holes/expected.json | 12 +- ansible/tests/firewall-holes/inventory.yml | 10 + docs/requirements.md | 7 +- scripts/sync-network-profiles.py | 28 +- 45 files changed, 2851 insertions(+), 84 deletions(-) create mode 100644 ansible/inventory/group_vars/sponsord_onramp_hosts.yml create mode 100644 ansible/molecule/sponsord-onramp/converge.yml create mode 100755 ansible/molecule/sponsord-onramp/files/sponsord-onramp-stub create mode 100644 ansible/molecule/sponsord-onramp/molecule.yml create mode 100644 ansible/molecule/sponsord-onramp/prepare.yml create mode 100644 ansible/molecule/sponsord-onramp/side_effect.yml create mode 100644 ansible/molecule/sponsord-onramp/verify.yml create mode 100644 ansible/roles/sponsord_onramp/README.md create mode 100644 ansible/roles/sponsord_onramp/defaults/main.yml create mode 100644 ansible/roles/sponsord_onramp/handlers/main.yml create mode 100644 ansible/roles/sponsord_onramp/meta/main.yml create mode 100644 ansible/roles/sponsord_onramp/tasks/caddy.yml create mode 100644 ansible/roles/sponsord_onramp/tasks/install.yml create mode 100644 ansible/roles/sponsord_onramp/tasks/main.yml create mode 100644 ansible/roles/sponsord_onramp/templates/Caddyfile.j2 create mode 100644 ansible/roles/sponsord_onramp/templates/sponsord-onramp.env.j2 create mode 100644 ansible/roles/sponsord_onramp/templates/sponsord-onramp.service.j2 create mode 100644 ansible/roles/sponsord_onramp/vars/main/arch.yml create mode 100644 ansible/roles/sponsord_onramp/vars/main/networks.yml diff --git a/.github/workflows/molecule.yml b/.github/workflows/molecule.yml index dcd8b89a..233e0dbf 100644 --- a/.github/workflows/molecule.yml +++ b/.github/workflows/molecule.yml @@ -27,7 +27,8 @@ jobs: runs-on: ubuntu-latest # Docker is preinstalled # ~4m41s at JOBS=3 with 8 scenarios. os-matrix (3 containers) and lifecycle took the # suite to ~10m at JOBS=3 on a local 16-thread box; re-measure from CI's first run. - # sponsord added a 12th scenario (2 containers, Debian 12 + Ubuntu 24.04). + # sponsord added a 12th scenario (2 containers, Debian 12 + Ubuntu 24.04), and + # sponsord-onramp a 13th (the same 2, plus an apt install of Caddy each). # A bound, not a target: without one a wedged privileged systemd container burns # the 360-minute default, and with cancel-in-progress above, some branch-protection # setups read the resulting cancelled check as "not failed" rather than as a failure. diff --git a/.github/workflows/upstream-drift.yml b/.github/workflows/upstream-drift.yml index aa249bb5..2bdd886a 100644 --- a/.github/workflows/upstream-drift.yml +++ b/.github/workflows/upstream-drift.yml @@ -3,8 +3,9 @@ name: Upstream drift # Three things in this repo mirror decdn/decdn and go stale when upstream moves: # - roles/decdn_node/vars/main/networks.yml (contract addresses; upstream -# redeploys change every one of them at once) and its subset -# roles/sponsord/vars/main/networks.yml (the same script and check) +# redeploys change every one of them at once) and its subsets +# roles/sponsord/vars/main/networks.yml and +# roles/sponsord_onramp/vars/main/networks.yml (the same script and check) # - charts/decdn-node/files/monitoring/ (dashboards + alert rules) # - ansible/molecule/schema/files/schema-keys.txt (the node.toml key inventory) # This job regenerates each from upstream main and fails when a committed copy diff --git a/AGENTS.md b/AGENTS.md index 2df5fbce..1a12a2e5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -34,7 +34,9 @@ mirror (`vars/main/networks.yml`), never hand-copied into inventory or docs. and admin RPC). A service that must accept public traffic declares exactly one hole (the node's QUIC udp/4433) via `baseline_extra_inbound`; if a service ever needs an HTTP-facing public path, front it with an explicit reverse proxy that terminates auth + TLS. Never - bind a *backend* to `0.0.0.0` or expose its raw port. + bind a *backend* to `0.0.0.0` or expose its raw port. The one HTTP-facing service today, + `sponsord-onramp`, binds loopback behind the `sponsord_onramp` role's Caddy (tcp/80 + + tcp/443), and its own gate (Turnstile) is the auth. **Kubernetes exception (chart only):** the node's metrics bind `0.0.0.0` inside the pod so kubelet probes and Prometheus can reach them. That is allowed only behind a ClusterIP-only Service and the chart's NetworkPolicy (metrics ingress limited to @@ -54,8 +56,8 @@ mirror (`vars/main/networks.yml`), never hand-copied into inventory or docs. ``` ansible/ # the deployment project (DevSec-hardened, lean roles) - playbooks/ # site.yml (decdn node + sponsord), sponsord.yml, backup.yml, decommission.yml - roles/ # baseline, decdn_node, grafana_alloy, sponsord + playbooks/ # site.yml (decdn node + sponsord), sponsord.yml (+ onramp), backup.yml, decommission.yml + roles/ # baseline, decdn_node, grafana_alloy, sponsord, sponsord_onramp inventory/ galaxy/ molecule/ # see ansible/README.md cloud-init/ # user-data.yaml + on-host bootstrap.sh; pinned ansible-core/collections (see its README.md) compose/ # Docker Compose deploy path for a single host (see its README.md) @@ -69,8 +71,9 @@ scripts/ # upstream-mirror generators + the release gate ``` **Generated mirrors of upstream — regenerate, never hand-edit:** -`ansible/roles/decdn_node/vars/main/networks.yml` and its subset -`ansible/roles/sponsord/vars/main/networks.yml` (both `scripts/sync-network-profiles.py`), +`ansible/roles/decdn_node/vars/main/networks.yml` and its subsets +`ansible/roles/sponsord/vars/main/networks.yml` and +`ansible/roles/sponsord_onramp/vars/main/networks.yml` (all `scripts/sync-network-profiles.py`), `charts/decdn-node/files/monitoring/` (`scripts/sync-monitoring.sh`) and `ansible/molecule/schema/files/schema-keys.txt` (`gen-schema-keys.py`). The weekly `upstream-drift` workflow flags staleness. These are the only protocol facts (contract @@ -157,7 +160,26 @@ addresses) the repo carries, and they carry their upstream commit. - **Molecule:** under docker, `/` must be `--make-rshared` (see `molecule/sponsord/prepare.yml`) or every credential directory is empty. - **Not in cloud-init:** its lint only allows `decdn_nodes`. - - The public `sponsord-onramp` is not deployed yet. + +- **`ansible/roles/sponsord_onramp`** — `sponsord-onramp`, sponsord's public side (the + Turnstile gate, the installers, the `decdn-sponsored` CLI API), as a second play of + `playbooks/sponsord.yml` on `sponsord_onramp_hosts` (asserted to be in `sponsord_hosts` + too: it reads the daemon's `/etc/sponsord/api-token` and calls it on loopback). + - **Install / unit / gate:** the sponsord role's patterns, copied: manual or + GPG-verified `sponsord-onramp-v*` release (the KEYS are the sponsord role's file, + via `role_path`), `DynamicUser` with the token and the operator-provisioned + Turnstile secret as `LoadCredential=`, `PartOf=sponsord.service`, a restart-inputs + record, and a fatal `/healthz` gate on a loopback listener. + - **Public inputs:** `sponsord_onramp_rpc_url` is served to every user, so preflight + refuses userinfo, a query and the characters upstream's installers cannot quote. + CapacityBond/SlashJudge come from the generated `vars/main/networks.yml`. + - **Caddy (default) or none:** the distro package, a role-owned `/etc/caddy/Caddyfile` + (marker-gated: a foreign one needs `sponsord_onramp_caddy_overwrite_config`), + `admin off`, so the handler restarts. tcp/80 + tcp/443 come from + `playbooks/group_vars/all.yml`, which reads `sponsord_onramp_proxy` from inventory. + - **Molecule:** `molecule/sponsord-onramp` runs the real Caddy with `tls internal` + and checks the https chain and that a forged `X-Forwarded-For` cannot pick the + client address. - **`cloud-init/`** — the Ansible path with no control machine. `user-data.yaml` carries only public material (the lint refuses secret-looking keys, credentials in URLs and @@ -240,8 +262,8 @@ regardless of inventory lives in `ansible/playbooks/group_vars/`. The public fir The Alloy per-daemon toggles also live in `all.yml`. Don't move any of it back under `inventory/`. -**Galaxy collection (`decdn.node`).** The four roles (`baseline` + `decdn_node` + -`grafana_alloy` + `sponsord`) ship as a +**Galaxy collection (`decdn.node`).** The five roles (`baseline` + `decdn_node` + +`grafana_alloy` + `sponsord` + `sponsord_onramp`) ship as a distributable collection. The overlay lives in `ansible/galaxy/` and is staged into a clean collection tree by `galaxy/build.sh` — there is **no** `galaxy.yml` at the `ansible/` root (that would make ansible-lint treat the deploy project as a collection). Build/validate with diff --git a/README.md b/README.md index bf28628b..cac19dcb 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,8 @@ This is the canonical statement; the per-path READMEs add only what is specific you choose. - **Localhost-only by default.** Backends bind `127.0.0.1`. A service that must accept public traffic declares its port explicitly, and the node declares exactly one: - udp/4433. On Kubernetes, metrics bind `0.0.0.0` in the pod only behind a ClusterIP + udp/4433. The sponsor's public onramp (Ansible only) stays on loopback too, behind + Caddy on tcp/80 + tcp/443. On Kubernetes, metrics bind `0.0.0.0` in the pod only behind a ClusterIP Service and a NetworkPolicy. - **Default-deny inbound** (Ansible's `baseline`, nftables). SSH is the only universally open port; extra public ports are declared via `baseline_extra_inbound`. @@ -97,7 +98,7 @@ This is the canonical statement; the per-path READMEs add only what is specific | Path | What it is | |------|------------| -| [`ansible/`](ansible/README.md) | The Ansible project: `inventory/`, `playbooks/` (`site.yml`, `sponsord.yml`, `backup.yml`, `decommission.yml`), `roles/` (`baseline`, `decdn_node`, `grafana_alloy`, `sponsord`), `galaxy/` (the `decdn.node` collection), `molecule/`. | +| [`ansible/`](ansible/README.md) | The Ansible project: `inventory/`, `playbooks/` (`site.yml`, `sponsord.yml`, `backup.yml`, `decommission.yml`), `roles/` (`baseline`, `decdn_node`, `grafana_alloy`, `sponsord`, `sponsord_onramp`), `galaxy/` (the `decdn.node` collection), `molecule/`. | | [`cloud-init/`](cloud-init/README.md) | The cloud-init deploy path: `user-data.yaml`, the on-host `bootstrap.sh`, and the pinned ansible-core and collections it installs. | | [`compose/`](compose/README.md) | The Docker Compose deploy path. | | [`charts/decdn-node/`](charts/decdn-node/README.md) | The Helm chart, with vendored dashboards and alert rules in `files/monitoring/`. | diff --git a/ansible/Makefile b/ansible/Makefile index c2ef7eec..1e5ded83 100644 --- a/ansible/Makefile +++ b/ansible/Makefile @@ -121,8 +121,10 @@ decommission: # /metrics probe timeout), grafana-cloud and grafana-cloud-token (opt-in Grafana # Cloud observability, host-provisioned vs inventory token), os-matrix (default's # plays on Debian 13, Ubuntu 24.04 and 26.04), lifecycle (network profile, -# backup + decommission) and sponsord (the sponsord role on a host with no node, -# through playbooks/sponsord.yml, on Debian 12 and Ubuntu 24.04). The glob below is the authority; this list is a guide. +# backup + decommission), sponsord (the sponsord role on a host with no node, +# through playbooks/sponsord.yml, on Debian 12 and Ubuntu 24.04) and sponsord-onramp +# (the onramp beside its daemon, behind the real distro Caddy). The glob below is the +# authority; this list is a guide. # # The scenarios run concurrently: distinct container names, no published host ports, # per-scenario ephemeral dirs, and what they share on the control machine (the @@ -209,8 +211,8 @@ molecule-all: deps molecule test --all # --- Galaxy collection (decdn.node) ------------------------------------------ -# Stage baseline + decdn_node + grafana_alloy + sponsord into a clean collection tree and -# build the artifact under build/. Only the four roles ship; see galaxy/README.md. Publishing +# Stage baseline + decdn_node + grafana_alloy + sponsord + sponsord_onramp into a clean collection tree and +# build the artifact under build/. Only the five roles ship; see galaxy/README.md. Publishing # runs from a vX.Y.Z tag in .github/workflows/release.yml (see RELEASING.md). build: ./galaxy/build.sh diff --git a/ansible/README.md b/ansible/README.md index a9d7e18f..d611c0a0 100644 --- a/ansible/README.md +++ b/ansible/README.md @@ -8,7 +8,7 @@ hardened host baseline. The roles also ship as the `decdn.node` Galaxy collectio | **`site.yml`** | Harden the host and deploy the public **deCDN node** (`decdn-node`). | `make check` / `make deploy` | | `backup.yml` | Encrypted backup of a node's identity or full state. | `make backup` | | `decommission.yml` | Stop a node and remove its service (keeps the identity; no on-chain steps). | `make decommission` | -| `sponsord.yml` | Harden the host and deploy **sponsord**, the onboarding sponsor, on hosts in `sponsord_hosts`, with or without a node. Also run by `site.yml`. | `make check-sponsord` / `make deploy-sponsord` | +| `sponsord.yml` | Harden the host and deploy **sponsord**, the onboarding sponsor, on hosts in `sponsord_hosts`, with or without a node; then its public **sponsord-onramp** (behind Caddy) on hosts in `sponsord_onramp_hosts`. Also run by `site.yml`. | `make check-sponsord` / `make deploy-sponsord` | ``` baseline host hardening: DevSec os/ssh, nftables default-deny inbound, @@ -17,7 +17,9 @@ baseline host hardening: DevSec os/ssh, nftables default-deny inbound, ├─ decdn_node public QUIC udp/4433; metrics + admin loopback; signed-tarball or │ local-build install; hardened systemd unit; backup/decommission └─ sponsord treasury signer + PaymentPool keeper; loopback API; DynamicUser - unit with systemd credentials; standalone or beside a node + │ unit with systemd credentials; standalone or beside a node + └─ sponsord_onramp the public gate + installers on sponsord's host; loopback + listener behind Caddy (ACME TLS, tcp/80 + tcp/443) ``` ## Security model @@ -26,8 +28,9 @@ The repo-wide model is in the [root README](../README.md#security-model). What i specific to this path: - **nftables is the firewall.** SSH plus the node's **udp/4433** (via - `baseline_extra_inbound`) are the only holes; metrics 9090 and the admin RPC 9191 stay - loopback. Baseline turns off `os_hardening`'s ufw config template + `baseline_extra_inbound`) are the only holes, plus **tcp/80 + tcp/443** for Caddy on a + `sponsord_onramp_hosts` host; metrics 9090, the admin RPC 9191, sponsord 8090 and the + onramp 8080 stay loopback. Baseline turns off `os_hardening`'s ufw config template (`ufw_manage_defaults: false`) so a misleading DROP-policy `/etc/default/ufw` is never written. Do not install or enable ufw: it would replace the nftables ruleset and drop QUIC and SSH. @@ -266,8 +269,19 @@ a dedicated host or a node host listed in both groups. Then run `sponsord_hosts` and the node on hosts in `decdn_nodes` (`playbooks/group_vars/all.yml`). -Setup and variables: [`roles/sponsord/README.md`](roles/sponsord/README.md). The public -`sponsord-onramp` is not deployed by this repo yet. +Setup and variables: [`roles/sponsord/README.md`](roles/sponsord/README.md). + +Its public side, `sponsord-onramp` (the Turnstile gate, the installers, the CLI API), +is the fifth role, `sponsord_onramp`. List the sponsord host in `sponsord_onramp_hosts` +as well; `make deploy-sponsord` runs it after the daemon. + +- **You provision** the Turnstile secret on the host. The onramp reads the daemon's own + API token. +- **Caddy** (the default) terminates TLS for `sponsord_onramp_domain` and opens tcp/80 + and tcp/443. Set `sponsord_onramp_proxy: none` to bring your own proxy. +- **The RPC URL it gets is public**: it is handed to every user, so it must hold no key. + +Setup and variables: [`roles/sponsord_onramp/README.md`](roles/sponsord_onramp/README.md). ### Backup and decommission @@ -308,7 +322,10 @@ itself is Debian 12), `lifecycle` (a `decdn_network` profile with an override, b backup scopes decrypted and checked, a rejected and a real decommission) and `sponsord` (`playbooks/sponsord.yml` on a host with no node: generated token, credential rotation restart, the fatal `/healthz` gate, Alloy scraping sponsord and not the node; -`grafana-cloud` covers sponsord co-located with a node). +`grafana-cloud` covers sponsord co-located with a node) and `sponsord-onramp` (the +onramp beside its daemon, behind the real distro Caddy with `tls internal`: the https +chain, the client address Caddy passes on, a Turnstile rotation restart, the fatal +`/healthz` gate, a refused foreign Caddyfile). They are independent, so they run concurrently, and each line of output is prefixed with its scenario name because the runs interleave. `make molecule-serial` is the escape hatch when that interleaving gets in the way of reading a failure. @@ -362,12 +379,12 @@ the RPC URL when it is not provisioned on the host instead). Highlights: ## Packaging as a Galaxy collection (`decdn.node`) -The four roles (`baseline`, `decdn_node`, `grafana_alloy`, `sponsord`) are also packaged as the +The five roles (`baseline`, `decdn_node`, `grafana_alloy`, `sponsord`, `sponsord_onramp`) are also packaged as the distributable **`decdn.node`** collection, for operators who bring their own playbooks. The collection overlay lives in [`galaxy/`](galaxy/) (`galaxy.yml`, the collection `README.md`/`CHANGELOG.md`, `meta/runtime.yml`, `build.sh`). It is deliberately **not** -a `galaxy.yml` at the project root: `galaxy/build.sh` stages only the four roles into a +a `galaxy.yml` at the project root: `galaxy/build.sh` stages only the five roles into a clean `ansible_collections/decdn/node/` tree and builds the artifact, so this project stays a plain Ansible project (the `make deploy`/`lint` flow is unchanged). diff --git a/ansible/galaxy/CHANGELOG.md b/ansible/galaxy/CHANGELOG.md index cc80314a..ab40ec02 100644 --- a/ansible/galaxy/CHANGELOG.md +++ b/ansible/galaxy/CHANGELOG.md @@ -8,6 +8,44 @@ collection adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) ### Added +- `sponsord_onramp` role: deploys `sponsord-onramp` (decdn/sponsord), the public + side of the sponsor: the Cloudflare Turnstile gate, the `decdn.sh` / `decdn.ps1` + installers and the API the `decdn-sponsored` CLI polls. It runs on the daemon's + host (it asserts the host is also in `sponsord_hosts`) and reads the daemon's own + `/etc/sponsord/api-token`. + - Install: a local binary (`sponsord_onramp_install_method: manual`, the default + until upstream tags a release; `sponsord_onramp_release_target_dir` follows + `sponsord_release_target_dir`) or a `sponsord-onramp-v` release + verified against its GPG-signed `SHA256SUMS`, with the keys vendored in the + `sponsord` role. + - Unit: `DynamicUser`, the daemon token and the host-provisioned Turnstile secret + (`0600 /etc/sponsord/turnstile-secret`, never handled by the role) as + `LoadCredential=` credentials, `PartOf=sponsord.service` so it re-reads the + daemon's limits when the daemon restarts. + - Inputs: `sponsord_onramp_rpc_url` is published to every user, so the role + refuses credentials, a query or any character the installers cannot quote. + The release pins (`sponsord_onramp_decdn_release` / `_cli_release` and their + `SHA256SUMS` digests) and the Turnstile sitekey are checked against upstream's + shapes. CapacityBond and SlashJudge come from + `roles/sponsord_onramp/vars/main/networks.yml`, generated by + `scripts/sync-network-profiles.py`, through `sponsord_onramp_network` (which + follows `sponsord_network` and must agree with it). + - Listener: loopback only (asserted). The deploy fails unless `/healthz` answers + from a listener owned by the running unit. A restart-input record restarts the + onramp after an out-of-band rotation of the token or the Turnstile secret. + - TLS front: `sponsord_onramp_proxy: caddy` (default) installs the distro's Caddy + and owns `/etc/caddy/Caddyfile` (validated before it lands, admin API off, + `tls internal` for staging). A Caddyfile without the role's marker on a host + that already had Caddy is refused unless + `sponsord_onramp_caddy_overwrite_config: true`. The client address is the + right-most `X-Forwarded-For` Caddy sets. `none` leaves the proxy, the public + URL and its ports to you. +- `grafana_alloy_sponsord_onramp_enabled` (default `false`): label the onramp's + journald stream with `service_name="sponsord-onramp"` and a `level` parsed from + each line, as for sponsord. It has no `/metrics`, so there is no scrape. The + systemd collector's default unit list now includes `sponsord-onramp.service` and + `caddy.service`, which changes `config.alloy` once on every Grafana-enabled host. + - `sponsord` role: deploys the deCDN onboarding sponsor (decdn/sponsord) on its own host or beside a node. It installs a local binary (`sponsord_install_method: manual`, the default until upstream tags a release) or a `sponsord-v` release @@ -125,6 +163,13 @@ collection adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) ### Fixed +- On a host in both `decdn_nodes` and `sponsord_hosts`, the sponsord play's + baseline could not render the node's udp/4433 hole: it read `decdn_bind_port`, a + `decdn_node` default that play does not load. The public holes are now built + from the host's groups in `playbooks/group_vars/all.yml`, so every play renders + the same firewall. With `sponsord_onramp_hosts` they include tcp/80 and tcp/443 + for Caddy. + - `grafana_alloy` no longer ships URL credentials in journald lines to Grafana Cloud Loki (#84). Every line was forwarded verbatim, so a daemon error that quoted the RPC URL (sponsord's `error sending request for url (…)`) put its diff --git a/ansible/galaxy/README.md b/ansible/galaxy/README.md index 7e35a2d4..18ee9273 100644 --- a/ansible/galaxy/README.md +++ b/ansible/galaxy/README.md @@ -2,14 +2,15 @@ Deploy and harden a **public [deCDN](https://decdn.org) node**. This collection is the public, reusable slice of the [`decdn/devops`](https://github.com/decdn/devops) -repository — four roles and nothing else: +repository — five roles and nothing else: | Role | Purpose | |------|---------| | `decdn.node.baseline` | Debian/Ubuntu host baseline — nftables default-deny inbound, fail2ban, unattended-upgrades, chrony, an admin sudo user, then DevSec OS + SSH hardening (applied last). | | `decdn.node.decdn_node` | The `decdn-node` daemon under a hardened systemd unit — from locally built binaries (the default until upstream tags a release) or a GPG-verified release tarball; public QUIC udp/4433, loopback metrics + admin RPC. `decdn_network` sets the chain from upstream's manifest; `tasks_from: backup` / `decommission` for day 2. | -| `decdn.node.grafana_alloy` | Opt-in Grafana Cloud observability agent — loopback-only Alloy receiver and hardened telemetry export; scrapes the node, `sponsord`, or both (`grafana_alloy_node_enabled` / `grafana_alloy_sponsord_enabled`). | +| `decdn.node.grafana_alloy` | Opt-in Grafana Cloud observability agent — loopback-only Alloy receiver and hardened telemetry export; scrapes the node, `sponsord`, or both (`grafana_alloy_node_enabled` / `grafana_alloy_sponsord_enabled`), and labels the onramp's journal (`grafana_alloy_sponsord_onramp_enabled`). | | `decdn.node.sponsord` | The `sponsord` onboarding sponsor (treasury signer + PaymentPool keeper), standalone or beside a node — local binary or GPG-verified `sponsord-v*` release, `DynamicUser` unit with the API token and treasury wallet as systemd credentials, loopback-only API, `/healthz` deploy gate. `sponsord_network` sets the chain and PaymentPool from upstream's manifest. | +| `decdn.node.sponsord_onramp` | `sponsord-onramp`, sponsord's public side (Turnstile gate, installers, CLI API), on the daemon's host — local binary or GPG-verified `sponsord-onramp-v*` release, `DynamicUser` unit with the daemon token and Turnstile secret as systemd credentials, loopback listener behind Caddy (distro package, ACME TLS, admin API off; or your own proxy), `/healthz` deploy gate. | ## Requirements @@ -81,10 +82,15 @@ full variable list, the eth-keystore prerequisite, and day-2 ops: - [`roles/decdn_node`](https://github.com/decdn/devops/tree/main/ansible/roles/decdn_node) - [`roles/grafana_alloy`](https://github.com/decdn/devops/tree/main/ansible/roles/grafana_alloy) - [`roles/sponsord`](https://github.com/decdn/devops/tree/main/ansible/roles/sponsord) +- [`roles/sponsord_onramp`](https://github.com/decdn/devops/tree/main/ansible/roles/sponsord_onramp) ## Security model Backends bind `127.0.0.1`; the node opens exactly one public hole (QUIC udp/4433). +`sponsord_onramp` also listens on loopback: Caddy is its public side, so a host that +runs it opens tcp/80 and tcp/443 for Caddy in `baseline_extra_inbound` (this repo's +playbooks derive that from `sponsord_onramp_hosts`; from your own playbook, add them +yourself). No secrets ship in the collection or are committed — the eth keystore is operator-provisioned on the host, and `rpc_url` (which may embed an API key) renders to a `0600` file. SSH hardening is applied last, after the admin key is in place, so diff --git a/ansible/galaxy/build.sh b/ansible/galaxy/build.sh index f81f88b2..d640ff88 100755 --- a/ansible/galaxy/build.sh +++ b/ansible/galaxy/build.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash # Stage and build the public `decdn.node` Galaxy collection. # -# Only the roles/baseline, roles/decdn_node, roles/grafana_alloy and roles/sponsord sources ship. All deploy machinery +# Only the roles/baseline, roles/decdn_node, roles/grafana_alloy, roles/sponsord and +# roles/sponsord_onramp sources ship. All deploy machinery # (inventory, Makefile, ansible.cfg) is excluded BY CONSTRUCTION — it is simply # never copied into the staging tree. This keeps the artifact clean and leaves the # internal project untouched (no galaxy.yml at the project root, so ansible-lint @@ -16,7 +17,7 @@ repo_root="$(cd "$ansible_dir/.." && pwd)" # repo root build_dir="$ansible_dir/build" stage="$build_dir/ansible_collections/decdn/node" -roles=(baseline decdn_node grafana_alloy sponsord) +roles=(baseline decdn_node grafana_alloy sponsord sponsord_onramp) echo "staging decdn.node -> $stage" rm -rf "$stage" diff --git a/ansible/galaxy/galaxy.yml b/ansible/galaxy/galaxy.yml index c54cfcf8..3e63d976 100644 --- a/ansible/galaxy/galaxy.yml +++ b/ansible/galaxy/galaxy.yml @@ -3,7 +3,8 @@ # slice of this DevOps repo: deploy + harden a deCDN node. # # This file is an OVERLAY, not the project root manifest. It is assembled into a -# clean collection tree (with the baseline, decdn_node, grafana_alloy and sponsord roles) by galaxy/build.sh; +# clean collection tree (with the baseline, decdn_node, grafana_alloy, sponsord and +# sponsord_onramp roles) by galaxy/build.sh; # it is deliberately NOT placed at ansible/ root, so the deploy project stays a # plain Ansible project (bare role names, roles_path, production-profile # ansible-lint) rather than being reinterpreted as a collection. See galaxy/README.md. @@ -15,8 +16,8 @@ authors: - deCDN Contributors description: >- Deploy and harden a public deCDN node: DevSec host baseline, the decdn_node - daemon (with backup and decommission), the sponsord onboarding sponsor, and - opt-in Grafana Cloud observability. + daemon (with backup and decommission), the sponsord onboarding sponsor and its + public onramp, and opt-in Grafana Cloud observability. license: - MIT # Galaxy tags must match ^[a-z0-9]+$ (no hyphens/underscores). @@ -36,6 +37,7 @@ tags: # decdn_node -> ansible.builtin only # grafana_alloy -> ansible.builtin only # sponsord -> ansible.builtin only +# sponsord_onramp -> ansible.builtin only (Caddy comes from the distro's apt) # community.general is NOT used by any role, so it is intentionally absent here. dependencies: devsec.hardening: ">=10.0.0" diff --git a/ansible/inventory/group_vars/sponsord_onramp_hosts.yml b/ansible/inventory/group_vars/sponsord_onramp_hosts.yml new file mode 100644 index 00000000..75d92688 --- /dev/null +++ b/ansible/inventory/group_vars/sponsord_onramp_hosts.yml @@ -0,0 +1,35 @@ +--- +# Shared config for sponsord-onramp hosts (the second play of playbooks/sponsord.yml, +# also run by site.yml). Each host must be in sponsord_hosts too. Uncomment and fill +# in; per-host values can move to host_vars//main.yml. +# Full variable list: roles/sponsord_onramp/defaults/main.yml and its README.md. +# +# Before the first deploy, on the host as root (the role never handles it): +# /etc/sponsord/turnstile-secret the Turnstile widget's secret key, 0600 +# The daemon's API token (/etc/sponsord/api-token) is the sponsord role's. + +# --- Binary ------------------------------------------------------------------------ +# Follows sponsord_release_target_dir (`cargo build --release` in a decdn/sponsord +# checkout builds both binaries). Once a release exists: +# sponsord_onramp_install_method: release +# sponsord_onramp_version: "0.1.0" + +# --- Public address ---------------------------------------------------------------- +# Caddy terminates TLS for this name (ACME). Its A/AAAA records must point at the +# host; the playbooks open tcp/80 + tcp/443. To bring your own proxy instead, set +# sponsord_onramp_proxy: none (HERE, not in a play: the firewall holes read it) and +# sponsord_onramp_public_url. +# sponsord_onramp_domain: downloads.example.org +# sponsord_onramp_acme_email: ops@example.org + +# --- What users get (REQUIRED) ----------------------------------------------------- +# PUBLIC: served to every user. A keyless endpoint, never sponsord_rpc_url. +# sponsord_onramp_rpc_url: https://sepolia-rollup.arbitrum.io/rpc +# sponsord_onramp_turnstile_sitekey: "0x…" +# The releases the installers install: tags and the SHA-256 of each SHA256SUMS +# (decdn-sponsored's release notes print both of its values). +# sponsord_onramp_decdn_release: v0.1.0 +# sponsord_onramp_decdn_sums_sha256: "…" +# sponsord_onramp_cli_release: decdn-sponsored-v0.1.0 +# sponsord_onramp_cli_sums_sha256: "…" +# CapacityBond and SlashJudge come from sponsord_network (group_vars/sponsord_hosts.yml). diff --git a/ansible/inventory/hosts.yml.example b/ansible/inventory/hosts.yml.example index e4e97d53..10e99c39 100644 --- a/ansible/inventory/hosts.yml.example +++ b/ansible/inventory/hosts.yml.example @@ -60,3 +60,12 @@ decdn_nodes: # hosts: # sponsord-1: # ansible_host: REPLACE_WITH_SPONSORD_VPS_IP_OR_DNS +# +# OPTIONAL: sponsord-onramp, sponsord's public side (Turnstile gate, installers, the +# decdn-sponsored CLI API), behind Caddy on tcp/80 + tcp/443. List a sponsord host +# again here; it must be in sponsord_hosts too. Its knobs are in +# group_vars/sponsord_onramp_hosts.yml; the Turnstile secret is provisioned on the +# host (roles/sponsord_onramp/README.md). +# sponsord_onramp_hosts: +# hosts: +# sponsord-1: diff --git a/ansible/molecule/sponsord-onramp/converge.yml b/ansible/molecule/sponsord-onramp/converge.yml new file mode 100644 index 00000000..2e0611e0 --- /dev/null +++ b/ansible/molecule/sponsord-onramp/converge.yml @@ -0,0 +1,7 @@ +--- +# The real entry point: both plays of playbooks/sponsord.yml (the daemon, then the +# onramp), so playbooks/group_vars/all.yml derives the Alloy toggles exactly as a +# deploy would. Variables come from molecule.yml's inventory group_vars; baseline +# is skipped there. +- name: Converge + ansible.builtin.import_playbook: ../../playbooks/sponsord.yml diff --git a/ansible/molecule/sponsord-onramp/files/sponsord-onramp-stub b/ansible/molecule/sponsord-onramp/files/sponsord-onramp-stub new file mode 100755 index 00000000..1e46caef --- /dev/null +++ b/ansible/molecule/sponsord-onramp/files/sponsord-onramp-stub @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +"""Stub sponsord-onramp binary for the Molecule scenarios. + +Stands in for the real `sponsord-onramp` so the role can be exercised end to end +without a release or a Turnstile account: + + * `--version` -> print a version line and exit 0 (the role's version backstop). + * no argument -> enforce the startup contract the real onramp enforces, then + serve on ONRAMP_BIND until killed: + - ONRAMP_PUBLIC_URL, _RPC_URL, _CAPACITY_BOND_ADDR, + _TURNSTILE_SITEKEY and the four release-pin variables set; + - ONRAMP_DAEMON_TOKEN_FILE readable and >= 32 bytes; + - ONRAMP_TURNSTILE_SECRET_FILE readable and non-empty; + - ONRAMP_BRAND_NAME, when set, equal to EXPECTED_BRAND byte for + byte (proves the role's EnvironmentFile quoting); + - the daemon at ONRAMP_DAEMON_URL answers /healthz (upstream + reads its /v1/info at start and exits if it can't). + Any violation exits 1 with the reason on stderr (the journal). + The marker file UNHEALTHY_MARKER makes it exit 1 on start, + which is how the scenario proves the readiness gate is fatal. + Routes: /healthz, and /molecule/client-ip, which echoes the + header named by ONRAMP_CLIENT_IP_HEADER, so the scenario can + check what Caddy passes on as the client address. + * anything else -> exit 2. +""" +import os +import sys +import urllib.request +from http.server import BaseHTTPRequestHandler, HTTPServer + +VERSION = "sponsord-onramp 0.0.0-molecule-stub" +UNHEALTHY_MARKER = "/etc/sponsord/onramp-stub-unhealthy" +# Must match sponsord_onramp_brand_name in the scenario's inventory. +EXPECTED_BRAND = 'deCDN "molecule" $HOME' +REQUIRED = ( + "ONRAMP_PUBLIC_URL", "ONRAMP_RPC_URL", "ONRAMP_CAPACITY_BOND_ADDR", "ONRAMP_TURNSTILE_SITEKEY", + "ONRAMP_DECDN_RELEASE", "ONRAMP_DECDN_SUMS_SHA256", "ONRAMP_CLI_RELEASE", "ONRAMP_CLI_SUMS_SHA256", +) + + +def die(msg): + print(f"sponsord-onramp-stub: {msg}", file=sys.stderr) + sys.exit(1) + + +def read_secret(var): + path = os.environ.get(var) + if not path: + die(f"{var} is not set") + try: + with open(path, "rb") as f: + value = f.read() + except OSError as e: + die(f"{var}: cannot read {path}: {e}") + value = value.removesuffix(b"\n").removesuffix(b"\r") + if not value: + die(f"{var}: {path} is empty") + return value + + +def check_startup(): + if os.path.exists(UNHEALTHY_MARKER): + die(f"{UNHEALTHY_MARKER} exists: refusing to start") + for var in REQUIRED: + if not os.environ.get(var): + die(f"{var} is not set") + if os.environ.get("ONRAMP_BRAND_NAME", EXPECTED_BRAND) != EXPECTED_BRAND: + die("ONRAMP_BRAND_NAME does not match the inventory literal (EnvironmentFile quoting broke it)") + if len(read_secret("ONRAMP_DAEMON_TOKEN_FILE")) < 32: + die("daemon token is shorter than 32 bytes") + read_secret("ONRAMP_TURNSTILE_SECRET_FILE") + daemon = os.environ.get("ONRAMP_DAEMON_URL", "http://127.0.0.1:8090") + try: + with urllib.request.urlopen(f"{daemon}/healthz", timeout=5) as r: + if r.status != 200: + die(f"daemon {daemon} answered {r.status}") + except OSError as e: + die(f"daemon {daemon} is unreachable: {e}") + + +class Handler(BaseHTTPRequestHandler): + def do_GET(self): + if self.path == "/healthz": + body = b'{"ok":true}' + elif self.path == "/molecule/client-ip": + header = os.environ.get("ONRAMP_CLIENT_IP_HEADER", "") + body = (self.headers.get(header, "") if header else self.client_address[0]).encode() + else: + self.send_error(404) + return + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args): + pass + + +def main(): + if sys.argv[1:] == ["--version"]: + print(VERSION) + return + if sys.argv[1:]: + print(f"sponsord-onramp-stub: unexpected arguments {sys.argv[1:]}", file=sys.stderr) + sys.exit(2) + check_startup() + host, _, port = os.environ.get("ONRAMP_BIND", "127.0.0.1:8080").rpartition(":") + HTTPServer((host, int(port)), Handler).serve_forever() + + +if __name__ == "__main__": + main() diff --git a/ansible/molecule/sponsord-onramp/molecule.yml b/ansible/molecule/sponsord-onramp/molecule.yml new file mode 100644 index 00000000..48e53154 --- /dev/null +++ b/ansible/molecule/sponsord-onramp/molecule.yml @@ -0,0 +1,109 @@ +--- +# sponsord-onramp beside its daemon: a host in sponsord_hosts AND +# sponsord_onramp_hosts, converged through the real playbooks/sponsord.yml (both +# plays) against the stub daemon (../sponsord/files/sponsord-stub), a stub onramp +# (files/sponsord-onramp-stub) and the REAL distro Caddy, with Caddy's internal CA +# in place of ACME (no public DNS in a container). +# +# converge the daemon (as ../sponsord, minus its RPC literal check), then the +# onramp: the host-provisioned Turnstile secret locked to 0600, the +# daemon's token as a credential, loopback listener, /healthz gate, +# Caddy terminating TLS for the domain with the role's Caddyfile; +# Alloy labelling the onramp's journal stream. +# idempotence an unchanged host must not restart anything. +# side_effect rotate the Turnstile secret out of band -> the next converge must +# restart the onramp (and only it); make the onramp refuse to start +# -> the /healthz gate must FAIL the converge; strip the role's +# marker from the Caddyfile -> the role must refuse to replace it +# until sponsord_onramp_caddy_overwrite_config is set. +# verify files, modes, unit hardening, credentials, listeners, the whole +# https chain through Caddy, the client address Caddy passes on. +# +# `baseline` is skipped (real-host-only — see ../default/molecule.yml), so the +# tcp/80 + tcp/443 holes are covered by tests/firewall-holes instead. Release +# mode is covered for the sponsord role (../sponsord), whose install tasks this +# role's mirror. The broken-input matrix lives in ../validation. +driver: + name: docker +platforms: + - name: onramp-molecule + # Same digest-pinned images as ../sponsord (repo convention). + image: geerlingguy/docker-debian12-ansible@sha256:a131e3b7a9a38dda40e98b49b39d6200ffab0ad853c714a513f5eea41bab9a25 + pre_build_image: true + command: /usr/lib/systemd/systemd + privileged: true + cgroupns_mode: host + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + groups: + - sponsord_hosts + - sponsord_onramp_hosts + # systemd 255 writes LoadCredential= files 0440 and Caddy comes from universe. + - name: onramp-molecule-u2404 + image: geerlingguy/docker-ubuntu2404-ansible@sha256:3394e335df58c961c00b78ae601ef30cdd7463b112f21b0c5f78674a70526302 + pre_build_image: true + command: /usr/lib/systemd/systemd + privileged: true + cgroupns_mode: host + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + groups: + - sponsord_hosts + - sponsord_onramp_hosts +provisioner: + name: ansible + options: + skip-tags: baseline + env: + ANSIBLE_ROLES_PATH: "${MOLECULE_PROJECT_DIRECTORY}/roles" + ANSIBLE_COLLECTIONS_PATH: "${MOLECULE_PROJECT_DIRECTORY}/collections" + # Same pipelining note as ../default/molecule.yml. + ANSIBLE_PIPELINING: "true" + inventory: + group_vars: + sponsord_hosts: + sponsord_install_method: manual + sponsord_manual_bin_src: "${MOLECULE_PROJECT_DIRECTORY}/molecule/sponsord/files/sponsord-stub" + sponsord_version: "0.0.0-molecule-stub" + sponsord_network: arbitrum-sepolia + sponsord_pool_id: "0x00000000000000000000000000000000000000000000000000000000000000aa" + sponsord_rpc_url: "https://rpc.example.invalid/v2/molecule" + sponsord_readiness_retries: 10 + sponsord_readiness_delay: 1 + decdn_grafana_cloud_enabled: true + grafana_alloy_install_method: manual + grafana_alloy_manual_bin_src: "${MOLECULE_PROJECT_DIRECTORY}/molecule/grafana-cloud/files/alloy-stub" + grafana_alloy_region: US + grafana_alloy_loki_url: https://logs-prod-xx.molecule.invalid/loki/api/v1/push + sponsord_onramp_hosts: + sponsord_onramp_install_method: manual + sponsord_onramp_manual_bin_src: "${MOLECULE_SCENARIO_DIRECTORY}/files/sponsord-onramp-stub" + sponsord_onramp_version: "0.0.0-molecule-stub" + # sponsord_onramp_network follows sponsord_network (arbitrum-sepolia). + sponsord_onramp_rpc_url: https://sepolia-rollup.arbitrum.io/rpc + sponsord_onramp_domain: onramp.molecule.test + sponsord_onramp_caddy_tls: internal + sponsord_onramp_turnstile_sitekey: 1x00000000000000000000AA + # Must equal EXPECTED_BRAND in files/sponsord-onramp-stub: quotes and a + # dollar sign prove the env file's quoting ($$ is molecule's escape). + sponsord_onramp_brand_name: 'deCDN "molecule" $$HOME' + sponsord_onramp_decdn_release: v0.1.0 + sponsord_onramp_decdn_sums_sha256: "1111111111111111111111111111111111111111111111111111111111111111" + sponsord_onramp_cli_release: decdn-sponsored-v0.1.0 + sponsord_onramp_cli_sums_sha256: "2222222222222222222222222222222222222222222222222222222222222222" + sponsord_onramp_fund_rate_per_min: 0 + sponsord_onramp_readiness_retries: 10 + sponsord_onramp_readiness_delay: 1 +verifier: + name: ansible +scenario: + test_sequence: + # Leading destroy: see ../default/molecule.yml. + - destroy + - create + - prepare + - converge + - idempotence + - side_effect + - verify + - destroy diff --git a/ansible/molecule/sponsord-onramp/prepare.yml b/ansible/molecule/sponsord-onramp/prepare.yml new file mode 100644 index 00000000..ff10630a --- /dev/null +++ b/ansible/molecule/sponsord-onramp/prepare.yml @@ -0,0 +1,51 @@ +--- +# Stage what an operator provisions before the first deploy: +# 1. the daemon's treasury keystore + password (as ../sponsord/prepare.yml); +# 2. the Turnstile secret, at 0644 so verify.yml can attribute the 0600 +# lock-down to the role; +# 3. the Grafana Cloud credential env file (as ../grafana-cloud/prepare.yml). +# The API token is NOT staged: the sponsord role generates it, and the onramp +# reads that same file. +# +# CONTAINER-ONLY step first, as ../sponsord/prepare.yml explains: without shared +# propagation on / every LoadCredential= directory comes up empty under docker. +- name: Prepare + hosts: all + become: true + tasks: + - name: Give / shared mount propagation, as systemd does on a real host # noqa: command-instead-of-module + ansible.builtin.command: + cmd: mount --make-rshared / + changed_when: true + + - name: Create the sponsord config directory + ansible.builtin.file: + path: /etc/sponsord + state: directory + mode: "0755" + + - name: Stage the placeholder treasury files and the Turnstile secret + ansible.builtin.copy: + dest: "/etc/sponsord/{{ item.name }}" + content: "{{ item.content }}" + mode: "0644" + loop: + - {name: treasury-keystore.json, content: "{{ '{}' }}\n"} + - {name: treasury-password, content: "molecule-placeholder\n"} + - {name: turnstile-secret, content: "1x0000000000000000000000000000000AA\n"} + loop_control: + label: "{{ item.name }}" + + - name: Stage the Grafana Cloud credential fixture on the host + ansible.builtin.copy: + dest: /etc/grafana-alloy.env + owner: root + group: root + mode: "0600" + content: | + GC_PROM_REMOTE_WRITE_URL=https://prometheus-prod-xx.molecule.invalid/api/prom/push + GC_OTLP_ENDPOINT=https://otlp-gateway-prod-xx.molecule.invalid/otlp + GC_PROM_USERNAME=999888777 + GC_OTLP_USERNAME=999888779 + GC_LOKI_USERNAME=999888778 + GC_API_TOKEN=molecule-test-token diff --git a/ansible/molecule/sponsord-onramp/side_effect.yml b/ansible/molecule/sponsord-onramp/side_effect.yml new file mode 100644 index 00000000..50db1814 --- /dev/null +++ b/ansible/molecule/sponsord-onramp/side_effect.yml @@ -0,0 +1,148 @@ +--- +# Three behaviours a green converge + idempotence cannot prove: +# 1. Rotating the Turnstile secret OUT OF BAND (the operator replaces the file) +# and re-running the role restarts the onramp onto it, and only the onramp: +# the daemon has nothing new to pick up. +# 2. The /healthz readiness gate is FATAL: an onramp that never binds (the stub +# refuses to start while its marker exists) fails the converge at the role's +# report task. +# 3. The role never silently replaces a Caddyfile it did not write: with its +# marker gone, the converge must stop at the refusal, then succeed once +# sponsord_onramp_caddy_overwrite_config confirms the takeover. +# The host is left healthy on the role's Caddyfile for verify.yml. +- name: Side effect — Turnstile rotation, the fatal readiness gate, a foreign Caddyfile + hosts: all + become: true + tasks: + - name: Read the unit states before the rotation + ansible.builtin.systemd_service: + name: "{{ item }}" + register: _or_before + loop: [sponsord-onramp, sponsord] + + - name: Assert the onramp was running cleanly before the rotation + ansible.builtin.assert: + that: + - _or_before.results[0].status.ActiveState == 'active' + - _or_before.results[0].status.MainPID | int > 0 + fail_msg: >- + Expected sponsord-onramp active before the rotation (got + ActiveState={{ _or_before.results[0].status.ActiveState }}): + journalctl -u sponsord-onramp -e + + - name: Rotate the Turnstile secret out of band + ansible.builtin.copy: + dest: /etc/sponsord/turnstile-secret + content: "1x0000000000000000000000000000000BB\n" + mode: "0600" + + - name: Converge the onramp again after the rotation + ansible.builtin.include_role: + name: sponsord_onramp + + - name: Read the unit states after the rotation converge + ansible.builtin.systemd_service: + name: "{{ item }}" + register: _or_after + loop: [sponsord-onramp, sponsord] + + - name: Require the rotation to have restarted the onramp, and only the onramp + ansible.builtin.assert: + that: + - _or_after.results[0].status.ActiveState == 'active' + - _or_after.results[0].status.MainPID != _or_before.results[0].status.MainPID + - _or_after.results[1].status.MainPID == _or_before.results[1].status.MainPID + fail_msg: >- + After the Turnstile secret was rotated, the onramp's PID went + {{ _or_before.results[0].status.MainPID }} -> {{ _or_after.results[0].status.MainPID }} + (must change) and the daemon's {{ _or_before.results[1].status.MainPID }} -> + {{ _or_after.results[1].status.MainPID }} (must not). + + - name: Stop the onramp, then make it refuse to start + ansible.builtin.systemd_service: + name: sponsord-onramp + state: stopped + + - name: Stage the stub's refuse-to-start marker + ansible.builtin.copy: + dest: /etc/sponsord/onramp-stub-unhealthy + content: "" + mode: "0644" + + - name: Converge against an onramp that never becomes healthy + block: + - name: Run the role with a short readiness window + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_readiness_retries: 3 + sponsord_onramp_readiness_delay: 1 + + - name: Fail — the readiness gate let an unhealthy onramp through + ansible.builtin.fail: + msg: The role finished although sponsord-onramp never answered /healthz. + rescue: + - name: Require the failure to come from the readiness gate itself + ansible.builtin.assert: + that: + - ansible_failed_task.name == 'Report a sponsord-onramp that did not become healthy' + fail_msg: >- + Expected the role to fail at its readiness report, but it failed at + "{{ ansible_failed_task.name }}". + always: + - name: Remove the refuse-to-start marker + ansible.builtin.file: + path: /etc/sponsord/onramp-stub-unhealthy + state: absent + + - name: Converge once more to bring the onramp back + ansible.builtin.include_role: + name: sponsord_onramp + + # A Caddyfile without the marker on a host where Caddy is already installed is + # exactly what an operator's own Caddy looks like to the role. + - name: Replace the Caddyfile with one this role did not write + ansible.builtin.copy: + dest: /etc/caddy/Caddyfile + mode: "0644" + content: | + # the operator's own site + other.molecule.test { + tls internal + respond "not the onramp" + } + + - name: Converge against the foreign Caddyfile + block: + - name: Run the role without the overwrite confirmation + ansible.builtin.include_role: + name: sponsord_onramp + + - name: Fail — the role replaced a Caddyfile it did not write + ansible.builtin.fail: + msg: The role finished although /etc/caddy/Caddyfile carried no marker. + rescue: + - name: Require the failure to come from the Caddyfile refusal + ansible.builtin.assert: + that: + - ansible_failed_task.name == 'Refuse to replace a Caddyfile this role did not write' + fail_msg: >- + Expected the role to refuse the foreign Caddyfile, but it failed at + "{{ ansible_failed_task.name }}". + + - name: Read the Caddyfile after the refusal + ansible.builtin.slurp: + src: /etc/caddy/Caddyfile + register: _or_foreign + + - name: Require the foreign Caddyfile to be untouched + ansible.builtin.assert: + that: + - "'not the onramp' in (_or_foreign.content | b64decode)" + fail_msg: The refused run still changed /etc/caddy/Caddyfile. + + - name: Converge with the takeover confirmed + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_caddy_overwrite_config: true diff --git a/ansible/molecule/sponsord-onramp/verify.yml b/ansible/molecule/sponsord-onramp/verify.yml new file mode 100644 index 00000000..4541f2f5 --- /dev/null +++ b/ansible/molecule/sponsord-onramp/verify.yml @@ -0,0 +1,169 @@ +--- +# What a converged onramp host must look like. Runs after side_effect.yml, which +# leaves the host healthy on the role's own Caddyfile. +- name: Verify + hosts: all + become: true + gather_facts: false + tasks: + - name: Stat the onramp's files + ansible.builtin.stat: + path: "{{ item }}" + register: _or_files + loop: + - /etc/sponsord/turnstile-secret + - /etc/sponsord/sponsord-onramp.env + - /etc/sponsord/.onramp-inputs.sha256 + - /etc/caddy/Caddyfile + + - name: Require the secret and records root-only, the config world-readable + ansible.builtin.assert: + that: + - _or_files.results[0].stat.isreg and _or_files.results[0].stat.mode == '0600' + - _or_files.results[0].stat.pw_name == 'root' + - _or_files.results[1].stat.isreg and _or_files.results[1].stat.mode == '0644' + - _or_files.results[2].stat.isreg and _or_files.results[2].stat.mode == '0600' + - _or_files.results[3].stat.isreg and _or_files.results[3].stat.mode == '0644' + fail_msg: "{{ _or_files.results | map(attribute='stat') | map(attribute='mode', default='absent') | list }}" + + - name: Read the env file and the Caddyfile + ansible.builtin.slurp: + src: "{{ item }}" + register: _or_text + loop: + - /etc/sponsord/sponsord-onramp.env + - /etc/caddy/Caddyfile + + - name: Require the env file to carry the settings and no secret + ansible.builtin.assert: + that: + - "'ONRAMP_BIND=127.0.0.1:8080' in _env" + - "'ONRAMP_DAEMON_URL=http://127.0.0.1:8090' in _env" + - "'ONRAMP_PUBLIC_URL=https://onramp.molecule.test' in _env" + - "'ONRAMP_RPC_URL=https://sepolia-rollup.arbitrum.io/rpc' in _env" + # From the network profile mirror (the daemon's sponsord_network). + - "'ONRAMP_CAPACITY_BOND_ADDR=0x' in _env" + - "'ONRAMP_SLASH_JUDGE_ADDR=0x' in _env" + - "'ONRAMP_CLIENT_IP_HEADER=X-Forwarded-For' in _env" + - "'ONRAMP_FUND_RATE_PER_MIN=0' in _env" + - "'ONRAMP_TTL_SECS' not in _env" + - "'TOKEN' not in _env" + - "'TURNSTILE_SECRET' not in _env" + - "'1x0000000000000000000000000000000' not in _env" + fail_msg: "{{ _env }}" + vars: + _env: "{{ _or_text.results[0].content | b64decode }}" + + - name: Require the role's Caddyfile, with its admin API off + ansible.builtin.assert: + that: + - "'MANAGED BY the sponsord_onramp role' in _caddy" + - "'admin off' in _caddy" + - "'onramp.molecule.test {' in _caddy" + - "'tls internal' in _caddy" + - "'reverse_proxy 127.0.0.1:8080' in _caddy" + - "'not the onramp' not in _caddy" + fail_msg: "{{ _caddy }}" + vars: + _caddy: "{{ _or_text.results[1].content | b64decode }}" + + - name: Read the onramp unit + ansible.builtin.command: + argv: [systemctl, cat, sponsord-onramp] + register: _or_unit + changed_when: false + + - name: Require credentials, hardening and the tie to the daemon + ansible.builtin.assert: + that: + # kics-scan ignore-line (a credential file PATH in the unit, not a secret) + - "'LoadCredential=api-token:/etc/sponsord/api-token' in _or_unit.stdout" + # kics-scan ignore-line (a credential file PATH in the unit, not a secret) + - "'LoadCredential=turnstile-secret:/etc/sponsord/turnstile-secret' in _or_unit.stdout" + - "'Environment=ONRAMP_DAEMON_TOKEN_FILE=%d/api-token' in _or_unit.stdout" + - "'Environment=ONRAMP_TURNSTILE_SECRET_FILE=%d/turnstile-secret' in _or_unit.stdout" + - "'DynamicUser=true' in _or_unit.stdout" + - "'ProtectSystem=strict' in _or_unit.stdout" + - "'CapabilityBoundingSet=\n' in _or_unit.stdout" + - "'PartOf=sponsord.service' in _or_unit.stdout" + fail_msg: "{{ _or_unit.stdout }}" + + - name: Read the TCP listeners + ansible.builtin.command: + argv: [ss, -Hltn] + register: _or_ss + changed_when: false + + - name: Require the onramp on loopback only, Caddy on 80 and 443 + ansible.builtin.assert: + that: + - _or_ss.stdout is search('127\.0\.0\.1:8080\s') + - _or_ss.stdout is not search('(0\.0\.0\.0|\*|\[::\]):8080\s') + - _or_ss.stdout is search('(\*|0\.0\.0\.0|\[::\]):443\s') + - _or_ss.stdout is search('(\*|0\.0\.0\.0|\[::\]):80\s') + # The admin API is off: nothing on Caddy's default 2019. + - _or_ss.stdout is not search(':2019\s') + fail_msg: "{{ _or_ss.stdout }}" + + - name: Reach the onramp through Caddy, as a user would + ansible.builtin.command: + argv: + - curl + - --silent + - --show-error + - --fail + - --insecure + - --resolve + - onramp.molecule.test:443:127.0.0.1 + - https://onramp.molecule.test/healthz + register: _or_https + changed_when: false + + - name: Require the https chain to answer + ansible.builtin.assert: + that: + - (_or_https.stdout | from_json).ok is true + fail_msg: "{{ _or_https.stdout }}" + + # The onramp rate-limits and gates by the right-most X-Forwarded-For address. + # A client that sends its own must not be able to choose it. + - name: Send a forged X-Forwarded-For through Caddy + ansible.builtin.command: + argv: + - curl + - --silent + - --show-error + - --fail + - --insecure + - --header + - "X-Forwarded-For: 203.0.113.9" + - --resolve + - onramp.molecule.test:443:127.0.0.1 + - https://onramp.molecule.test/molecule/client-ip + register: _or_xff + changed_when: false + + - name: Require the right-most address to be the real client + ansible.builtin.assert: + that: + - (_or_xff.stdout.split(',') | last | trim) == '127.0.0.1' + fail_msg: >- + The onramp saw X-Forwarded-For "{{ _or_xff.stdout }}": its right-most address + must be the client Caddy saw (127.0.0.1), not the forged 203.0.113.9. + + - name: Read the Alloy config + ansible.builtin.slurp: + src: /etc/alloy/config.alloy + register: _or_alloy + + - name: Require Alloy to label the onramp's journal stream + ansible.builtin.assert: + that: + # The journal service_name rule, and the unit in the level selector. + - "'replacement = \"sponsord-onramp\"' in _cfg" + - "'(sponsord|sponsord-onramp)' in _cfg" + - "'prometheus.scrape \"sponsord\"' in _cfg" + - "'prometheus.scrape \"decdn_node\"' not in _cfg" + fail_msg: Alloy's config lacks the onramp's journal labels (or scrapes a node). + vars: + _cfg: "{{ _or_alloy.content | b64decode }}" diff --git a/ansible/molecule/validation/converge.yml b/ansible/molecule/validation/converge.yml index 948c41be..618a11f7 100644 --- a/ansible/molecule/validation/converge.yml +++ b/ansible/molecule/validation/converge.yml @@ -1,7 +1,8 @@ --- # NEGATIVE-path scenario: proves the role's fail-loud tuning-knob validation actually # REJECTS bad values (AGENTS.md hard-rule 4), so a regression that weakens an assert -# fails CI. Each case runs one role (decdn_node, grafana_alloy or sponsord) with a +# fails CI. Each case runs one role (decdn_node, grafana_alloy, sponsord or +# sponsord_onramp) with a # VALID required config plus ONE bad input, wrapped in block/rescue. The role aborts at its validation asserts # (which for the tuning-knob cases run before any host mutation), so those cases # don't interfere and no service starts. The env-file cases below are the @@ -2108,6 +2109,401 @@ - /etc/sponsord - /root/sponsord-secret-target +- name: Converge sponsord_onramp (expect validation failures) + hosts: all + become: true + # A second play only to stay under ansible-lint's per-play task cap. The tracker + # is a host fact, so it carries over from the play above. + vars: + # The daemon's network, which the onramp's follows. + sponsord_network: arbitrum-sepolia + # Valid sponsord_onramp baseline for the onramp-* cases. + sponsord_onramp_manual_bin_src: >- + {{ lookup('ansible.builtin.env', 'MOLECULE_PROJECT_DIRECTORY') + }}/molecule/sponsord-onramp/files/sponsord-onramp-stub + sponsord_onramp_rpc_url: https://sepolia-rollup.arbitrum.io/rpc + sponsord_onramp_domain: onramp.validation.test + sponsord_onramp_turnstile_sitekey: 1x00000000000000000000AA + sponsord_onramp_decdn_release: v0.1.0 + sponsord_onramp_decdn_sums_sha256: "1111111111111111111111111111111111111111111111111111111111111111" + sponsord_onramp_cli_release: decdn-sponsored-v0.1.0 + sponsord_onramp_cli_sums_sha256: "2222222222222222222222222222222222222222222222222222222222222222" + tasks: + # --- sponsord_onramp: each case breaks ONE input over the valid baseline in play + # vars. All but the last three abort at an assert before any host mutation; those + # stage /etc/sponsord files and remove them again. + - name: "Case onramp-release-version — release mode with no pinned version" + block: + - name: Run sponsord_onramp with release mode with no pinned version + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_install_method: release + sponsord_onramp_version: "" + rescue: + - name: Record onramp-release-version rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-release-version'] }}" + when: ansible_failed_task.name == "Require a pinned release version" + + - name: "Case onramp-manual-src — manual mode with no binary source" + block: + - name: Run sponsord_onramp with manual mode with no binary source + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_manual_bin_src: "" + sponsord_onramp_release_target_dir: "" + rescue: + - name: Record onramp-manual-src rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-manual-src'] }}" + when: ansible_failed_task.name == "Require a manual binary source" + + - name: "Case onramp-network-unknown — an unknown network profile" + block: + - name: Run sponsord_onramp with an unknown network profile + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_network: nope + rescue: + - name: Record onramp-network-unknown rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-network-unknown'] }}" + when: ansible_failed_task.name == "Require a known sponsord_onramp_network" + + - name: "Case onramp-network-mismatch — a network other than the daemon's" + block: + - name: Run sponsord_onramp with a network other than the daemon's + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_network: some-other-chain + sponsord_onramp_network: arbitrum-sepolia + rescue: + - name: Record onramp-network-mismatch rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-network-mismatch'] }}" + when: ansible_failed_task.name == "Require the onramp and the daemon to name the same network" + + - name: "Case onramp-zero-address — the zero CapacityBond address" + block: + - name: Run sponsord_onramp with the zero CapacityBond address + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_capacity_bond_address: "0x0000000000000000000000000000000000000000" + rescue: + - name: Record onramp-zero-address rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-zero-address'] }}" + when: ansible_failed_task.name == "Require the CapacityBond (and a well-formed SlashJudge) address" + + - name: "Case onramp-bind — a wildcard listener" + block: + - name: Run sponsord_onramp with a wildcard listener + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_bind_address: "0.0.0.0" + rescue: + - name: Record onramp-bind rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-bind'] }}" + when: ansible_failed_task.name == "Require the listener to stay on loopback" + + - name: "Case onramp-bind — a daemon URL off the host" + block: + - name: Run sponsord_onramp with a daemon URL off the host + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_daemon_url: "http://10.0.0.5:8090" + rescue: + - name: Record onramp-bind rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-bind'] }}" + when: ansible_failed_task.name == "Require the listener to stay on loopback" + + - name: "Case onramp-proxy — an unknown proxy" + block: + - name: Run sponsord_onramp with an unknown proxy + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_proxy: nginx + rescue: + - name: Record onramp-proxy rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-proxy'] }}" + when: ansible_failed_task.name == "Validate the proxy settings" + + - name: "Case onramp-domain — Caddy with no domain" + block: + - name: Run sponsord_onramp with Caddy with no domain + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_domain: "" + rescue: + - name: Record onramp-domain rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-domain'] }}" + when: ansible_failed_task.name == "Require a domain for Caddy" + + - name: "Case onramp-public-url — an http public URL behind your own proxy" + block: + - name: Run sponsord_onramp with an http public URL behind your own proxy + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_proxy: none + sponsord_onramp_public_url: "http://onramp.validation.test" + rescue: + - name: Record onramp-public-url rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-public-url'] }}" + when: ansible_failed_task.name == "Require an https public base URL the installers can hold" + + - name: "Case onramp-public-url — no public URL behind your own proxy" + block: + - name: Run sponsord_onramp with no public URL behind your own proxy + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_proxy: none + rescue: + - name: Record onramp-public-url rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-public-url'] }}" + when: ansible_failed_task.name == "Require an https public base URL the installers can hold" + + - name: "Case onramp-rpc — a public RPC URL with credentials" + block: + - name: Run sponsord_onramp with a public RPC URL with credentials + ansible.builtin.include_role: + name: sponsord_onramp + vars: + # kics-scan ignore-line (the bad input this case must reject, not a secret) + sponsord_onramp_rpc_url: "https://user:SECRET@rpc.example.invalid/" + rescue: + - name: Record onramp-rpc rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-rpc'] }}" + when: ansible_failed_task.name == "Require a public RPC URL with no credentials in it" + + - name: "Case onramp-rpc — a public RPC URL with an API key in its query" + block: + - name: Run sponsord_onramp with a public RPC URL with an API key in its query + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_rpc_url: "https://rpc.example.invalid/?apikey=SECRET" + rescue: + - name: Record onramp-rpc rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-rpc'] }}" + when: ansible_failed_task.name == "Require a public RPC URL with no credentials in it" + + - name: "Case onramp-rpc — a public RPC URL the installers cannot quote" + block: + - name: Run sponsord_onramp with a public RPC URL the installers cannot quote + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_rpc_url: "https://rpc.example.invalid/$(id)" + rescue: + - name: Record onramp-rpc rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-rpc'] }}" + when: ansible_failed_task.name == "Require a public RPC URL with no credentials in it" + + - name: "Case onramp-rpc — a public RPC URL with a single quote" + block: + - name: Run sponsord_onramp with a public RPC URL with a single quote + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_rpc_url: "https://rpc.example.invalid/o'brien" + rescue: + - name: Record onramp-rpc rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-rpc'] }}" + when: ansible_failed_task.name == "Require a public RPC URL with no credentials in it" + + - name: "Case onramp-public-url — a public URL with a backslash" + block: + - name: Run sponsord_onramp with a public URL with a backslash + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_proxy: none + sponsord_onramp_public_url: "https://onramp.validation.test/a\\b" + rescue: + - name: Record onramp-public-url rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-public-url'] }}" + when: ansible_failed_task.name == "Require an https public base URL the installers can hold" + + - name: "Case onramp-sitekey — a sitekey with a space" + block: + - name: Run sponsord_onramp with a sitekey with a space + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_turnstile_sitekey: "1x00 AA" + rescue: + - name: Record onramp-sitekey rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-sitekey'] }}" + when: ansible_failed_task.name == "Validate the Turnstile sitekey and gate settings" + + - name: "Case onramp-pin — a decdn release tag without its v" + block: + - name: Run sponsord_onramp with a decdn release tag without its v + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_decdn_release: "0.1.0" + rescue: + - name: Record onramp-pin rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-pin'] }}" + when: ansible_failed_task.name == "Validate the release pins the installers use" + + - name: "Case onramp-pin — an upper-case SHA256SUMS digest" + block: + - name: Run sponsord_onramp with an upper-case SHA256SUMS digest + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_cli_sums_sha256: "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + rescue: + - name: Record onramp-pin rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-pin'] }}" + when: ansible_failed_task.name == "Validate the release pins the installers use" + + - name: "Case onramp-limit — a zero TTL" + block: + - name: Run sponsord_onramp with a zero TTL + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_ttl_secs: 0 + rescue: + - name: Record onramp-limit rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-limit'] }}" + when: ansible_failed_task.name == "Validate the onramp's limits" + + - name: "Case onramp-ip-header — a header name with a space" + block: + - name: Run sponsord_onramp with a header name with a space + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_client_ip_header: "X Forwarded For" + rescue: + - name: Record onramp-ip-header rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-ip-header'] }}" + when: ansible_failed_task.name == "Validate the client IP header name" + + - name: "Case onramp-token — no daemon API token on the host" + block: + - name: Run sponsord_onramp with no daemon API token on the host + ansible.builtin.include_role: + name: sponsord_onramp + rescue: + - name: Record onramp-token rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-token'] }}" + when: ansible_failed_task.name == "Require the daemon API token as a regular file" + + - name: "Case onramp-turnstile — no Turnstile secret on the host" + block: + - name: Stage the daemon's API token only + ansible.builtin.shell: + cmd: | + set -eu + mkdir -p /etc/sponsord + umask 077 + printf '%064d' 0 > /etc/sponsord/api-token + changed_when: true + + - name: Run sponsord_onramp with no Turnstile secret + ansible.builtin.include_role: + name: sponsord_onramp + rescue: + - name: Record onramp-turnstile rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-turnstile'] }}" + when: ansible_failed_task.name == "Require the Turnstile secret as a regular file" + always: + - name: Remove the onramp fixtures + ansible.builtin.file: + path: /etc/sponsord + state: absent + + # The lock-down must not chmod a symlink's target as root. + - name: "Case onramp-turnstile — the Turnstile secret is a symlink" + block: + - name: Stage the API token and a symlinked Turnstile secret + ansible.builtin.shell: + cmd: | + set -eu + mkdir -p /etc/sponsord + umask 077 + printf '%064d' 0 > /etc/sponsord/api-token + echo 'not-a-secret' > /root/onramp-turnstile-target + ln -sfn /root/onramp-turnstile-target /etc/sponsord/turnstile-secret + changed_when: true + + - name: Run sponsord_onramp against the symlinked Turnstile secret + ansible.builtin.include_role: + name: sponsord_onramp + rescue: + - name: Record onramp-turnstile symlink rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-turnstile'] }}" + when: ansible_failed_task.name == "Require the Turnstile secret as a regular file" + always: + - name: Remove the onramp symlink fixtures + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - /etc/sponsord + - /root/onramp-turnstile-target + + - name: "Case onramp-turnstile-empty — an empty Turnstile secret" + block: + - name: Stage the API token and an empty Turnstile secret + ansible.builtin.shell: + cmd: | + set -eu + mkdir -p /etc/sponsord + umask 077 + printf '%064d' 0 > /etc/sponsord/api-token + printf '\n' > /etc/sponsord/turnstile-secret + changed_when: true + + - name: Run sponsord_onramp with an empty Turnstile secret + ansible.builtin.include_role: + name: sponsord_onramp + rescue: + - name: Record onramp-turnstile-empty rejection (only if its check failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-turnstile-empty'] }}" + when: ansible_failed_task.name == "Report an empty Turnstile secret" + always: + - name: Remove the empty-secret fixtures + ansible.builtin.file: + path: /etc/sponsord + state: absent + - name: Confirm every bad value was rejected by the role's own validation ansible.builtin.assert: that: @@ -2115,7 +2511,7 @@ # lets one broken sibling hide behind another case that uses the same gate. - decdn_rejected | sort == _decdn_expected | sort fail_msg: >- - decdn_node, grafana_alloy and sponsord did not reject every bad value at + decdn_node, grafana_alloy, sponsord and sponsord_onramp did not reject every bad value at their validation asserts. Expected {{ _decdn_expected | sort }}, got {{ decdn_rejected | sort }} — a missing tag means that bad value slipped past validation (it would reach @@ -2147,4 +2543,10 @@ "sponsord-dir-missing", "sponsord-dir-format", "sponsord-dir-cross", "sponsord-bind", "sponsord-overwrite-untracked", "sponsord-token", "sponsord-rpc-unusable", "sponsord-secret-env-keys", "sponsord-secret-env-keys", - "sponsord-secret-env-symlink"] + "sponsord-secret-env-symlink", + "onramp-release-version", "onramp-manual-src", "onramp-network-unknown", "onramp-network-mismatch", + "onramp-zero-address", "onramp-bind", "onramp-bind", "onramp-proxy", + "onramp-domain", "onramp-public-url", "onramp-public-url", "onramp-rpc", + "onramp-rpc", "onramp-rpc", "onramp-rpc", "onramp-public-url", "onramp-sitekey", "onramp-pin", + "onramp-pin", "onramp-limit", "onramp-ip-header", "onramp-token", + "onramp-turnstile", "onramp-turnstile", "onramp-turnstile-empty"] diff --git a/ansible/molecule/validation/molecule.yml b/ansible/molecule/validation/molecule.yml index 983c1622..7e131d14 100644 --- a/ansible/molecule/validation/molecule.yml +++ b/ansible/molecule/validation/molecule.yml @@ -26,6 +26,11 @@ platforms: cgroupns_mode: host volumes: - /sys/fs/cgroup:/sys/fs/cgroup:rw + # sponsord_onramp asserts its host also runs the daemon; the cases include the + # roles directly, so membership has no other effect here. + groups: + - sponsord_hosts + - sponsord_onramp_hosts provisioner: name: ansible env: diff --git a/ansible/playbooks/group_vars/all.yml b/ansible/playbooks/group_vars/all.yml index a35a2cec..ac450b45 100644 --- a/ansible/playbooks/group_vars/all.yml +++ b/ansible/playbooks/group_vars/all.yml @@ -2,9 +2,10 @@ # Playbook-adjacent group_vars for every host (see decdn_nodes.yml beside this for # why playbook-adjacent: they load for any INVENTORY=, an overlay included). # -# Which daemons the opt-in Grafana Alloy agent scrapes, derived from the groups a -# host is in, so a sponsord-only host does not scrape a decdn-node /metrics that -# does not exist, and a co-located host gets both. Host-scoped on purpose: site.yml +# Which daemons the opt-in Grafana Alloy agent scrapes (the onramp has no /metrics: +# its toggle only labels its journal), derived from the groups a host is in, so a +# sponsord-only host does not scrape a decdn-node /metrics that does not exist, and +# a co-located host gets both. Host-scoped on purpose: site.yml # runs grafana_alloy once per play on a co-located host, and play vars would make # the two runs render different configs. # @@ -15,6 +16,7 @@ # host_vars, so override them there. grafana_alloy_node_enabled: "{{ 'decdn_nodes' in group_names }}" grafana_alloy_sponsord_enabled: "{{ 'sponsord_hosts' in group_names }}" +grafana_alloy_sponsord_onramp_enabled: "{{ 'sponsord_onramp_hosts' in group_names }}" # The public holes in the baseline firewall, from the groups a host is in. Each # group that runs baseline (decdn_nodes.yml, sponsord_hosts.yml) sets @@ -24,6 +26,14 @@ grafana_alloy_sponsord_enabled: "{{ 'sponsord_hosts' in group_names }}" # decdn_nodes: the node's QUIC listener. decdn_bind_port is a decdn_node role # default, which a play without that role (sponsord.yml) does not load; the # fallback must match roles/decdn_node/defaults/main.yml. +# sponsord_onramp_hosts: Caddy, the onramp's TLS front (80 for the ACME challenge +# and the https redirect, 443), unless sponsord_onramp_proxy is "none" (your +# own proxy, which opens its own ports). Same fallback rule: "caddy" is +# roles/sponsord_onramp/defaults/main.yml's default. _baseline_service_inbound: >- {{ ([{'proto': 'udp', 'port': decdn_bind_port | default(4433), 'comment': 'decdn QUIC'}] - if 'decdn_nodes' in group_names else []) }} + if 'decdn_nodes' in group_names else []) + + ([{'proto': 'tcp', 'port': 80, 'comment': 'sponsord-onramp (Caddy, http)'}, + {'proto': 'tcp', 'port': 443, 'comment': 'sponsord-onramp (Caddy, https)'}] + if 'sponsord_onramp_hosts' in group_names + and sponsord_onramp_proxy | default('caddy') == 'caddy' else []) }} diff --git a/ansible/playbooks/sponsord.yml b/ansible/playbooks/sponsord.yml index befb125c..001215bd 100644 --- a/ansible/playbooks/sponsord.yml +++ b/ansible/playbooks/sponsord.yml @@ -1,14 +1,18 @@ --- -# sponsord: the deCDN onboarding sponsor (treasury signer + PaymentPool keeper). -# baseline -> grafana_alloy (opt-in, off by default) -> sponsord -# Standalone entry point (`make deploy-sponsord`), also imported by site.yml. It -# needs no decdn-node on the host and opens no firewall hole: its only listener is -# loopback, for the onramp on the same host. A host in both decdn_nodes and -# sponsord_hosts runs baseline and Alloy once per play; both converge to the same -# state, because Alloy's per-daemon toggles are host-scoped -# (playbooks/group_vars/all.yml), not play-scoped. +# sponsord: the deCDN onboarding sponsor (treasury signer + PaymentPool keeper), +# and its public onramp. +# sponsord_hosts: baseline -> grafana_alloy (opt-in, off by default) -> sponsord +# sponsord_onramp_hosts: sponsord_onramp (+ Caddy in front, unless turned off) +# Standalone entry point (`make deploy-sponsord`), also imported by site.yml. The +# daemon needs no decdn-node on the host and listens on loopback only, for the +# onramp on the same host. The onramp listens on loopback too; Caddy is its public +# side (tcp/80 + tcp/443, opened by playbooks/group_vars/all.yml). A host in both +# decdn_nodes and sponsord_hosts runs baseline and Alloy once per play; both +# converge to the same state, because the firewall holes and Alloy's per-daemon +# toggles are host-scoped (playbooks/group_vars/), not play-scoped. # Opening and funding the treasury's pool is an operator action — see -# roles/sponsord/README.md. +# roles/sponsord/README.md; the Turnstile widget and the release pins are too — +# see roles/sponsord_onramp/README.md. - name: Provision sponsord hosts: sponsord_hosts become: true @@ -19,3 +23,13 @@ tags: [observability, sponsord] - role: sponsord tags: [sponsord] + +# After the daemon's play, whose /healthz gate is fatal, so the onramp starts +# against a daemon that is up. Baseline and Alloy already ran for these hosts +# above (the role asserts each one is also in sponsord_hosts). +- name: Provision sponsord-onramp + hosts: sponsord_onramp_hosts + become: true + roles: + - role: sponsord_onramp + tags: [sponsord_onramp, sponsord] diff --git a/ansible/roles/grafana_alloy/README.md b/ansible/roles/grafana_alloy/README.md index f8d02cca..789e8933 100644 --- a/ansible/roles/grafana_alloy/README.md +++ b/ansible/roles/grafana_alloy/README.md @@ -198,7 +198,7 @@ Binary/package removal is manual. | Node `/metrics` | `decdn-node` | your own dashboards/queries (`grafana_alloy_node_enabled`) | | sponsord `/metrics` | `sponsord` | pool balance, top-ups, keeper failures, issued capabilities (`grafana_alloy_sponsord_enabled`); `service_name=sponsord` | | Machine metrics | `integrations/node_exporter` | Grafana Cloud's **Linux Server** integration dashboards + alerts, unmodified | -| journald | `integrations/node_exporter` | the same integration's logs dashboards (correlated by `instance`); the daemon's own unit (`decdn-node.service`) also carries `service_name=` (default `decdn-node`), and `sponsord.service` carries `service_name=sponsord` when that scrape is on | +| journald | `integrations/node_exporter` | the same integration's logs dashboards (correlated by `instance`); the daemon's own unit (`decdn-node.service`) also carries `service_name=` (default `decdn-node`), `sponsord.service` carries `service_name=sponsord` when that scrape is on, and `sponsord-onramp.service` carries `service_name=sponsord-onramp` when `grafana_alloy_sponsord_onramp_enabled` is | | Alloy self-metrics | `integrations/alloy` | agent health | | Traces | — | Application Observability | @@ -268,6 +268,9 @@ deliberately lean and every lever is a variable: `Caused by: …`, a panic) are labelled `error`. Other unlevelled lines keep journald's level. With the toggle off, nothing re-levels its stream, so an info-dropping priority regex would drop its warnings and errors too. +- sponsord-onramp, when `grafana_alloy_sponsord_onramp_enabled` is on, logs the same + plain-text format and is treated exactly like sponsord. It has no `/metrics`, so + that toggle adds no scrape. ## Hardening: the two relaxations machine monitoring requires @@ -303,7 +306,7 @@ upstream version/sha256). Highlights: | `grafana_alloy_host_metrics_enabled` | `true` | Machine metrics (Alloy's in-process `node_exporter`) | | `grafana_alloy_host_collectors` | 13 collectors | Replacement set; `_host_extra_collectors` adds, `_host_disable_collectors` vetoes | | `grafana_alloy_host_scrape_interval` | `60s` | Host metrics only; the node keeps `30s` | -| `grafana_alloy_host_systemd_unit_include` | `(decdn-node\|sponsord\|alloy\|ssh\|sshd)\.service` | Scope of the per-unit `systemd` collector | +| `grafana_alloy_host_systemd_unit_include` | `(decdn-node\|sponsord\|sponsord-onramp\|caddy\|alloy\|ssh\|sshd)\.service` | Scope of the per-unit `systemd` collector | | `grafana_alloy_logs_enabled` | `true` | journald → Grafana Cloud Loki | | `grafana_alloy_logs_max_age` | `12h` | Bounds the catch-up burst after an outage | | `grafana_alloy_self_metrics_enabled` | `true` | Alloy's own health | @@ -315,6 +318,8 @@ upstream version/sha256). Highlights: | `grafana_alloy_node_enabled` | `true` | Scrape decdn-node's `/metrics`. This repo's playbooks set it from membership of `decdn_nodes` (`playbooks/group_vars/all.yml`) | | `grafana_alloy_sponsord_enabled` | `false` | Scrape sponsord's `/metrics`, label and re-level its journald stream. Set from membership of `sponsord_hosts` by the same file | | `grafana_alloy_sponsord_job` / `_sponsord_service_name` | `sponsord` / `sponsord` | sponsord's job label and `service_name` | +| `grafana_alloy_sponsord_onramp_enabled` | `false` | Label and re-level sponsord-onramp's journald stream (no scrape: it has no `/metrics`). Set from membership of `sponsord_onramp_hosts` by the same file | +| `grafana_alloy_sponsord_onramp_service_name` | `sponsord-onramp` | The onramp stream's `service_name` | Label variables (`service_name`, `service_namespace`, `instance_id`, `deployment_environment`, `region`) ship on most series, spans and log lines — @@ -365,10 +370,11 @@ Three layers, because the first one cannot prove correctness on its own: container with no D-Bus and no real journal, so the `systemd` collector and journald collect nothing there — expected, and not what this layer proves. 2. **`make lint-alloy`** (CI job `alloy-config`) — renders these templates in - twelve variable combinations (defaults, minimal identity, fully overridden, + thirteen variable combinations (defaults, minimal identity, fully overridden, host-metrics-only, logs-only, every sub-knob off, inventory-supplied endpoints, OTLP username fallback, vetoed collector + blanked node job, - info-dropping guardrail, sponsord-only, node + sponsord co-located) and feeds them to the REAL pinned Alloy binary: `alloy validate` + info-dropping guardrail, sponsord-only, node + sponsord co-located, and the + same with the onramp) and feeds them to the REAL pinned Alloy binary: `alloy validate` (component graph, not just syntax), a check that every `ExecStart` flag exists in `alloy run --help`, and greps proving each Jinja branch actually switched — including that "every sub-knob off" reproduces the pre-machine-monitoring diff --git a/ansible/roles/grafana_alloy/defaults/main.yml b/ansible/roles/grafana_alloy/defaults/main.yml index 574d8e3b..67e64355 100644 --- a/ansible/roles/grafana_alloy/defaults/main.yml +++ b/ansible/roles/grafana_alloy/defaults/main.yml @@ -149,6 +149,12 @@ grafana_alloy_node_enabled: true grafana_alloy_sponsord_enabled: false grafana_alloy_sponsord_job: sponsord grafana_alloy_sponsord_service_name: sponsord +# sponsord-onramp (roles/sponsord_onramp) has no /metrics: this labels its journald +# stream only, with a service_name and a `level` parsed from each line, exactly as +# for sponsord above. playbooks/group_vars/all.yml derives it from +# sponsord_onramp_hosts. +grafana_alloy_sponsord_onramp_enabled: false +grafana_alloy_sponsord_onramp_service_name: sponsord-onramp # --- Host (machine) metrics --------------------------------------------------- # Alloy's built-in node_exporter. Runs IN-PROCESS: no extra listener, no extra @@ -211,7 +217,7 @@ grafana_alloy_host_disk_device_exclude: '^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n # Per-unit state, deliberately scoped: one series set per unit on the box is a # real cost, and only these units matter. Units must match include AND NOT match # exclude. -grafana_alloy_host_systemd_unit_include: '(decdn-node|sponsord|alloy|ssh|sshd)\.service' +grafana_alloy_host_systemd_unit_include: '(decdn-node|sponsord|sponsord-onramp|caddy|alloy|ssh|sshd)\.service' grafana_alloy_host_systemd_unit_exclude: '.+\.(automount|device|mount|scope|slice)' # --- Agent self-monitoring ---------------------------------------------------- diff --git a/ansible/roles/grafana_alloy/tasks/preflight.yml b/ansible/roles/grafana_alloy/tasks/preflight.yml index 19fb7ec5..b0d9b305 100644 --- a/ansible/roles/grafana_alloy/tasks/preflight.yml +++ b/ansible/roles/grafana_alloy/tasks/preflight.yml @@ -54,9 +54,11 @@ - grafana_alloy_host_include_exporter_metrics is boolean - grafana_alloy_node_enabled is boolean - grafana_alloy_sponsord_enabled is boolean + - grafana_alloy_sponsord_onramp_enabled is boolean fail_msg: >- grafana_alloy_host_metrics_enabled, _self_metrics_enabled, _logs_enabled, - _host_include_exporter_metrics, _node_enabled and _sponsord_enabled must be + _host_include_exporter_metrics, _node_enabled, _sponsord_enabled and + _sponsord_onramp_enabled must be booleans (true/false), not quoted strings — each gates whole component groups in config.alloy. @@ -157,11 +159,13 @@ resource attributes on most series/spans/log lines, so they are restricted to a low-cardinality charset (letters, digits, dot, underscore, slash, ≤ 63 chars): {{ item }} does not fit. Offending variables: - grafana_alloy_service_name / sponsord_service_name / service_namespace / + grafana_alloy_service_name / sponsord_service_name / + sponsord_onramp_service_name / service_namespace / instance_id / deployment_environment / region. loop: - "{{ grafana_alloy_service_name }}" - "{{ grafana_alloy_sponsord_service_name }}" + - "{{ grafana_alloy_sponsord_onramp_service_name }}" - "{{ grafana_alloy_service_namespace }}" # _ga_instance_id is REQUIRED (never "") — it has a derived non-empty default. - "{{ _ga_instance_id }}" diff --git a/ansible/roles/grafana_alloy/templates/config.alloy.j2 b/ansible/roles/grafana_alloy/templates/config.alloy.j2 index fc28c527..2a72ecc5 100644 --- a/ansible/roles/grafana_alloy/templates/config.alloy.j2 +++ b/ansible/roles/grafana_alloy/templates/config.alloy.j2 @@ -301,15 +301,19 @@ loki.relabel "journal_rules" { source_labels = ["__journal__boot_id"] target_label = "boot_id" } +{# The units that log tracing-subscriber's plain text (the sponsord daemon and its + onramp), parsed for a level by loki.process "daemon_level" below. #} +{% set ga_text_units = (['sponsord'] if grafana_alloy_sponsord_enabled | bool else []) + + (['sponsord-onramp'] if grafana_alloy_sponsord_onramp_enabled | bool else []) %} {% if grafana_alloy_logs_drop_priority_regex | length > 0 %} // Cost guardrail: the regex is anchored to the WHOLE value by Prometheus // relabel semantics, so "debug" drops debug and nothing else. // - // decdn-node{% if grafana_alloy_sponsord_enabled | bool %} and sponsord are{% else %} is{% endif %} exempt: journald reports every line it writes as info, so + // decdn-node{% if ga_text_units %} and {{ ga_text_units | join(' and ') }} are{% else %} is{% endif %} exempt: journald reports every line it writes as info, so // judging it by priority would be judging it by a level it never set (a // regex containing "info" would drop its errors). loki.process - // "daemon_level" below applies the same regex to its real{% if not grafana_alloy_sponsord_enabled | bool %}, JSON{% endif %} level. The + // "daemon_level" below applies the same regex to its real{% if not ga_text_units %}, JSON{% endif %} level. The // exemption blanks a scratch copy of the keyword for that one unit; like // every __-prefixed label it never leaves this ruleset. rule { @@ -319,7 +323,7 @@ loki.relabel "journal_rules" { rule { source_labels = ["__journal__systemd_unit"] - regex = {{ ('(?:decdn-node|sponsord)\\.service' if grafana_alloy_sponsord_enabled | bool else 'decdn-node\\.service') | tojson }} + regex = {{ ('(?:' ~ (['decdn-node'] + ga_text_units) | join('|') ~ ')\\.service') | tojson }} target_label = "__priority_guardrail" replacement = "" } @@ -457,13 +461,14 @@ loki.process "daemon_level" { } {{ ga_daemon_level_stages() }} } -{% if grafana_alloy_sponsord_enabled | bool %} +{% if ga_text_units %} +{% set ga_text_selector = '{unit=~"(' ~ ga_text_units | join('|') ~ ')\\\\.service"}' %} - // sponsord logs tracing-subscriber's plain-text format with colour off - // (NO_COLOR=1 in its unit): " WARN : ". The + // {{ ga_text_units | join(' and ') }} log{% if ga_text_units | length == 1 %}s{% endif %} tracing-subscriber's plain-text format with colour off + // (NO_COLOR=1 in the units): " WARN : ". The // level is the second whitespace-separated field. stage.match { - selector = `{unit="sponsord.service"}` + selector = `{{ ga_text_selector }}` stage.regex { expression = `^\S+\s+(?PTRACE|DEBUG|INFO|WARN|ERROR)\s` @@ -471,12 +476,12 @@ loki.process "daemon_level" { {{ ga_daemon_level_stages() }} } - // What sponsord writes raw when it exits on an error carries no level field: + // What these write raw when they exit on an error carries no level field: // anyhow's "Error: ..." and "Caused by:" lines, and a panic message. They are // the lines that say WHY it stopped, so they are errors. Any other unlevelled // line keeps its journald level. stage.match { - selector = `{unit="sponsord.service"} |~ "^(Error: |Caused by:|thread '.+' panicked)"` + selector = `{{ ga_text_selector }} |~ "^(Error: |Caused by:|thread '.+' panicked)"` stage.template { source = "decdn_level" @@ -542,6 +547,17 @@ loki.relabel "journal_identity" { replacement = {{ grafana_alloy_sponsord_service_name | tojson }} } {% endif %} +{% if grafana_alloy_sponsord_onramp_enabled | bool and grafana_alloy_sponsord_onramp_service_name | length > 0 %} +{# The onramp's stream (the unit roles/sponsord_onramp installs). It has no + /metrics, so this and the daemon_level parsing above are all it gets. #} + + rule { + source_labels = ["unit"] + regex = "sponsord-onramp\\.service" + target_label = "service_name" + replacement = {{ grafana_alloy_sponsord_onramp_service_name | tojson }} + } +{% endif %} } loki.write "cloud" { diff --git a/ansible/roles/sponsord/README.md b/ansible/roles/sponsord/README.md index d7d544d1..696d4929 100644 --- a/ansible/roles/sponsord/README.md +++ b/ansible/roles/sponsord/README.md @@ -11,8 +11,9 @@ It is independent of the node. It needs an RPC endpoint and the PaymentPool cont never a local `decdn-node`, so it can run on its own host or beside a node. The role covers the daemon only. Its public companion, `sponsord-onramp` (Turnstile -gate, installers, behind a TLS reverse proxy), is not deployed here yet. The onramp -expects the daemon on the same host and reads the API token from +gate, installers, behind a TLS reverse proxy), is the +[`sponsord_onramp`](../sponsord_onramp/README.md) role: list the host in +`sponsord_onramp_hosts` too. It runs on the same host and reads the API token from `/etc/sponsord/api-token`. ## What it does @@ -29,7 +30,7 @@ expects the daemon on the same host and reads the API token from | File | Who provides it | |------|-----------------| - | `api-token` | The role generates it on the host when absent and never replaces it, because the onramp holds a copy. | + | `api-token` | The role generates it on the host when absent and never replaces it, because the onramp (or your own gate) uses it. | | `treasury-keystore.json`, `treasury-password` | **You** do. The role never creates the treasury wallet. | | `secret.env` (`SPONSORD_RPC_URL`) | `sponsord_rpc_url` from inventory, or a file you write on the host. The two-way rules match `decdn_rpc_url`: an empty `sponsord_rpc_url` with a file the role wrote earlier fails the deploy, because that means `secret.yml` went missing. A host-written file may hold `SPONSORD_RPC_URL` **only**: as an `EnvironmentFile` it would override every other setting, so the role refuses any other key in it. | @@ -112,8 +113,9 @@ yourself. ## Day 2 -- **Rotate the API token:** write the new token to `/etc/sponsord/api-token` and to - the onramp, then re-run the role, which restarts sponsord. +- **Rotate the API token:** write the new token to `/etc/sponsord/api-token`, then + re-run the role, which restarts sponsord. The onramp reads the same file and + restarts with the daemon (`PartOf=`). - **Rotate the treasury:** replace the keystore and password files, then re-run. The new wallet must own `sponsord_pool_id`, or sponsord refuses to start and the deploy fails at `/healthz`. diff --git a/ansible/roles/sponsord_onramp/README.md b/ansible/roles/sponsord_onramp/README.md new file mode 100644 index 00000000..2719b012 --- /dev/null +++ b/ansible/roles/sponsord_onramp/README.md @@ -0,0 +1,182 @@ +# Role: `sponsord_onramp` + +Deploys `sponsord-onramp` from [decdn/sponsord](https://github.com/decdn/sponsord), +the public side of the deCDN onboarding sponsor. It serves: + +- the gate page people pass (Cloudflare Turnstile) before the sponsor pays for + their download; +- the `decdn.sh` / `decdn.ps1` installers, which install `decdn` and + `decdn-sponsored` from pinned releases; +- the API the `decdn-sponsored` CLI polls for its capability. + +It holds no wallet. For each person who passes the gate it asks the +[`sponsord`](../sponsord/README.md) daemon on the same host for a capability, using +the daemon's API token. So the host must be in `sponsord_hosts` as well as +`sponsord_onramp_hosts`, and the role checks that it is. + +## What it does + +- **Installs the binary**, exactly as the `sponsord` role does: + - `manual` (the default) copies a binary built in a `decdn/sponsord` checkout. + `sponsord_onramp_release_target_dir` defaults to `sponsord_release_target_dir`, + so one `cargo build --release` covers both. + - `release` downloads `sponsord-onramp-v` and verifies it against the + GPG-signed `SHA256SUMS`, with the KEYS file vendored in the `sponsord` role. + - Upstream has cut no release yet, so `manual` is the only working method today. +- **Reads two secrets, writes none.** Both stay root `0600` under `/etc/sponsord` + and reach the onramp as `LoadCredential=` credentials, never as environment: + + | File | Who provides it | + |------|-----------------| + | `api-token` | The `sponsord` role (generated on the host). The onramp uses the daemon's own file. | + | `turnstile-secret` | **You** do. The role only checks it is a non-empty regular file and locks it to `0600`. | + +- **Writes non-secret settings** to `/etc/sponsord/sponsord-onramp.env` (0644): the + public URL, the public RPC URL, CapacityBond and SlashJudge, the sitekey, the + release pins and any limits you set. +- **Runs a hardened unit.** `sponsord-onramp.service` runs with `DynamicUser=`, no + capabilities and a read-only filesystem. + - `PartOf=sponsord.service`: a daemon restart restarts the onramp, because the + onramp reads the daemon's limits only at start. + - It exits at start while the daemon is unreachable, so it retries every 5 s + with no start limit. +- **Gates the deploy on `/healthz`.** The onramp binds only after it has reached the + daemon and validated its settings (the release pins, its cap and TTL against the + daemon's maximum). If `/healthz` does not answer 200 within the readiness window, + the deploy fails. +- **Restarts on out-of-band changes.** As in the `sponsord` role, a hash record of + every restart input (the token, the Turnstile secret, the env file, the gate page, + the unit, the binary) restarts the onramp when one changes outside the role. +- **Puts Caddy in front** (`sponsord_onramp_proxy: caddy`, the default): + - Installs the distribution's `caddy` package. No third-party apt repository is + added. + - Writes `/etc/caddy/Caddyfile`: TLS for `sponsord_onramp_domain`, then a reverse + proxy to the onramp on loopback. The file is validated with `caddy validate` + before it lands, and the admin API is off, so nothing on the host can + reconfigure the proxy. + - Opens tcp/80 (ACME challenge, https redirect) and tcp/443, through + `playbooks/group_vars/all.yml`. + - Probes `https:///healthz` through Caddy on the host. With ACME a failure + only warns, since the certificate can lag DNS. With `tls internal` it fails the + deploy. + - Refuses to replace a Caddyfile it did not write on a host where Caddy was + already installed, unless you set `sponsord_onramp_caddy_overwrite_config: true`. + +## Before the first deploy + +1. **Deploy `sponsord`** on the host (see [its README](../sponsord/README.md)). The + same `make deploy-sponsord` run can do both. +2. **Create a Turnstile widget** for your domain in the Cloudflare dashboard. Note + its sitekey (public) and secret. +3. **Copy the secret onto the host**, as root: + + ```bash + umask 077 + printf '%s' '' | sudo tee /etc/sponsord/turnstile-secret >/dev/null + sudo chmod 600 /etc/sponsord/turnstile-secret + ``` + +4. **Point DNS** for the domain (A/AAAA) at the host, so Caddy can get a + certificate. +5. **Pick the releases the installers install.** For `decdn`, a `vX.Y.Z` tag and the + SHA-256 of that release's `SHA256SUMS`. For `decdn-sponsored`, a + `decdn-sponsored-vX.Y.Z` tag and the same digest; both values are printed in its + release notes. +6. **Set inventory** (`group_vars/sponsord_onramp_hosts.yml`): the domain, the + public RPC URL, the sitekey and the four release-pin values. The contracts come + from `sponsord_network`. + +Then run `make deploy-sponsord` from `ansible/`. + +## Variables + +See [`defaults/main.yml`](defaults/main.yml) for the full list with comments. + +| Variable | Default | Notes | +|----------|---------|-------| +| `sponsord_onramp_install_method` | `manual` | `manual` or `release` | +| `sponsord_onramp_manual_bin_src` / `_release_target_dir` | `""` / `sponsord_release_target_dir` | `manual` only. | +| `sponsord_onramp_version` | `""` | Required in `release` mode, e.g. `0.1.0` → tag `sponsord-onramp-v0.1.0`. | +| `sponsord_onramp_network` | `sponsord_network` | Supplies CapacityBond and SlashJudge. Must match the daemon's network. | +| `sponsord_onramp_capacity_bond_address`, `_slash_judge_address` | from profile | Explicit values win. SlashJudge may be `""`. | +| `sponsord_onramp_rpc_url` | `""` | **Required. Public**: served to every user. No credentials, query or API key. | +| `sponsord_onramp_proxy` | `caddy` | `caddy` or `none`. Set it in inventory: the firewall holes read it. | +| `sponsord_onramp_domain` | `""` | Required with Caddy. | +| `sponsord_onramp_public_url` | `https://` | Required with `none`: the https URL your proxy serves. | +| `sponsord_onramp_caddy_tls` | `acme` | `internal` uses Caddy's local CA (staging, tests). | +| `sponsord_onramp_acme_email` | `""` | Optional ACME contact. | +| `sponsord_onramp_caddy_overwrite_config` | `false` | Let the role take over a Caddyfile it did not write. | +| `sponsord_onramp_client_ip_header` | `X-Forwarded-For` with Caddy, else `""` | With your own proxy, set it only if that proxy is the only way in. | +| `sponsord_onramp_turnstile_sitekey` | `""` | **Required.** | +| `sponsord_onramp_decdn_release`, `_decdn_sums_sha256` | `""` | **Required.** `vX.Y.Z` and 64 lowercase hex. | +| `sponsord_onramp_cli_release`, `_cli_sums_sha256` | `""` | **Required.** `decdn-sponsored-vX.Y.Z` and 64 lowercase hex. | +| `sponsord_onramp_min_cli_version` | `""` | Older `decdn-sponsored` CLIs are told to re-run the installer. | +| `sponsord_onramp_brand_name`, `_gate_template_src` | `""` | The gate page's name, or your own HTML page (a control-machine file). | +| `sponsord_onramp_spending_cap_micro_usdc`, `_ttl_secs` | `""` | Requested per capability. `""` takes the daemon's maximum; more than it fails the deploy. | +| `sponsord_onramp_fund_rate_per_min`, `_poll_rate_per_min` | `""` | Per-address rate limits (upstream 10 and 120; `0` turns one off). | +| `sponsord_onramp_bind_address` / `_port` | `127.0.0.1` / `8080` | IPv4 loopback only (asserted). | +| `sponsord_onramp_readiness_retries` / `_delay` | `30` / `2` | `/healthz` window (about 60 s). | + +## Bringing your own proxy + +With `sponsord_onramp_proxy: none` the role installs no Caddy and opens no port. +Your proxy must terminate TLS for `sponsord_onramp_public_url` and forward to +`127.0.0.1:8080`. Set `sponsord_onramp_client_ip_header` only if the proxy is the +only way to reach the onramp: `CF-Connecting-IP` behind Cloudflare, or +`X-Forwarded-For` if your proxy appends the client address. Otherwise clients can +choose their own address and get around the rate limits. Switching from `caddy` +to `none` closes the firewall holes but leaves the Caddy package installed and +running; remove it yourself. + +## Observability + +With `decdn_grafana_cloud_enabled: true`, the `grafana_alloy` role ships the +onramp's journal with `service_name="sponsord-onramp"` and a `level` parsed from each +line (it has no `/metrics`). This repo's playbooks turn that on for hosts in +`sponsord_onramp_hosts`. From your own playbook, set +`grafana_alloy_sponsord_onramp_enabled`. Caddy's unit state is in the systemd +collector; its access logs are not shipped. + +## Day 2 + +- **Rotate the Turnstile secret:** write the new one to + `/etc/sponsord/turnstile-secret` and re-run the role. It restarts the onramp. +- **Ship a new CLI:** point `sponsord_onramp_cli_release` / `_cli_sums_sha256` (and + the `decdn` pair) at the new release and re-run. New installs get it; set + `sponsord_onramp_min_cli_version` to make existing users re-run the installer. +- **Certificate trouble:** `sudo journalctl -u caddy -e`. Check that the domain + resolves to the host and that tcp/80 and tcp/443 reach it. +- **Logs:** `journalctl -u sponsord-onramp`. +- **Removal:** + + ```bash + sudo systemctl disable --now sponsord-onramp caddy + sudo rm /etc/systemd/system/sponsord-onramp.service + sudo systemctl daemon-reload + sudo rm -f /usr/local/bin/sponsord-onramp /etc/sponsord/sponsord-onramp.env \ + /etc/sponsord/onramp-gate.html /etc/sponsord/.onramp-inputs.sha256 \ + /etc/sponsord/turnstile-secret + sudo rm -rf /usr/local/lib/sponsord-onramp + sudo apt-get remove caddy + ``` + + Then take the host out of `sponsord_onramp_hosts`, so the next deploy closes + tcp/80 and tcp/443. + +## Testing + +- `molecule/sponsord-onramp` runs `playbooks/sponsord.yml` (daemon, then onramp) on + Debian 12 and Ubuntu 24.04, against stub binaries and the real distro Caddy with + `tls internal`. It checks: + - idempotence; + - the `https://` chain through Caddy, and that a client-sent `X-Forwarded-For` + cannot choose the address the onramp sees; + - that a Turnstile rotation restarts the onramp and not the daemon; + - that the `/healthz` gate fails the deploy when the onramp will not start; + - the refusal of a foreign Caddyfile, and the confirmed takeover. +- `molecule/validation` holds the negative cases. +- `tests/firewall-holes` (`make test-scripts`) pins the tcp/80 and tcp/443 holes. +- `tests/alloy-config` (`make lint-alloy`) runs the onramp's log lines through the + real Alloy. +- Release mode is not exercised here. Its tasks mirror the `sponsord` role's, which + `molecule/sponsord` tests against a signed mirror. diff --git a/ansible/roles/sponsord_onramp/defaults/main.yml b/ansible/roles/sponsord_onramp/defaults/main.yml new file mode 100644 index 00000000..5b93a425 --- /dev/null +++ b/ansible/roles/sponsord_onramp/defaults/main.yml @@ -0,0 +1,137 @@ +--- +# sponsord_onramp role defaults. sponsord-onramp is the public side of the deCDN +# onboarding sponsor: the gate page people pass (Cloudflare Turnstile), the +# decdn.sh / decdn.ps1 installers, and the API the decdn-sponsored CLI polls. It +# holds no wallet. It calls the sponsord daemon on the same host with the daemon's +# API token, so a host in sponsord_onramp_hosts must also be in sponsord_hosts. +# +# The onramp listens on loopback. Caddy, installed by this role unless you turn it +# off, terminates TLS for sponsord_onramp_domain and is the only public listener +# (AGENTS.md hard rule 2). Upstream: github.com/decdn/sponsord, docs/operator.md. + +# --- Install method ----------------------------------------------------------- +# "release": download the pinned sponsord-onramp-v GitHub Release tarball +# and verify it against the GPG-signed SHA256SUMS. "manual": copy a locally-built +# binary from the control machine. Upstream has cut NO sponsord-onramp-v* tag yet, +# so "manual" is the default. +sponsord_onramp_install_method: manual # "release" | "manual" +sponsord_onramp_manual_bin_src: "" # manual: control-machine path to the binary +# Alternative to sponsord_onramp_manual_bin_src: the control-machine `target/release` +# dir of a decdn/sponsord checkout (falls back to target//release). The role +# checks the binary is an ELF for the host's architecture before shipping it. +sponsord_onramp_release_target_dir: "{{ sponsord_release_target_dir | default('') }}" + +# --- Release to install (REQUIRED in "release" mode) --------------------------- +sponsord_onramp_version: "" # e.g. "0.1.0" -> tag sponsord-onramp-v0.1.0 +# Derived from the host's gathered architecture (vars/main/arch.yml). +sponsord_onramp_target: "{{ _sponsord_onramp_arch_targets.get(ansible_facts.get('architecture', ''), '') }}" +sponsord_onramp_release_base: "https://github.com/decdn/sponsord/releases/download" +sponsord_onramp_verify_release_signature: true +# Upstream signs every decdn/sponsord release with the keys in its one KEYS file, +# vendored once, in the sponsord role. Both roles ship side by side (this repo and +# the decdn.node collection), so the path holds in either. +sponsord_onramp_release_keyring: "{{ role_path }}/../sponsord/files/sponsord-release-KEYS.asc" + +# --- Chain ---------------------------------------------------------------------- +# The onramp takes the chain id and the PaymentPool from the daemon (/v1/info). It +# hands users' decdn the CapacityBond (node discovery) and, if set, the SlashJudge +# contract. With a network profile both come from upstream's deployment manifest, +# mirrored into vars/main/networks.yml by scripts/sync-network-profiles.py; an +# explicit inventory value still wins. Defaults to the daemon's sponsord_network. +sponsord_onramp_network: "{{ sponsord_network | default('') }}" +# Internal: set sponsord_onramp_network, not this. +_sponsord_onramp_profile: "{{ _sponsord_onramp_network_profiles.get(sponsord_onramp_network, {}) }}" +sponsord_onramp_capacity_bond_address: "{{ _sponsord_onramp_profile.get('capacity_bond_address', '') }}" +# "" omits it. +sponsord_onramp_slash_judge_address: "{{ _sponsord_onramp_profile.get('slash_judge_address', '') }}" + +# REQUIRED. The JSON-RPC endpoint users' decdn reads the chain through. NOT a +# secret, and it must not hold one: the onramp publishes it to every user +# (GET /v1/profile) and bakes it into the installers. Use a public endpoint, never +# the keyed URL the daemon uses (sponsord_rpc_url). +sponsord_onramp_rpc_url: "" + +# --- Listener ------------------------------------------------------------------- +# IPv4 loopback only (asserted): Caddy, or your own proxy, is the public side. +sponsord_onramp_bind_address: 127.0.0.1 +sponsord_onramp_port: 8080 +# The daemon, on this host. Follows the sponsord role's listener when inventory +# sets it; the fallbacks match roles/sponsord/defaults/main.yml. +sponsord_onramp_daemon_url: >- + http://{{ sponsord_bind_address | default('127.0.0.1') }}:{{ sponsord_port | default(8090) }} + +# --- Public address + TLS front ------------------------------------------------- +# "caddy": install Caddy from the distro and let it terminate TLS for +# sponsord_onramp_domain (ACME by default), opening tcp/80 and tcp/443. +# "none": bring your own proxy (nginx, a Cloudflare Tunnel, ...); no package, no +# firewall hole, and sponsord_onramp_public_url is required. +# playbooks/group_vars/all.yml reads this to open the ports, so set it in +# inventory, not in a play's vars. +sponsord_onramp_proxy: caddy # "caddy" | "none" +sponsord_onramp_domain: "" # caddy: REQUIRED, e.g. downloads.example.org +# The base URL users reach, baked into the installers and the gate links. https +# only. Derived from the domain in caddy mode. +sponsord_onramp_public_url: >- + {{ ('https://' ~ sponsord_onramp_domain) if sponsord_onramp_proxy == 'caddy' else '' }} +# "acme": a publicly trusted certificate (Let's Encrypt / ZeroSSL). The domain's +# A/AAAA records must point at this host and tcp/80 + tcp/443 must reach it. +# "internal": Caddy's own local CA, for staging and tests; browsers will not trust it. +sponsord_onramp_caddy_tls: acme # "acme" | "internal" +sponsord_onramp_acme_email: "" # optional ACME account contact +# The role owns /etc/caddy/Caddyfile. On a host where Caddy was installed before +# this role and that file lacks the role's marker, the role refuses to replace it +# unless this is true. +sponsord_onramp_caddy_overwrite_config: false +sponsord_onramp_caddyfile: /etc/caddy/Caddyfile +# The header Caddy (or your proxy) puts the client address in; the onramp rate +# limits and gates by it. Caddy without trusted_proxies replaces any incoming +# X-Forwarded-For, so the right-most address is the real client. With your own +# proxy, set it only if that proxy is the ONLY way in (CF-Connecting-IP behind +# Cloudflare); otherwise clients pick their own address. "" = the TCP peer. +sponsord_onramp_client_ip_header: "{{ 'X-Forwarded-For' if sponsord_onramp_proxy == 'caddy' else '' }}" + +# --- Gate ----------------------------------------------------------------------- +# Cloudflare Turnstile: create a widget for sponsord_onramp_domain in the +# Cloudflare dashboard. The sitekey is public; the secret is provisioned on the +# host (0600 root) and reaches the onramp as a systemd credential. +sponsord_onramp_turnstile_sitekey: "" # REQUIRED +sponsord_onramp_brand_name: "" # [deCDN] name on the gate page +# A control-machine HTML file served as the gate page instead of the built-in one +# (placeholders: upstream docs/operator.md). "" = the built-in page. +sponsord_onramp_gate_template_src: "" + +# --- Releases the installers install (REQUIRED) ------------------------------- +# A tag and the SHA-256 of that release's SHA256SUMS file. Both values for +# decdn-sponsored are printed in its release notes (upstream RELEASING.md). +sponsord_onramp_decdn_release: "" # decdn/decdn tag, e.g. v0.1.0 +sponsord_onramp_decdn_sums_sha256: "" # 64 lowercase hex +sponsord_onramp_cli_release: "" # e.g. decdn-sponsored-v0.1.0 +sponsord_onramp_cli_sums_sha256: "" # 64 lowercase hex +# Oldest decdn-sponsored accepted; older CLIs are told to re-run the installer. +sponsord_onramp_min_cli_version: "" +sponsord_onramp_releases_base: "" # [https://github.com/decdn] + +# --- Limits ("" = the onramp's own default, shown in brackets) ----------------- +sponsord_onramp_spending_cap_micro_usdc: "" # [the daemon's maximum] cap per capability +sponsord_onramp_ttl_secs: "" # [the daemon's maximum] TTL per capability +sponsord_onramp_fund_rate_per_min: "" # [10] POST /v1/fund per address; 0 = off +sponsord_onramp_poll_rate_per_min: "" # [120] GET /v1/capability per address; 0 = off + +# --- Files on the host ---------------------------------------------------------- +# Under the daemon's directory: the onramp reads the daemon's own API token. +sponsord_onramp_etc: "{{ sponsord_etc | default('/etc/sponsord') }}" +sponsord_onramp_api_token_file: "{{ sponsord_api_token_file | default('/etc/sponsord/api-token') }}" +sponsord_onramp_turnstile_secret_file: /etc/sponsord/turnstile-secret # operator-provisioned (0600) +sponsord_onramp_env_file: /etc/sponsord/sponsord-onramp.env # non-secret (0644, role-owned) +sponsord_onramp_gate_template_file: /etc/sponsord/onramp-gate.html # when _src is set (0644) +# sha256 record of every restart input (token, Turnstile secret, env file, gate +# page, unit, binary) as of the last converge that saw the onramp healthy. +sponsord_onramp_inputs_checksum_file: /etc/sponsord/.onramp-inputs.sha256 + +sponsord_onramp_bin: /usr/local/bin/sponsord-onramp + +# --- Readiness -------------------------------------------------------------------- +# The onramp binds only after reading the daemon's /v1/info, so /healthz answering +# 200 is the config gate. A timeout FAILS the deploy. Window ~60s. +sponsord_onramp_readiness_retries: 30 +sponsord_onramp_readiness_delay: 2 diff --git a/ansible/roles/sponsord_onramp/handlers/main.yml b/ansible/roles/sponsord_onramp/handlers/main.yml new file mode 100644 index 00000000..604b04c7 --- /dev/null +++ b/ansible/roles/sponsord_onramp/handlers/main.yml @@ -0,0 +1,19 @@ +--- +# Each handler reloads systemd itself rather than relying on a separate "Reload +# systemd" handler running first (see roles/sponsord/handlers/main.yml). +- name: Restart sponsord-onramp + ansible.builtin.systemd: + name: sponsord-onramp + state: restarted + daemon_reload: true + +# Not a reload: the role's Caddyfile turns the admin API off, which `caddy reload` +# (the packaged unit's ExecReload) needs. +# Under --check on a host without Caddy its install is only simulated, so there +# is no unit to restart (tasks/caddy.yml guards its start the same way). +- name: Restart caddy + ansible.builtin.systemd: + name: caddy + state: restarted + daemon_reload: true + when: not ansible_check_mode or 'caddy' in ansible_facts.packages diff --git a/ansible/roles/sponsord_onramp/meta/main.yml b/ansible/roles/sponsord_onramp/meta/main.yml new file mode 100644 index 00000000..bbd70a3b --- /dev/null +++ b/ansible/roles/sponsord_onramp/meta/main.yml @@ -0,0 +1,17 @@ +--- +galaxy_info: + role_name: sponsord_onramp + author: deCDN Contributors + description: >- + deCDN sponsord-onramp, the public gate in front of sponsord — release-tarball or + manual local-binary install, hardened DynamicUser systemd unit with credentials, + loopback listener behind an optional Caddy TLS front. + license: MIT + min_ansible_version: "2.15" + galaxy_tags: [decdn, sponsord, caddy, systemd, web3, debian] + platforms: + - name: Debian + versions: [bookworm, trixie] + - name: Ubuntu + versions: [noble, resolute] +dependencies: [] diff --git a/ansible/roles/sponsord_onramp/tasks/caddy.yml b/ansible/roles/sponsord_onramp/tasks/caddy.yml new file mode 100644 index 00000000..d73bbe6b --- /dev/null +++ b/ansible/roles/sponsord_onramp/tasks/caddy.yml @@ -0,0 +1,85 @@ +--- +# Caddy in front of the onramp: the host's only public HTTP listener (tcp/80 for +# the ACME challenge and the https redirect, tcp/443). playbooks/group_vars/all.yml +# opens both in the baseline firewall for hosts in sponsord_onramp_hosts while +# sponsord_onramp_proxy is "caddy". The distro package brings its own hardened unit +# (user caddy, CAP_NET_BIND_SERVICE only); main.yml has already refused a Caddy +# this role does not own. + +# curl: the probe at the end of this file. +- name: Install Caddy from the distribution + ansible.builtin.apt: + name: [caddy, curl] + state: present + update_cache: true + cache_valid_time: 3600 + +# `caddy validate` loads the config the way `caddy run` does (it also provisions +# the TLS automation), so a Caddyfile that would not start never lands. +- name: Template the Caddyfile + ansible.builtin.template: + src: Caddyfile.j2 + dest: "{{ sponsord_onramp_caddyfile }}" + owner: root + group: root + mode: "0644" + validate: caddy validate --adapter caddyfile --config %s + notify: Restart caddy + +- name: Apply pending Caddy changes + ansible.builtin.meta: flush_handlers + +# Under --check on a host without Caddy the install above is only simulated, so +# there is no unit to start yet. +- name: Enable and start Caddy + ansible.builtin.systemd: + name: caddy + enabled: true + state: started + when: not ansible_check_mode or 'caddy' in ansible_facts.packages + +# The whole chain, as a user would reach it but pinned to this host: TLS for the +# domain, the proxy, the onramp. With ACME the certificate can lag (DNS not yet +# pointed here, a rate limit), so that probe only warns; with Caddy's internal CA +# nothing outside the host is involved, so it is fatal. +- name: Probe the onramp through Caddy + ansible.builtin.command: + argv: >- + {{ ['curl', '--silent', '--show-error', '--fail', '--max-time', '5', + '--retry', sponsord_onramp_readiness_retries | string, + '--retry-delay', sponsord_onramp_readiness_delay | string, + '--retry-max-time', '60', '--retry-all-errors', + '--resolve', sponsord_onramp_domain ~ ':443:127.0.0.1'] + + (['--insecure'] if sponsord_onramp_caddy_tls == 'internal' else []) + + ['https://' ~ sponsord_onramp_domain ~ '/healthz'] }} + register: sponsord_onramp_caddy_probe + changed_when: false + failed_when: false + when: not ansible_check_mode + +- name: Report an onramp that is not reachable through Caddy + ansible.builtin.fail: + msg: |- + https://{{ sponsord_onramp_domain }}/healthz did not answer through Caddy on this + host (curl exit {{ sponsord_onramp_caddy_probe.rc }}: {{ sponsord_onramp_caddy_probe.stderr | trim }}). + The onramp itself is healthy on loopback, so the fault is in Caddy: + sudo journalctl -u caddy -e + when: + - not ansible_check_mode + - sponsord_onramp_caddy_tls == 'internal' + - sponsord_onramp_caddy_probe.rc != 0 + +- name: Warn that the public certificate is not in place yet + ansible.builtin.debug: + msg: |- + WARNING: https://{{ sponsord_onramp_domain }}/healthz did not answer through Caddy + on this host yet (curl exit {{ sponsord_onramp_caddy_probe.rc }}: + {{ sponsord_onramp_caddy_probe.stderr | trim }}). With ACME that usually means the + certificate is not issued yet: check that the domain's A/AAAA records point at + this host and that tcp/80 and tcp/443 reach it, then read + sudo journalctl -u caddy -e + Users cannot reach the onramp until this answers. + when: + - not ansible_check_mode + - sponsord_onramp_caddy_tls == 'acme' + - sponsord_onramp_caddy_probe.rc != 0 diff --git a/ansible/roles/sponsord_onramp/tasks/install.yml b/ansible/roles/sponsord_onramp/tasks/install.yml new file mode 100644 index 00000000..ffb8ddc4 --- /dev/null +++ b/ansible/roles/sponsord_onramp/tasks/install.yml @@ -0,0 +1,257 @@ +--- +# Install the sponsord-onramp binary. Two methods, selected by sponsord_onramp_install_method: +# "release": download the pinned sponsord-onramp-v GitHub Release tarball, +# verify it against the GPG-signed SHA256SUMS manifest, then extract. +# Idempotent and upgrade-aware via a version stamp. +# "manual" (default today): copy a locally-built binary from the control machine. +# The verification steps mirror roles/decdn_node/tasks/install.yml, which carries +# the full rationale for each (status-parsed gpg, exact manifest entry, private +# staging dir kept on failure). + +- name: Install from the pinned GitHub Release tarball + when: sponsord_onramp_install_method == "release" + block: + - name: Name the release archive + ansible.builtin.set_fact: + # The archive holds ONE flat binary at its root (upstream release.yml). + sponsord_onramp_release_archive: >- + sponsord-onramp-{{ sponsord_onramp_version }}-{{ sponsord_onramp_target }}.tar.gz + sponsord_onramp_release_url: "{{ sponsord_onramp_release_base }}/sponsord-onramp-v{{ sponsord_onramp_version }}" + + - name: Read the installed-version stamp + ansible.builtin.slurp: + src: /usr/local/lib/sponsord-onramp/installed-version + register: sponsord_onramp_stamp + failed_when: false + changed_when: false + + - name: Decide whether (re)install is needed + ansible.builtin.set_fact: + sponsord_onramp_need_install: >- + {{ (sponsord_onramp_stamp.content is not defined) + or ((sponsord_onramp_stamp.content | b64decode | trim) != sponsord_onramp_version) }} + + - name: Report that the release download is not simulated in check mode + ansible.builtin.debug: + msg: >- + check mode: skipping the download + signature verification of sponsord-onramp + {{ sponsord_onramp_version }}. The config and unit below still diff normally. + when: + - sponsord_onramp_need_install + - ansible_check_mode + + - name: Install sponsord-onramp {{ sponsord_onramp_version }} + when: + - sponsord_onramp_need_install + - not ansible_check_mode + block: + - name: Create a private download staging directory + ansible.builtin.tempfile: + state: directory + prefix: sponsord-onramp-install- + register: sponsord_onramp_stage + changed_when: false + + - name: Restrict the staging directory to root + ansible.builtin.file: + path: "{{ sponsord_onramp_stage.path }}" + state: directory + owner: root + group: root + mode: "0700" + + - name: WARN that release signature verification is DISABLED + ansible.builtin.debug: + msg: |- + sponsord_onramp_verify_release_signature is false. SHA256SUMS is fetched + UNAUTHENTICATED from {{ sponsord_onramp_release_url }}, so the checksum check + proves only that the download was not corrupted — NOT that the binary + came from deCDN. This binary holds the daemon API token. + when: not sponsord_onramp_verify_release_signature + + - name: Ensure gnupg is available for signature verification + ansible.builtin.package: + name: gnupg + state: present + when: sponsord_onramp_verify_release_signature + + - name: Download the release tarball + ansible.builtin.get_url: + url: "{{ sponsord_onramp_release_url }}/{{ sponsord_onramp_release_archive }}" + dest: "{{ sponsord_onramp_stage.path }}/{{ sponsord_onramp_release_archive }}" + owner: root + group: root + mode: "0600" + + - name: Download the SHA256SUMS manifest and signature + ansible.builtin.get_url: + url: "{{ sponsord_onramp_release_url }}/{{ item }}" + dest: "{{ sponsord_onramp_stage.path }}/{{ item }}" + owner: root + group: root + mode: "0600" + loop: "{{ ['SHA256SUMS', 'SHA256SUMS.asc'] if sponsord_onramp_verify_release_signature else ['SHA256SUMS'] }}" + + - name: Create the keyring staging directory + ansible.builtin.file: + path: "{{ sponsord_onramp_stage.path }}/keyring" + state: directory + owner: root + group: root + mode: "0700" + when: sponsord_onramp_verify_release_signature + + - name: Stage the release signing keyring + ansible.builtin.copy: + src: "{{ sponsord_onramp_release_keyring }}" + dest: "{{ sponsord_onramp_stage.path }}/keyring/KEYS.asc" + owner: root + group: root + mode: "0600" + when: sponsord_onramp_verify_release_signature + + # Isolated GNUPGHOME; VALIDSIG required, an expired/revoked key refused, + # --no-autostart so no gpg-agent outlives the task (see decdn_node). + - name: Verify the SHA256SUMS signature against the release keyring + ansible.builtin.shell: + chdir: "{{ sponsord_onramp_stage.path }}" + executable: /bin/bash + cmd: | + set -euo pipefail + export GNUPGHOME="{{ sponsord_onramp_stage.path }}/gnupg" + mkdir -p "$GNUPGHOME" + chmod 700 "$GNUPGHOME" + gpg --batch --quiet --no-autostart --import keyring/KEYS.asc + gpg --batch --no-autostart --status-fd 3 --verify SHA256SUMS.asc SHA256SUMS \ + 3>gpg-status.txt + if grep -qE '^\[GNUPG:\] (EXPKEYSIG|REVKEYSIG|EXPSIG)' gpg-status.txt; then + echo "SHA256SUMS carries a good signature from an EXPIRED or REVOKED" >&2 + echo "release key. Refusing to install. Re-sync" >&2 + echo "roles/sponsord/files/sponsord-release-KEYS.asc from upstream KEYS," >&2 + echo "or verify this release out of band." >&2 + exit 1 + fi + grep -q '^\[GNUPG:\] VALIDSIG ' gpg-status.txt + register: sponsord_onramp_sig_check + changed_when: false + failed_when: sponsord_onramp_sig_check.rc != 0 + when: sponsord_onramp_verify_release_signature + + - name: Verify the tarball against SHA256SUMS + ansible.builtin.shell: + chdir: "{{ sponsord_onramp_stage.path }}" + executable: /bin/bash + cmd: | + set -euo pipefail + f={{ sponsord_onramp_release_archive | quote }} + entry=$(awk -v f="$f" 'substr($0, index($0, " ") + 2) == f' SHA256SUMS) + n=$(printf '%s\n' "$entry" | grep -c . || true) + if [ "$n" -ne 1 ]; then + echo "SHA256SUMS has $n entries for $f (expected exactly 1)." >&2 + exit 1 + fi + printf '%s\n' "$entry" | sha256sum --check --strict - + register: sponsord_onramp_sum_check + changed_when: false + failed_when: sponsord_onramp_sum_check.rc != 0 + + - name: Extract the binary into /usr/local/bin + ansible.builtin.unarchive: + src: "{{ sponsord_onramp_stage.path }}/{{ sponsord_onramp_release_archive }}" + dest: "{{ sponsord_onramp_bin | dirname }}" + remote_src: true + owner: root + group: root + mode: "0755" + notify: Restart sponsord-onramp + + always: + - name: Report the retained staging directory after a failure + ansible.builtin.debug: + msg: >- + Verification failed; leaving {{ sponsord_onramp_stage.path }} in place for + inspection (gpg-status.txt, SHA256SUMS, SHA256SUMS.asc, the archive). + Remove it by hand once done. + when: + - sponsord_onramp_stage.path is defined + - sponsord_onramp_sig_check.failed | default(false) or sponsord_onramp_sum_check.failed | default(false) + + - name: Remove the download staging directory + ansible.builtin.file: + path: "{{ sponsord_onramp_stage.path }}" + state: absent + when: + - sponsord_onramp_stage.path is defined + - not (sponsord_onramp_sig_check.failed | default(false) + or sponsord_onramp_sum_check.failed | default(false)) + +- name: Install a locally-built binary from the control machine + when: sponsord_onramp_install_method == "manual" + block: + - name: Copy the sponsord-onramp binary from the control machine + ansible.builtin.copy: + src: "{{ _sponsord_onramp_bin_src }}" + dest: "{{ sponsord_onramp_bin }}" + owner: root + group: root + mode: "0755" + notify: Restart sponsord-onramp + + # A leftover stamp would make a later release deploy of the SAME version skip + # the download, keeping this manual binary. + - name: Clear the release version stamp in manual mode + ansible.builtin.file: + path: /usr/local/lib/sponsord-onramp/installed-version + state: absent + +# Runs every converge, so a binary that drifted from the pin is caught. With +# sponsord_onramp_version empty (the usual manual case) it degrades to a liveness check. +# An exact match on clap's `sponsord-onramp ` line, so a 0.1.0 pin does not +# accept 10.1.0 or 0.1.0-rc.1. +- name: Run the installed sponsord-onramp --version (version backstop) + ansible.builtin.command: "{{ sponsord_onramp_bin }} --version" + register: sponsord_onramp_installed_version + changed_when: false + failed_when: false + # The binary only exists once the copy/extract above really ran. + when: not ansible_check_mode + +- name: Require the installed sponsord-onramp to run and match the pinned version + ansible.builtin.assert: + that: + - sponsord_onramp_installed_version.rc == 0 + - >- + sponsord_onramp_version | length == 0 + or (sponsord_onramp_installed_version.stdout | trim) == 'sponsord-onramp ' ~ sponsord_onramp_version + fail_msg: >- + {{ sponsord_onramp_bin }} --version exited {{ sponsord_onramp_installed_version.rc }} with + "{{ sponsord_onramp_installed_version.stdout | trim }}"{{ + (', expected "sponsord-onramp ' ~ sponsord_onramp_version ~ '"') + if sponsord_onramp_version | length > 0 else '' }}. + The binary on the host is broken or is not the pinned release. In release mode, + remove /usr/local/lib/sponsord-onramp/installed-version to force a fresh download. + when: not ansible_check_mode + +# Written only once the version check passed, so a tarball that installed a wrong +# or broken binary is fetched again next run instead of being trusted via the stamp. +- name: Record the installed release version + when: + - sponsord_onramp_install_method == "release" + - sponsord_onramp_need_install | default(false) + - not ansible_check_mode + block: + - name: Ensure the stamp directory exists + ansible.builtin.file: + path: /usr/local/lib/sponsord-onramp + state: directory + owner: root + group: root + mode: "0755" + + - name: Record the installed version + ansible.builtin.copy: + dest: /usr/local/lib/sponsord-onramp/installed-version + content: "{{ sponsord_onramp_version }}\n" + owner: root + group: root + mode: "0644" diff --git a/ansible/roles/sponsord_onramp/tasks/main.yml b/ansible/roles/sponsord_onramp/tasks/main.yml new file mode 100644 index 00000000..d94950f5 --- /dev/null +++ b/ansible/roles/sponsord_onramp/tasks/main.yml @@ -0,0 +1,626 @@ +--- +# Provision sponsord-onramp: validate, gate on the host's secrets, install, +# configure, start, gate on /healthz, then put Caddy in front. Every pure-input +# assert runs before the first host mutation, and every gate (host files included) +# before the binary is installed. The Turnstile widget and the release pins are +# operator inputs — see roles/sponsord_onramp/README.md. + +# --- Placement --------------------------------------------------------------------- +# The onramp calls the daemon on loopback with the daemon's own API token, and the +# sponsord play that runs before this one is what makes the daemon healthy. +- name: Require the host to run the sponsord daemon too + ansible.builtin.assert: + that: + - "'sponsord_hosts' in group_names" + fail_msg: >- + {{ inventory_hostname }} is in sponsord_onramp_hosts but not in sponsord_hosts. + The onramp needs the sponsord daemon on the same host (it reads the daemon's + API token and calls it on loopback). Add the host to sponsord_hosts too. + +# --- Install method + target ---------------------------------------------------- +- name: Require a valid install method + ansible.builtin.assert: + that: + - sponsord_onramp_install_method in ["release", "manual"] + fail_msg: >- + sponsord_onramp_install_method must be "release" (pinned GitHub Release + tarball) or "manual" (copy a locally-built binary from the control machine). + +- name: Require a pinned release version + ansible.builtin.assert: + that: + - sponsord_onramp_version | length > 0 + fail_msg: >- + sponsord_onramp_install_method "release" requires sponsord_onramp_version (e.g. + "0.1.0") and a matching sponsord-onramp-v GitHub Release. Until upstream + cuts one, build the binary from a decdn/sponsord checkout and use + sponsord_onramp_install_method "manual". + when: sponsord_onramp_install_method == "release" + +- name: Validate sponsord_onramp_verify_release_signature is a real boolean + ansible.builtin.assert: + that: + - sponsord_onramp_verify_release_signature is boolean + fail_msg: >- + sponsord_onramp_verify_release_signature must be a boolean (true/false). Leave it + true unless you install from a mirror that strips SHA256SUMS.asc. + when: sponsord_onramp_install_method == "release" + +- name: Resolve the sponsord-onramp target triple + when: >- + sponsord_onramp_install_method == "release" + or (sponsord_onramp_release_target_dir | length > 0 and sponsord_onramp_manual_bin_src | length == 0) + block: + - name: Require a supported target triple + ansible.builtin.assert: + that: + - sponsord_onramp_target in _sponsord_onramp_arch_targets.values() + fail_msg: >- + sponsord_onramp_target resolved to "{{ sponsord_onramp_target }}", which is not + one of the Linux targets upstream releases + ({{ _sponsord_onramp_arch_targets.values() | join(', ') }}). It is derived from + the host's gathered architecture + ("{{ ansible_facts.get('architecture', 'not gathered') }}"): gather facts or set + sponsord_onramp_target explicitly. + + - name: Require the target triple to match the host architecture + ansible.builtin.assert: + that: + - _sponsord_onramp_arch_targets[ansible_facts['architecture']] == sponsord_onramp_target + fail_msg: >- + sponsord_onramp_target is {{ sponsord_onramp_target }}, but this host is + {{ ansible_facts['architecture'] }} and needs + {{ _sponsord_onramp_arch_targets[ansible_facts['architecture']] }}. Drop the + explicit sponsord_onramp_target (it is derived from the host) or fix it. + when: ansible_facts.get('architecture', '') in _sponsord_onramp_arch_targets + +# The same resolution as roles/sponsord: an explicit sponsord_onramp_manual_bin_src +# is copied verbatim (a test stub is legitimately not an ELF); a target dir is +# searched and the result must be an ELF for the host. +- name: Resolve and validate the manual binary source + when: sponsord_onramp_install_method == "manual" + block: + - name: Require a manual binary source + ansible.builtin.assert: + that: + - sponsord_onramp_manual_bin_src | length > 0 or sponsord_onramp_release_target_dir | length > 0 + fail_msg: >- + sponsord_onramp_install_method "manual" needs sponsord_onramp_manual_bin_src (a + control-machine path to the sponsord-onramp binary) or + sponsord_onramp_release_target_dir (the control-machine target/release dir of + a decdn/sponsord checkout; it defaults to sponsord_release_target_dir). + + - name: Decide whether target-dir resolution applies + ansible.builtin.set_fact: + _sponsord_onramp_use_dir: "{{ sponsord_onramp_manual_bin_src | length == 0 }}" + _sponsord_onramp_target_dir: "{{ sponsord_onramp_release_target_dir | regex_replace('/+$', '') }}" + _sponsord_onramp_bin_src: "{{ sponsord_onramp_manual_bin_src }}" + + - name: Stat the binary in each candidate directory + ansible.builtin.stat: + path: "{{ item }}/sponsord-onramp" + register: _sponsord_onramp_bin_stat + loop: + - "{{ _sponsord_onramp_target_dir }}" + - "{{ (_sponsord_onramp_target_dir | dirname) ~ '/' ~ sponsord_onramp_target ~ '/release' }}" + loop_control: + label: "{{ item }}" + delegate_to: localhost + become: false + when: _sponsord_onramp_use_dir | bool + + - name: Select the first candidate directory that holds the binary + ansible.builtin.set_fact: + _sponsord_onramp_resolved_dir: >- + {{ (_sponsord_onramp_bin_stat.results | selectattr('stat.exists') | map(attribute='item') + | first) | default('', true) }} + when: _sponsord_onramp_use_dir | bool + + - name: Fail if no candidate directory held the binary + ansible.builtin.assert: + that: + - _sponsord_onramp_resolved_dir | length > 0 + fail_msg: >- + No sponsord-onramp binary under sponsord_onramp_release_target_dir + ({{ sponsord_onramp_release_target_dir }}) or its cross output dir + ({{ (_sponsord_onramp_target_dir | dirname) ~ '/' ~ sponsord_onramp_target ~ '/release' }}). + Build it in a decdn/sponsord checkout — `cargo build --release -p + sponsord-onramp`, or cross-compiling for the host, `cross build --release -p + sponsord-onramp --target {{ sponsord_onramp_target }}`. + when: _sponsord_onramp_use_dir | bool + + - name: Derive the effective binary source path + ansible.builtin.set_fact: + _sponsord_onramp_bin_src: "{{ _sponsord_onramp_resolved_dir ~ '/sponsord-onramp' }}" + when: _sponsord_onramp_use_dir | bool + + - name: Identify the on-disk format of the resolved binary + ansible.builtin.command: + argv: [file, -b, --, "{{ _sponsord_onramp_bin_src }}"] + register: _sponsord_onramp_bin_file + changed_when: false + check_mode: false + delegate_to: localhost + become: false + when: _sponsord_onramp_use_dir | bool + + - name: Require the binary to be an ELF for the host's architecture + ansible.builtin.assert: + that: + - "'ELF' in _sponsord_onramp_bin_file.stdout" + - _sponsord_onramp_arch_file_tokens[sponsord_onramp_target] in _sponsord_onramp_bin_file.stdout + fail_msg: >- + {{ _sponsord_onramp_bin_src }} is "{{ _sponsord_onramp_bin_file.stdout }}" — not + an ELF {{ _sponsord_onramp_arch_file_tokens[sponsord_onramp_target] }} binary for + {{ sponsord_onramp_target }}. Cross-compile for the host: `cross build --release + -p sponsord-onramp --target {{ sponsord_onramp_target }}`. + when: _sponsord_onramp_use_dir | bool + +# --- Chain ------------------------------------------------------------------------ +- name: Require a known sponsord_onramp_network + ansible.builtin.assert: + that: + - sponsord_onramp_network == "" or sponsord_onramp_network in _sponsord_onramp_network_profiles + fail_msg: >- + sponsord_onramp_network "{{ sponsord_onramp_network }}" is not a known network. + Known: {{ _sponsord_onramp_network_profiles.keys() | join(', ') }}. Leave it "" + to set sponsord_onramp_capacity_bond_address yourself. + +# The onramp takes its chain from the daemon, so contract addresses from another +# network's profile would point users' decdn at the wrong chain's contracts. +- name: Require the onramp and the daemon to name the same network + ansible.builtin.assert: + that: + - sponsord_onramp_network == sponsord_network + fail_msg: >- + sponsord_onramp_network is "{{ sponsord_onramp_network }}" but the daemon's + sponsord_network is "{{ sponsord_network }}". The onramp serves the daemon's + chain, so both must name the same network. Drop sponsord_onramp_network (it + follows sponsord_network) or fix one of them. + when: + - sponsord_onramp_network | length > 0 + - sponsord_network | default('') | length > 0 + +- name: Report a contract address overridden in inventory + ansible.builtin.debug: + msg: >- + sponsord_onramp_network "{{ sponsord_onramp_network }}" is in use, but inventory + overrides {{ item }}. The inventory value wins over the profile; make sure that + is deliberate (a stale hand-copied address is the usual cause). + loop: [capacity_bond_address, slash_judge_address] + when: + - sponsord_onramp_network | length > 0 + - (lookup('ansible.builtin.vars', 'sponsord_onramp_' ~ item) | lower) != (_sponsord_onramp_profile[item] | lower) + +- name: Require the CapacityBond (and a well-formed SlashJudge) address + ansible.builtin.assert: + that: + - sponsord_onramp_capacity_bond_address is match('^0x[0-9a-fA-F]{40}$') + - sponsord_onramp_capacity_bond_address != '0x' ~ '0' * 40 + - >- + sponsord_onramp_slash_judge_address == '' + or (sponsord_onramp_slash_judge_address is match('^0x[0-9a-fA-F]{40}$') + and sponsord_onramp_slash_judge_address != '0x' ~ '0' * 40) + fail_msg: >- + sponsord_onramp_capacity_bond_address must be 0x + 40 hex, not zero, and + sponsord_onramp_slash_judge_address "" or the same shape. Set + sponsord_onramp_network (or the daemon's sponsord_network) to take both from the + role's mirror of upstream's deployment manifest — never hand-copy them. + +# --- Listener ----------------------------------------------------------------------- +# AGENTS.md hard rule 2: the onramp's own listener stays on loopback; the public +# side is Caddy (or your proxy) terminating TLS in front of it. +- name: Require the listener to stay on loopback + ansible.builtin.assert: + that: + - sponsord_onramp_bind_address is match('^127(\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])){3}$') + - sponsord_onramp_port | string is match('^[0-9]+$') + - sponsord_onramp_port | int >= 1 + - sponsord_onramp_port | int <= 65535 + - sponsord_onramp_daemon_url is match('^http://127(\.[0-9]{1,3}){3}:[0-9]+$') + fail_msg: >- + sponsord_onramp_bind_address must be an IPv4 loopback address (got + "{{ sponsord_onramp_bind_address }}"), sponsord_onramp_port a TCP port (got + "{{ sponsord_onramp_port }}") and sponsord_onramp_daemon_url the daemon's loopback + listener as http://127.x.y.z:port (got "{{ sponsord_onramp_daemon_url }}"). The + public side is the TLS proxy in front (AGENTS.md hard rule 2), never the onramp + itself. + +# --- Public address + TLS front ---------------------------------------------------- +- name: Validate the proxy settings + ansible.builtin.assert: + that: + - sponsord_onramp_proxy in ['caddy', 'none'] + - sponsord_onramp_caddy_tls in ['acme', 'internal'] + - sponsord_onramp_caddy_overwrite_config is boolean + fail_msg: >- + sponsord_onramp_proxy must be "caddy" or "none", sponsord_onramp_caddy_tls "acme" + or "internal", and sponsord_onramp_caddy_overwrite_config a boolean. + +- name: Require a domain for Caddy + ansible.builtin.assert: + that: + # A DNS name: labels of letters, digits and inner hyphens, at least one dot. + - sponsord_onramp_domain is match('^([A-Za-z0-9]([A-Za-z0-9-]{0,61}[A-Za-z0-9])?\.)+[A-Za-z]{2,63}$') + - sponsord_onramp_acme_email == '' or sponsord_onramp_acme_email is match('^[^@\s]+@[^@\s]+$') + fail_msg: >- + sponsord_onramp_proxy "caddy" needs sponsord_onramp_domain, the DNS name users + reach (got "{{ sponsord_onramp_domain }}"), and sponsord_onramp_acme_email must be + "" or an email address. For acme its A/AAAA records must point at this host. + when: sponsord_onramp_proxy == 'caddy' + +# Mirrors upstream's base_url + script_safe_url: the value is baked into the POSIX +# and PowerShell installers inside quoted strings. In the character class below, +# \x27 is ' and \x5c\x5c a backslash: a '' or \\ there would not survive Jinja's +# string-literal decoding. +- name: Require an https public base URL the installers can hold + ansible.builtin.assert: + that: + - sponsord_onramp_public_url is match('^https://[^/?#@]+(/[^?#]*)?$') + - sponsord_onramp_public_url is not search('[\s\x27"`$\x5c\x5c{}]') + fail_msg: >- + sponsord_onramp_public_url must be an https base URL with no credentials, query + or fragment, and none of whitespace ' " ` $ \ { } (got + "{{ sponsord_onramp_public_url }}"). With sponsord_onramp_proxy "none" it is + required: the address your proxy serves the onramp at. + +# Public by design: GET /v1/profile hands it to every user. A key in it is a leaked +# key, so the role refuses anything that looks like a credential. The message never +# echoes the value, in case it does hold one. +- name: Require a public RPC URL with no credentials in it + ansible.builtin.assert: + that: + - sponsord_onramp_rpc_url is match('^https?://[^/?#@]+(/[^?#]*)?$') + - sponsord_onramp_rpc_url is not search('[\s\x27"`$\x5c\x5c{}]') + fail_msg: >- + sponsord_onramp_rpc_url must be an http(s) URL with no userinfo, query or + fragment, and none of whitespace ' " ` $ \ { }. The onramp PUBLISHES it to every + user and bakes it into the installers, so it must be a public endpoint (e.g. + https://sepolia-rollup.arbitrum.io/rpc), never the keyed URL the daemon uses. + Check its path holds no API key either. + +# --- Gate + release pins -------------------------------------------------------------- +- name: Validate the Turnstile sitekey and gate settings + ansible.builtin.assert: + that: + - sponsord_onramp_turnstile_sitekey is match('^[A-Za-z0-9_-]+$') + - sponsord_onramp_brand_name | string is match('^[^\x00-\x1f\x7f]*\Z') + fail_msg: >- + sponsord_onramp_turnstile_sitekey is required: letters, digits, '_' and '-' (got + "{{ sponsord_onramp_turnstile_sitekey }}"), from the Turnstile widget for + {{ sponsord_onramp_domain | default('your domain', true) }}. sponsord_onramp_brand_name + may hold no control characters. + +- name: Validate the release pins the installers use + ansible.builtin.assert: + that: + - sponsord_onramp_decdn_release is match('^v' ~ _semver ~ '$') + - sponsord_onramp_cli_release is match('^decdn-sponsored-v' ~ _semver ~ '$') + - sponsord_onramp_decdn_sums_sha256 is match('^[0-9a-f]{64}$') + - sponsord_onramp_cli_sums_sha256 is match('^[0-9a-f]{64}$') + - sponsord_onramp_min_cli_version == '' or sponsord_onramp_min_cli_version is match('^' ~ _semver ~ '$') + - >- + sponsord_onramp_releases_base == '' + or (sponsord_onramp_releases_base is match('^https://[^/?#@]+(/[^?#]*)?$') + and sponsord_onramp_releases_base is not search('[\s\x27"`$\x5c\x5c{}]')) + fail_msg: >- + The installers need sponsord_onramp_decdn_release (vMAJOR.MINOR.PATCH), + sponsord_onramp_cli_release (decdn-sponsored-vMAJOR.MINOR.PATCH), and the SHA-256 + of each release's SHA256SUMS in sponsord_onramp_decdn_sums_sha256 / + sponsord_onramp_cli_sums_sha256 (64 lowercase hex; decdn-sponsored's release notes + print both values). sponsord_onramp_min_cli_version must be "" or MAJOR.MINOR.PATCH, + sponsord_onramp_releases_base "" or an https base URL. + vars: + _semver: '(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-[0-9A-Za-z.-]+)?' + +- name: Validate the onramp's limits + ansible.builtin.assert: + that: + - item.value | string is match('^[0-9]+$') + - item.key is search('_rate_per_min$') or item.value | int > 0 + fail_msg: >- + {{ item.key }} must be "" (the onramp's default) or a non-negative integer + ({{ 'zero turns the limit off' if item.key is search('_rate_per_min$') else 'greater than zero' }}), + got "{{ item.value }}". + loop: "{{ _sponsord_onramp_limits | dict2items | rejectattr('value', 'equalto', '') | list }}" + loop_control: + label: "{{ item.key }}" + vars: + _sponsord_onramp_limits: + sponsord_onramp_spending_cap_micro_usdc: "{{ sponsord_onramp_spending_cap_micro_usdc }}" + sponsord_onramp_ttl_secs: "{{ sponsord_onramp_ttl_secs }}" + sponsord_onramp_fund_rate_per_min: "{{ sponsord_onramp_fund_rate_per_min }}" + sponsord_onramp_poll_rate_per_min: "{{ sponsord_onramp_poll_rate_per_min }}" + +- name: Validate the client IP header name + ansible.builtin.assert: + that: + - sponsord_onramp_client_ip_header == '' or sponsord_onramp_client_ip_header is match('^[A-Za-z0-9-]+$') + fail_msg: >- + sponsord_onramp_client_ip_header must be "" (the TCP peer) or an HTTP header name + such as X-Forwarded-For or CF-Connecting-IP (got + "{{ sponsord_onramp_client_ip_header }}"). + +# --- Host secrets (the daemon's token, the Turnstile secret) ------------------------ +- name: Check the daemon API token and the Turnstile secret + ansible.builtin.stat: + path: "{{ item }}" + follow: false + get_checksum: false + register: sponsord_onramp_secret_stat + loop: + - "{{ sponsord_onramp_api_token_file }}" + - "{{ sponsord_onramp_turnstile_secret_file }}" + +# The sponsord role, which runs first on this host, generated (or gated on) the +# token; under --check its generation is only simulated, so a fresh host has none. +- name: Require the daemon API token as a regular file + ansible.builtin.assert: + that: + - sponsord_onramp_secret_stat.results[0].stat.isreg | default(false) + fail_msg: >- + {{ sponsord_onramp_api_token_file }} is + {{ 'missing' if not sponsord_onramp_secret_stat.results[0].stat.exists else 'not a regular file' }}. + The sponsord role creates it; run the sponsord play on this host first + (`make deploy-sponsord` runs both). + when: not (ansible_check_mode and not sponsord_onramp_secret_stat.results[0].stat.exists) + +- name: Require the Turnstile secret as a regular file + ansible.builtin.assert: + that: + - sponsord_onramp_secret_stat.results[1].stat.isreg | default(false) + fail_msg: |- + {{ sponsord_onramp_turnstile_secret_file }} is {{ _what }}. + The role never handles the Turnstile secret: copy it from the Cloudflare + dashboard (the widget for {{ sponsord_onramp_domain | default('your domain', true) }}) + onto this host, as root: + umask 077 + printf '%s' '' | sudo tee {{ sponsord_onramp_turnstile_secret_file }} >/dev/null + sudo chmod 600 {{ sponsord_onramp_turnstile_secret_file }} + vars: + _what: >- + {{ 'missing' if not sponsord_onramp_secret_stat.results[1].stat.exists + else 'not a regular file (symlink/directory?)' }} + +# Ownership + mode only; the content is the operator's. +- name: Lock down the Turnstile secret + ansible.builtin.file: + path: "{{ sponsord_onramp_turnstile_secret_file }}" + owner: root + group: root + mode: "0600" + follow: false + +# grep -q: the secret never reaches stdout. Upstream strips one trailing newline; +# an empty value would only fail at the first person through the gate. +- name: Confirm the Turnstile secret is not empty + ansible.builtin.command: + cmd: grep -q '[^[:space:]]' {{ sponsord_onramp_turnstile_secret_file | quote }} + register: sponsord_onramp_turnstile_grep + changed_when: false + check_mode: false + failed_when: false + +- name: Report an empty Turnstile secret + ansible.builtin.fail: + msg: >- + {{ sponsord_onramp_turnstile_secret_file }} is empty. Write the widget's secret + key into it (see the previous task's message for the recipe). + when: sponsord_onramp_turnstile_grep.rc != 0 + +- name: Read whether Caddy is already installed + ansible.builtin.package_facts: + when: sponsord_onramp_proxy == 'caddy' + +# Before any mutation: a Caddyfile someone else wrote is never replaced silently. +- name: Check the Caddyfile for this role's marker + ansible.builtin.command: + argv: [grep, -qF, "MANAGED BY the sponsord_onramp role", "{{ sponsord_onramp_caddyfile }}"] + register: sponsord_onramp_caddyfile_marker + changed_when: false + check_mode: false + failed_when: false + when: + - sponsord_onramp_proxy == 'caddy' + - "'caddy' in ansible_facts.packages" + +- name: Refuse to replace a Caddyfile this role did not write + ansible.builtin.assert: + that: + - sponsord_onramp_caddy_overwrite_config + fail_msg: >- + Caddy was installed on this host before this role and + {{ sponsord_onramp_caddyfile }} is not the role's (no "MANAGED BY the + sponsord_onramp role" line). The role owns that whole file, so it would discard + whatever else this Caddy serves. Move the other sites elsewhere (or onto your own + proxy, with sponsord_onramp_proxy "none"), or set + sponsord_onramp_caddy_overwrite_config: true to let the role replace it. + when: + - sponsord_onramp_proxy == 'caddy' + - "'caddy' in ansible_facts.packages" + - sponsord_onramp_caddyfile_marker.rc != 0 + +# --- Install, configure, start ------------------------------------------------------ +- name: Install the sponsord-onramp binary + ansible.builtin.import_tasks: install.yml + +- name: Install the custom gate page + ansible.builtin.copy: + src: "{{ sponsord_onramp_gate_template_src }}" + dest: "{{ sponsord_onramp_gate_template_file }}" + owner: root + group: root + mode: "0644" + notify: Restart sponsord-onramp + when: sponsord_onramp_gate_template_src | length > 0 + +- name: Remove a custom gate page that is no longer configured + ansible.builtin.file: + path: "{{ sponsord_onramp_gate_template_file }}" + state: absent + notify: Restart sponsord-onramp + when: sponsord_onramp_gate_template_src | length == 0 + +- name: Template the non-secret sponsord-onramp.env + ansible.builtin.template: + src: sponsord-onramp.env.j2 + dest: "{{ sponsord_onramp_env_file }}" + owner: root + group: root + mode: "0644" + notify: Restart sponsord-onramp + +- name: Install the sponsord-onramp systemd unit + ansible.builtin.template: + src: sponsord-onramp.service.j2 + dest: /etc/systemd/system/sponsord-onramp.service + owner: root + group: root + mode: "0644" + notify: Restart sponsord-onramp + +# --- Restart inputs ----------------------------------------------------------------- +# As roles/sponsord: hashes of everything the onramp starts from, recorded once it +# is healthy, so a rotated daemon token or Turnstile secret (both edited on the +# host, out of band) or a restart lost to a failed run is caught next converge. +- name: Hash the restart inputs + ansible.builtin.stat: + path: "{{ item }}" + get_checksum: true + checksum_algorithm: sha256 + register: sponsord_onramp_inputs_stat + loop: + - "{{ sponsord_onramp_api_token_file }}" + - "{{ sponsord_onramp_turnstile_secret_file }}" + - "{{ sponsord_onramp_env_file }}" + - "{{ sponsord_onramp_gate_template_file }}" + - /etc/systemd/system/sponsord-onramp.service + - "{{ sponsord_onramp_bin }}" + +- name: Read the restart-input checksums this role last recorded + ansible.builtin.slurp: + src: "{{ sponsord_onramp_inputs_checksum_file }}" + register: sponsord_onramp_inputs_recorded + failed_when: >- + sponsord_onramp_inputs_recorded.failed + and 'not found' not in (sponsord_onramp_inputs_recorded.msg | default('')) + changed_when: false + +- name: Read whether sponsord-onramp is already running + ansible.builtin.systemd_service: + name: sponsord-onramp + register: sponsord_onramp_is_active + +- name: Summarise the restart-input checksums + ansible.builtin.set_fact: + sponsord_onramp_inputs_sums: >- + {{ sponsord_onramp_inputs_stat.results | map(attribute='stat') | map('default', {}, true) + | map(attribute='checksum', default='') | join(' ') }} + sponsord_onramp_inputs_recorded_sums: >- + {{ sponsord_onramp_inputs_recorded.content | default('') | b64decode | trim }} + +- name: Note restart inputs that changed since the onramp last started healthy + ansible.builtin.debug: + msg: >- + {{ 'The daemon token, the Turnstile secret, the env file, the gate page, the unit + or the binary differs from what sponsord-onramp was last confirmed healthy on' + if sponsord_onramp_inputs_recorded_sums | length > 0 + else 'sponsord-onramp is running but was never confirmed healthy on its current inputs' }} + — restarting sponsord-onramp. + changed_when: true + notify: Restart sponsord-onramp + when: >- + (sponsord_onramp_inputs_recorded_sums | length > 0 + and sponsord_onramp_inputs_sums != sponsord_onramp_inputs_recorded_sums) + or (sponsord_onramp_inputs_recorded_sums | length == 0 + and sponsord_onramp_is_active.status.ActiveState | default('') == 'active') + +- name: Apply pending sponsord-onramp changes + ansible.builtin.meta: flush_handlers + +- name: Enable and start sponsord-onramp + ansible.builtin.systemd: + name: sponsord-onramp + enabled: true + state: started + daemon_reload: true + +# --- Readiness (fatal) ------------------------------------------------------------ +# The onramp binds only after it has read the daemon's /v1/info and checked its own +# cap and TTL against the daemon's maximum, so this is the deploy's config gate. +# Retry while nothing answers (-1); any other answer than 200 stops at once. +# KICS flags plain HTTP here; the target is the loopback listener asserted above. +- name: Wait for sponsord-onramp /healthz + ansible.builtin.uri: + url: "http://{{ sponsord_onramp_bind_address }}:{{ sponsord_onramp_port }}/healthz" + status_code: 200 + return_content: true + timeout: 5 + register: sponsord_onramp_health + retries: "{{ sponsord_onramp_readiness_retries }}" + delay: "{{ sponsord_onramp_readiness_delay }}" + until: sponsord_onramp_health.status | default(-1) != -1 + changed_when: false + failed_when: false + when: not ansible_check_mode + +- name: Report a sponsord-onramp that did not become healthy + ansible.builtin.fail: + msg: |- + sponsord-onramp did not answer 200 on + http://{{ sponsord_onramp_bind_address }}:{{ sponsord_onramp_port }}/healthz within + ~{{ (sponsord_onramp_readiness_retries | int) * (sponsord_onramp_readiness_delay | int) }}s + (last status {{ sponsord_onramp_health.status | default('none') }}). It binds only + after reaching the daemon at {{ sponsord_onramp_daemon_url }} with its API token + and validating its settings, so the usual causes are a daemon that is down, a + cap or TTL above the daemon's maximum, or a malformed release pin. Read the + reason on the host: + sudo journalctl -u sponsord-onramp -e + when: + - not ansible_check_mode + - >- + sponsord_onramp_health.status | default(-1) != 200 + or sponsord_onramp_health.json.ok | default(false) is not true + +# As roles/sponsord: the 200 must come from THIS unit, not another holder of the port. +- name: Read the sponsord-onramp unit state + ansible.builtin.systemd_service: + name: sponsord-onramp + register: sponsord_onramp_unit_state + when: not ansible_check_mode + +- name: Read the owner of the sponsord-onramp listener + ansible.builtin.command: + argv: [ss, -Hltnp, "sport = :{{ sponsord_onramp_port }}"] + register: sponsord_onramp_listener + changed_when: false + when: not ansible_check_mode + +- name: Require the listener to belong to the running sponsord-onramp unit + ansible.builtin.assert: + that: + - sponsord_onramp_unit_state.status.ActiveState == 'active' + - sponsord_onramp_unit_state.status.SubState == 'running' + - ('pid=' ~ sponsord_onramp_unit_state.status.MainPID ~ ',') in sponsord_onramp_listener.stdout + fail_msg: >- + Something answered /healthz on {{ sponsord_onramp_bind_address }}:{{ sponsord_onramp_port }}, + but it is not the sponsord-onramp unit (state + {{ sponsord_onramp_unit_state.status.ActiveState }}/{{ sponsord_onramp_unit_state.status.SubState }}, + MainPID {{ sponsord_onramp_unit_state.status.MainPID }}; listeners: + {{ sponsord_onramp_listener.stdout | default('none', true) }}). Another service holds + the port: free it or set sponsord_onramp_port. + when: not ansible_check_mode + +- name: Record the restart-input checksums + ansible.builtin.copy: + dest: "{{ sponsord_onramp_inputs_checksum_file }}" + owner: root + group: root + mode: "0600" + content: "{{ sponsord_onramp_inputs_sums }}\n" + when: not ansible_check_mode + +# --- TLS front ---------------------------------------------------------------------- +- name: Put Caddy in front of the onramp + ansible.builtin.import_tasks: caddy.yml + when: sponsord_onramp_proxy == 'caddy' diff --git a/ansible/roles/sponsord_onramp/templates/Caddyfile.j2 b/ansible/roles/sponsord_onramp/templates/Caddyfile.j2 new file mode 100644 index 00000000..ebae5ded --- /dev/null +++ b/ansible/roles/sponsord_onramp/templates/Caddyfile.j2 @@ -0,0 +1,22 @@ +# MANAGED BY the sponsord_onramp role — do not edit by hand. +# +# The public side of sponsord-onramp: TLS for {{ sponsord_onramp_domain }}, then a +# reverse proxy to the onramp on loopback. Caddy redirects http to https and, +# with no trusted_proxies, replaces any client-sent X-Forwarded-For with the real +# client address, which the onramp rate limits by. +{ + # No admin API: nothing on this host may reconfigure the proxy over loopback. + # Config changes restart Caddy instead of reloading it. + admin off +{% if sponsord_onramp_acme_email | length > 0 %} + email {{ sponsord_onramp_acme_email }} +{% endif %} +} + +{{ sponsord_onramp_domain }} { +{% if sponsord_onramp_caddy_tls == 'internal' %} + # Caddy's local CA: for staging and tests only, browsers do not trust it. + tls internal +{% endif %} + reverse_proxy {{ sponsord_onramp_bind_address }}:{{ sponsord_onramp_port }} +} diff --git a/ansible/roles/sponsord_onramp/templates/sponsord-onramp.env.j2 b/ansible/roles/sponsord_onramp/templates/sponsord-onramp.env.j2 new file mode 100644 index 00000000..db4803eb --- /dev/null +++ b/ansible/roles/sponsord_onramp/templates/sponsord-onramp.env.j2 @@ -0,0 +1,34 @@ +# MANAGED BY the sponsord_onramp role — do not edit by hand. NO secrets: the +# daemon's API token and the Turnstile secret reach the onramp as systemd +# credentials (see sponsord-onramp.service). Every value here is public: the RPC +# URL and the contract addresses are served to every user. +ONRAMP_BIND={{ sponsord_onramp_bind_address }}:{{ sponsord_onramp_port }} +ONRAMP_DAEMON_URL={{ sponsord_onramp_daemon_url }} +ONRAMP_PUBLIC_URL={{ sponsord_onramp_public_url }} +ONRAMP_RPC_URL={{ sponsord_onramp_rpc_url }} +ONRAMP_CAPACITY_BOND_ADDR={{ sponsord_onramp_capacity_bond_address }} +ONRAMP_GATE=turnstile +ONRAMP_TURNSTILE_SITEKEY={{ sponsord_onramp_turnstile_sitekey }} +ONRAMP_DECDN_RELEASE={{ sponsord_onramp_decdn_release }} +ONRAMP_DECDN_SUMS_SHA256={{ sponsord_onramp_decdn_sums_sha256 }} +ONRAMP_CLI_RELEASE={{ sponsord_onramp_cli_release }} +ONRAMP_CLI_SUMS_SHA256={{ sponsord_onramp_cli_sums_sha256 }} +{% if sponsord_onramp_brand_name | length > 0 %} +{# to_json: the double-quoted, C-escaped form systemd's EnvironmentFile parser reads. #} +ONRAMP_BRAND_NAME={{ sponsord_onramp_brand_name | string | to_json(ensure_ascii=false) }} +{% endif %} +{% if sponsord_onramp_gate_template_src | length > 0 %} +ONRAMP_GATE_TEMPLATE={{ sponsord_onramp_gate_template_file }} +{% endif %} +{% for name, value in [ + ('ONRAMP_SLASH_JUDGE_ADDR', sponsord_onramp_slash_judge_address), + ('ONRAMP_CLIENT_IP_HEADER', sponsord_onramp_client_ip_header), + ('ONRAMP_MIN_CLI_VERSION', sponsord_onramp_min_cli_version), + ('ONRAMP_RELEASES_BASE', sponsord_onramp_releases_base), + ('ONRAMP_SPENDING_CAP_MICRO_USDC', sponsord_onramp_spending_cap_micro_usdc), + ('ONRAMP_TTL_SECS', sponsord_onramp_ttl_secs), + ('ONRAMP_FUND_RATE_PER_MIN', sponsord_onramp_fund_rate_per_min), + ('ONRAMP_POLL_RATE_PER_MIN', sponsord_onramp_poll_rate_per_min), + ] if value | string | length > 0 %} +{{ name }}={{ value }} +{% endfor %} diff --git a/ansible/roles/sponsord_onramp/templates/sponsord-onramp.service.j2 b/ansible/roles/sponsord_onramp/templates/sponsord-onramp.service.j2 new file mode 100644 index 00000000..2e72fa60 --- /dev/null +++ b/ansible/roles/sponsord_onramp/templates/sponsord-onramp.service.j2 @@ -0,0 +1,66 @@ +# MANAGED BY the sponsord_onramp role — do not edit by hand. +# +# sponsord-onramp: the gate page, the installers and the CLI API, in front of the +# sponsord daemon on this host. It listens on loopback only +# ({{ sponsord_onramp_bind_address }}:{{ sponsord_onramp_port }}); {{ ('Caddy terminates TLS for ' ~ sponsord_onramp_domain) if sponsord_onramp_proxy == 'caddy' else "the operator's own proxy terminates TLS" }}. +# It keeps only in-flight hand-offs in memory. The daemon's API token and the +# Turnstile secret never sit in the environment: systemd reads the root-only files +# and hands them to this unit as credentials. +[Unit] +Description=sponsord-onramp: the public onramp in front of sponsord +Documentation=https://github.com/decdn/sponsord +After=network-online.target sponsord.service +Wants=network-online.target sponsord.service +# The onramp reads the daemon's limits once, at start; restart it with the daemon. +PartOf=sponsord.service +# It exits at start while the daemon is unreachable; keep retrying until it is up. +StartLimitIntervalSec=0 + +[Service] +Type=simple +ExecStart={{ sponsord_onramp_bin }} +EnvironmentFile={{ sponsord_onramp_env_file }} +# Plain-text logs into journald, without ANSI colour codes. +Environment=NO_COLOR=1 + +# Credentials land at $CREDENTIALS_DIRECTORY (%d), readable only by this unit. +LoadCredential=api-token:{{ sponsord_onramp_api_token_file }} +LoadCredential=turnstile-secret:{{ sponsord_onramp_turnstile_secret_file }} +Environment=ONRAMP_DAEMON_TOKEN_FILE=%d/api-token +Environment=ONRAMP_TURNSTILE_SECRET_FILE=%d/turnstile-secret + +KillSignal=SIGTERM +TimeoutStopSec=30 +Restart=on-failure +RestartSec=5 + +# --- Hardening ------------------------------------------------------------- +# Stateless: no StateDirectory, nothing on disk is writable. +DynamicUser=true +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +PrivateDevices=true +ProtectKernelTunables=true +ProtectKernelModules=true +ProtectKernelLogs=true +ProtectControlGroups=true +ProtectClock=true +ProtectHostname=true +CapabilityBoundingSet= +# AF_INET/AF_INET6 for the listener, the daemon and Turnstile's siteverify; +# AF_UNIX + AF_NETLINK for the libc resolver. +RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK +RestrictNamespaces=true +RestrictRealtime=true +RestrictSUIDSGID=true +LockPersonality=true +MemoryDenyWriteExecute=true +SystemCallArchitectures=native +SystemCallFilter=@system-service +SystemCallErrorNumber=EPERM +UMask=0077 + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/sponsord_onramp/vars/main/arch.yml b/ansible/roles/sponsord_onramp/vars/main/arch.yml new file mode 100644 index 00000000..0b4387bb --- /dev/null +++ b/ansible/roles/sponsord_onramp/vars/main/arch.yml @@ -0,0 +1,16 @@ +--- +# Host architecture (ansible_facts['architecture']) -> the upstream release target +# triple. These are the two Linux targets decdn/sponsord's release builds for +# sponsord-onramp (LINUX in .github/scripts/release_plan.py); the defaults derive +# sponsord_onramp_target from this map and tasks/main.yml asserts the result is one +# of its values. Role vars, not defaults: the supported set is a fact about +# upstream's release matrix, not an operator knob. +_sponsord_onramp_arch_targets: + x86_64: x86_64-unknown-linux-gnu + aarch64: aarch64-unknown-linux-gnu + +# sponsord_onramp_target -> the architecture token `file -b` prints for a matching +# ELF, used by the manual dir-mode binary check. +_sponsord_onramp_arch_file_tokens: + x86_64-unknown-linux-gnu: x86-64 + aarch64-unknown-linux-gnu: aarch64 diff --git a/ansible/roles/sponsord_onramp/vars/main/networks.yml b/ansible/roles/sponsord_onramp/vars/main/networks.yml new file mode 100644 index 00000000..38341b40 --- /dev/null +++ b/ansible/roles/sponsord_onramp/vars/main/networks.yml @@ -0,0 +1,15 @@ +--- +# GENERATED by scripts/sync-network-profiles.py -- do not hand-edit. +# Source: decdn/decdn @ 5ca3e7f1c5ac84890fb5b666b29b6f384e3797ec +# crates/cli/src/known_chains.rs (KNOWN_CHAINS, KnownChain::addresses) +# + contracts/deployments/.json +# +# The subset of roles/decdn_node/vars/main/networks.yml sponsord_onramp needs. The role's +# chain defaults read this map lazily when sponsord_onramp_network is set, so an explicit +# inventory value still wins. Protocol facts: upstream's deployment manifests are +# the source, this file only mirrors them. +_sponsord_onramp_network_profiles: + arbitrum-sepolia: + chain_id: 421614 + capacity_bond_address: "0x65a9e38F41B9bE2fcB97950a0E3c33e506E6E353" + slash_judge_address: "0x8e4d5945615bDF00106ff429da07bd08bCdEdB94" diff --git a/ansible/tests/alloy-config/render.yml b/ansible/tests/alloy-config/render.yml index 0ed9fbbf..f8f0bb85 100644 --- a/ansible/tests/alloy-config/render.yml +++ b/ansible/tests/alloy-config/render.yml @@ -316,6 +316,26 @@ grafana_alloy_sponsord_enabled: true grafana_alloy_logs_drop_priority_regex: 'info|debug|trace' + # The same host also running the onramp (sponsord_onramp_hosts): its journal + # stream gets a service_name and the plain-text level parsing, no scrape. + - name: Render the co-located node + sponsord + onramp configuration + ansible.builtin.template: + src: "{{ playbook_dir }}/../../roles/grafana_alloy/templates/{{ item.src }}" + dest: "{{ _ga_render_dir }}/colocatedonramp.{{ item.ext }}" + mode: "0644" + loop: *ga_render_pair + loop_control: + label: "colocatedonramp.{{ item.ext }}" + vars: + _ga_instance_id: decdn-node-1 + _ga_journal_groups: [systemd-journal, adm] + grafana_alloy_bin_effective: /usr/bin/alloy + grafana_alloy_deployment_environment: production + grafana_alloy_region: US + grafana_alloy_sponsord_enabled: true + grafana_alloy_sponsord_onramp_enabled: true + grafana_alloy_logs_drop_priority_regex: 'info|debug|trace' + - name: Export the pinned release for the validation script ansible.builtin.copy: dest: "{{ _ga_render_dir }}/pin.env" diff --git a/ansible/tests/alloy-config/validate.sh b/ansible/tests/alloy-config/validate.sh index af8dee9c..55bce9e3 100755 --- a/ansible/tests/alloy-config/validate.sh +++ b/ansible/tests/alloy-config/validate.sh @@ -269,10 +269,17 @@ assert_has sponsordonly.alloy 'forward_to = [prometheus.relabel.sponsord_id assert_has sponsordonly.alloy 'replacement = "sponsord"' "the sponsord metric service_name" assert_has sponsordonly.alloy 'regex = "sponsord\\.service"' "the sponsord log-stream unit match" assert_has sponsordonly.alloy 'replacement = "sponsord"' "the sponsord log-stream service_name" -assert_has sponsordonly.alloy '(decdn-node|sponsord|alloy|ssh|sshd)' "sponsord in the systemd collector's unit list" +assert_has sponsordonly.alloy '(decdn-node|sponsord|sponsord-onramp|caddy|alloy|ssh|sshd)' "sponsord, the onramp and Caddy in the systemd collector's unit list" assert_has colocated.alloy 'prometheus.scrape "decdn_node"' "the node scrape on a co-located host" assert_has colocated.alloy 'prometheus.scrape "sponsord"' "the sponsord scrape on a co-located host" assert_has colocated.alloy '"__address__" = "127.0.0.1:8090"' "the default sponsord listener" +assert_lacks defaults.alloy 'sponsord-onramp\\.service' "an onramp log rule while it is disabled" +assert_lacks colocated.alloy 'sponsord-onramp\\.service' "an onramp log rule while it is disabled" +assert_lacks colocatedonramp.alloy 'prometheus.scrape "sponsord_onramp"' "an onramp scrape (it has no /metrics)" +assert_has colocatedonramp.alloy 'regex = "sponsord-onramp\\.service"' "the onramp log-stream unit match" +assert_has colocatedonramp.alloy 'replacement = "sponsord-onramp"' "the onramp log-stream service_name" +assert_has colocatedonramp.alloy '"(?:decdn-node|sponsord|sponsord-onramp)\\.service"' "the onramp in the priority exemption" +assert_has colocatedonramp.alloy '{unit=~"(sponsord|sponsord-onramp)\\.service"}' "the onramp in the plain-text level selector" echo "ok: per-daemon scrape toggles" # --- Gate 1c: URLs are redacted, and the daemon's level comes from its line ---- @@ -308,6 +315,12 @@ samples_json() { # nanosecond timestamp ["$1","Caused by: s-cause connection refused"], ["$1","s-plain an unlevelled line"], ["$1","Error: s-rpcurl read PaymentPool.usdc(): error sending request for url (https://arb-sepolia.example-provider.io/v2/SECRET-KEY?x=SECRET-Q)"]]}, + {"stream":{"journal__systemd_unit":"sponsord-onramp.service","journal_priority_keyword":"info"},"values":[ + ["$1","2026-10-05T07:59:11.000001Z DEBUG sponsord_onramp::http: r-debug"], + ["$1","2026-10-05T07:59:11.000002Z INFO sponsord_onramp: r-info"], + ["$1","2026-10-05T07:59:11.000003Z WARN sponsord_onramp::gate::turnstile: r-warn"], + ["$1","Error: r-raw the daemon at http://127.0.0.1:8090 is unreachable"], + ["$1","r-plain an unlevelled line"]]}, {"stream":{"journal__systemd_unit":"other.service","journal_priority_keyword":"info"},"values":[ ["$1","{\"level\":\"ERROR\",\"fields\":{\"message\":\"m-other-info\"}}"]]}, {"stream":{"journal__systemd_unit":"other.service","journal_priority_keyword":"error"},"values":[ @@ -438,7 +451,7 @@ no_secrets() { # Defaults (guardrail "debug|trace"), sponsord not enabled: its plain-text lines # are not re-levelled and stand at journald's info. -run_level_harness defaults.alloy 34 +run_level_harness defaults.alloy 39 kept s-trace info sponsord.service kept s-error info sponsord.service kept m-warn-uc warning decdn-node.service @@ -450,6 +463,8 @@ kept m-weird info decdn-node.service # a non-string level is not a label valu kept m-text info decdn-node.service # not JSON: journald's stands kept m-other-info info other.service # another unit's JSON is never re-levelled kept m-other-error error other.service +kept r-warn info sponsord-onramp.service # onramp not enabled: journald's level +kept r-debug info sponsord-onramp.service dropped m-trace # TRACE is not a journald keyword, so listed dropped m-debug # padded + lower-cased before the drop dropped m-other-debug # the journald-priority drop, other units @@ -508,6 +523,7 @@ kept m-nolevel info decdn-node.service kept m-weird info decdn-node.service kept m-text info decdn-node.service kept m-other-error error other.service +dropped r-warn # onramp not enabled: judged at journald's info dropped m-info dropped m-trace dropped m-debug @@ -516,7 +532,7 @@ dropped m-other-debug echo "ok: daemon log level follows the JSON body (info in the guardrail)" # sponsord enabled: its level is parsed from the plain-text line. -run_level_harness sponsordonly.alloy 32 +run_level_harness sponsordonly.alloy 37 no_secrets kept s-rpcurl error sponsord.service # the "Error: " match still sees the redacted line kept s-info info sponsord.service @@ -528,6 +544,7 @@ kept s-plain info sponsord.service # any other unlevelled line: journald's dropped s-trace dropped s-debug kept m-error error decdn-node.service # the node's JSON stage is unaffected +kept r-warn info sponsord-onramp.service # the onramp is not enabled here echo "ok: sponsord log level follows its text line (defaults)" # Co-located, guardrail "info|debug|trace": both daemons exempt from the @@ -546,8 +563,22 @@ dropped s-info dropped s-trace dropped m-info dropped m-other-info +dropped r-warn # the onramp is not enabled: not exempt +dropped r-raw echo "ok: both daemons judged by their own level (info in the guardrail)" +# The onramp too: exempt, and levelled from its plain-text line like sponsord. +run_level_harness colocatedonramp.alloy 32 +no_secrets +kept r-warn warning sponsord-onramp.service +kept r-raw error sponsord-onramp.service # its exit error, with the daemon URL redacted +kept r-plain info sponsord-onramp.service # exempt: no level of its own to judge by +kept s-warn warning sponsord.service # sponsord's stage is unaffected +kept m-error error decdn-node.service +dropped r-info +dropped r-debug +echo "ok: the onramp judged by its own level (info in the guardrail)" + # --- Gate 2: every ExecStart flag exists ------------------------------------- # `alloy run` ignores nothing: an unknown flag exits non-zero, i.e. a systemd # crash-loop on the target host the moment the unit starts. diff --git a/ansible/tests/firewall-holes/expected.json b/ansible/tests/firewall-holes/expected.json index f9927011..d317da93 100644 --- a/ansible/tests/firewall-holes/expected.json +++ b/ansible/tests/firewall-holes/expected.json @@ -2,5 +2,15 @@ "node": [{"proto": "udp", "port": 4433, "comment": "decdn QUIC"}], "node-port": [{"proto": "udp", "port": 5555, "comment": "decdn QUIC"}], "sponsord": [], - "co-located": [{"proto": "udp", "port": 4433, "comment": "decdn QUIC"}] + "co-located": [{"proto": "udp", "port": 4433, "comment": "decdn QUIC"}], + "onramp": [ + {"proto": "tcp", "port": 80, "comment": "sponsord-onramp (Caddy, http)"}, + {"proto": "tcp", "port": 443, "comment": "sponsord-onramp (Caddy, https)"} + ], + "onramp-own-proxy": [], + "co-located-onramp": [ + {"proto": "udp", "port": 4433, "comment": "decdn QUIC"}, + {"proto": "tcp", "port": 80, "comment": "sponsord-onramp (Caddy, http)"}, + {"proto": "tcp", "port": 443, "comment": "sponsord-onramp (Caddy, https)"} + ] } diff --git a/ansible/tests/firewall-holes/inventory.yml b/ansible/tests/firewall-holes/inventory.yml index 312e2339..5d896b9b 100644 --- a/ansible/tests/firewall-holes/inventory.yml +++ b/ansible/tests/firewall-holes/inventory.yml @@ -11,7 +11,17 @@ all: node-port: decdn_bind_port: 5555 co-located: {} + co-located-onramp: {} sponsord_hosts: hosts: sponsord: {} co-located: {} + onramp: {} + onramp-own-proxy: {} + co-located-onramp: {} + sponsord_onramp_hosts: + hosts: + onramp: {} + onramp-own-proxy: + sponsord_onramp_proxy: none + co-located-onramp: {} diff --git a/docs/requirements.md b/docs/requirements.md index 47682b9a..6a7ac82e 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -27,7 +27,7 @@ and they run the same roles. | | Supported | How it is tested | |---|---|---| -| OS | Debian 12 (bookworm), Debian 13 (trixie), Ubuntu 24.04 (noble), Ubuntu 26.04 (resolute) | molecule converges `decdn_node` on all four in systemd containers; `grafana_alloy`'s install path on Debian 12 only (its disabled path on all four); `sponsord` on Debian 12 and Ubuntu 24.04; `baseline` on real hosts | +| OS | Debian 12 (bookworm), Debian 13 (trixie), Ubuntu 24.04 (noble), Ubuntu 26.04 (resolute) | molecule converges `decdn_node` on all four in systemd containers; `grafana_alloy`'s install path on Debian 12 only (its disabled path on all four); `sponsord` and `sponsord_onramp` on Debian 12 and Ubuntu 24.04; `baseline` on real hosts | | Architecture | x86_64, aarch64 | upstream builds both; the Ansible role derives the target from the host | | Ansible (control machine) | ansible-core ≥ 2.15 | CI runs the current release | | cloud-init | the provider image's own; the bootstrap installs its pinned ansible-core on the host | the user-data is booted with the distro's cloud-init in Debian 12 and Ubuntu 26.04 containers, to a running node | @@ -42,6 +42,11 @@ On Ubuntu 25.10 and later, `sudo` is sudo-rs; see the `ansible_become_exe` note - **Inbound: udp/4433 (QUIC)** from anywhere. This is the node's only public port. Open it in the host firewall (the Ansible `baseline` does this) **and** in your cloud provider's security group or firewall, which the host cannot see. +- **A sponsord-onramp host (Ansible only) also needs inbound tcp/80 and tcp/443** for + Caddy, from anywhere: 443 serves the onramp, 80 answers the ACME challenge and + redirects to https. `baseline` opens both for hosts in `sponsord_onramp_hosts`; open + them in the provider's firewall too, and point the domain's A/AAAA records at the + host. - **Outbound:** HTTPS to your RPC provider and to any cache origin; QUIC/UDP to peers; HTTPS to the iroh relays (upstream's defaults unless you set your own). - **NAT:** a node without direct inbound reachability still works through iroh relays, diff --git a/scripts/sync-network-profiles.py b/scripts/sync-network-profiles.py index a9a81384..36647753 100755 --- a/scripts/sync-network-profiles.py +++ b/scripts/sync-network-profiles.py @@ -8,9 +8,11 @@ would bake into node.toml. Operators then set `decdn_network: ` instead of hand-copying eight addresses that change wholesale on every upstream redeploy. -Also writes ansible/roles/sponsord/vars/main/networks.yml, the subset sponsord needs -(chain_id + the PaymentPool address), from the same parsed data, so the two roles can -never disagree about a chain. --check covers both files. +Also writes the subsets two other roles need, from the same parsed data, so the roles +can never disagree about a chain: ansible/roles/sponsord/vars/main/networks.yml +(chain_id + the PaymentPool address) and ansible/roles/sponsord_onramp/vars/main/ +networks.yml (chain_id + CapacityBond and SlashJudge, which the onramp hands to users' +decdn). --check covers all three files. Everything is read from git at --ref (default origin/main), never from the working tree, so a checkout sitting on a feature branch still yields main's deployment. Two @@ -40,6 +42,7 @@ REPO = Path(__file__).resolve().parent.parent OUT = REPO / "ansible/roles/decdn_node/vars/main/networks.yml" SPONSORD_OUT = REPO / "ansible/roles/sponsord/vars/main/networks.yml" +ONRAMP_OUT = REPO / "ansible/roles/sponsord_onramp/vars/main/networks.yml" # Upstream ChainAddresses field -> decdn_node variable suffix (decdn_). # Order is the output order. @@ -57,6 +60,8 @@ # The FIELDS values sponsord's profile keeps (its role variables are # sponsord_). Order is the output order. SPONSORD_FIELDS = ["payment_pool_address"] +# Likewise for sponsord_onramp (variables sponsord_onramp_). +ONRAMP_FIELDS = ["capacity_bond_address", "slash_judge_address"] ADDRESS = re.compile(r"^0x[0-9a-fA-F]{40}$") REGISTRY = re.compile(r"KNOWN_CHAINS\s*:\s*&\[\s*KnownChain\s*\]\s*=\s*&\[(?P.*?)\];", re.S) @@ -166,7 +171,7 @@ def render(commit: str, profiles: list[dict]) -> str: return "\n".join(lines) + "\n" -def render_sponsord(commit: str, profiles: list[dict]) -> str: +def render_subset(commit: str, profiles: list[dict], role: str, fields: list[str]) -> str: lines = [ "---", "# GENERATED by scripts/sync-network-profiles.py -- do not hand-edit.", @@ -174,15 +179,15 @@ def render_sponsord(commit: str, profiles: list[dict]) -> str: "# crates/cli/src/known_chains.rs (KNOWN_CHAINS, KnownChain::addresses)", "# + contracts/deployments/.json", "#", - "# The subset of roles/decdn_node/vars/main/networks.yml sponsord needs. The role's", - "# chain defaults read this map lazily when sponsord_network is set, so an explicit", + f"# The subset of roles/decdn_node/vars/main/networks.yml {role} needs. The role's", + f"# chain defaults read this map lazily when {role}_network is set, so an explicit", "# inventory value still wins. Protocol facts: upstream's deployment manifests are", "# the source, this file only mirrors them.", - "_sponsord_network_profiles:", + f"_{role}_network_profiles:", ] for p in profiles: lines += [f" {p['name']}:", f" chain_id: {p['chain_id']}"] - lines += [f' {var}: "{p[var]}"' for var in SPONSORD_FIELDS] + lines += [f' {var}: "{p[var]}"' for var in fields] return "\n".join(lines) + "\n" @@ -198,7 +203,11 @@ def main() -> None: args = ap.parse_args() commit, profiles = load(args.upstream, args.ref) - outputs = [(OUT, render(commit, profiles)), (SPONSORD_OUT, render_sponsord(commit, profiles))] + outputs = [ + (OUT, render(commit, profiles)), + (SPONSORD_OUT, render_subset(commit, profiles, "sponsord", SPONSORD_FIELDS)), + (ONRAMP_OUT, render_subset(commit, profiles, "sponsord_onramp", ONRAMP_FIELDS)), + ] if args.check: stale = False for out, text in outputs: @@ -212,6 +221,7 @@ def main() -> None: sys.exit(1) return for out, text in outputs: + out.parent.mkdir(parents=True, exist_ok=True) out.write_text(text) print(f"wrote {out.relative_to(REPO)}") From 6de524ee34963e4d2aabfcd661d5bf78eeb8265f Mon Sep 17 00:00:00 2001 From: Ant Somers Date: Mon, 5 Oct 2026 16:33:08 +0300 Subject: [PATCH 3/4] fix(sponsord_onramp): address the branch review - tests/firewall-holes: write each host's list with the core copy module. The json stdout callback lives in ansible.posix, so the check failed on a core-only install such as CI's. It also pins all.yml's fallbacks (4433, "caddy") to the role defaults. - Caddy probes: a plain-http probe must get Caddy's 308 redirect (fatal, no certificate needed). With ACME, only a not-yet-issued certificate (curl 35/60) warns; a 502, nothing on 443 or a hang now fails the run. HTTP/3 is off, since udp/443 is not opened. - The foreign-Caddyfile guard keys on the file, not the dpkg package: a Caddyfile without the marker is refused unless it is the package's untouched default (dpkg's conffile md5) or the overwrite is confirmed. An unreadable file is an error, not "no marker". - Drop the hard-coded sponsord_hosts group check (it broke the role for collection users); the token gate and /healthz already prove the daemon is there. Cross-check the onramp profile's chain id against an explicit sponsord_chain_id. Derive the onramp's files from sponsord_onramp_etc, and make the Caddyfile path a role var. Tighten the ACME email check, report an unreadable Turnstile secret as such, and warn while a role-owned Caddy is left behind under proxy "none". - Tests: - The stubs now use the daemon token (an authenticated /v1/info) and require ONRAMP_BIND/ONRAMP_DAEMON_URL. - verify asserts the exact X-Forwarded-For and contract addresses. - side_effect covers the no-op re-run, the record surviving a failed run, the missing-record restart, PartOf recovery, the gate page, proxy "none", ACME mode, a dead upstream behind Caddy and the package-default Caddyfile. - Seven more validation cases. - Unique Alloy harness markers. - Docs: hard rule 2 and the firewall notes name the Caddy holes, the role README says the playbooks (not the role) open them, Ubuntu needs universe, and the wording fixes from the comment review. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 21 +- README.md | 3 +- ansible/README.md | 11 +- ansible/galaxy/CHANGELOG.md | 2 +- .../molecule/sponsord-onramp/files/gate.html | 3 + .../files/sponsord-onramp-stub | 29 +- ansible/molecule/sponsord-onramp/molecule.yml | 4 +- .../molecule/sponsord-onramp/side_effect.yml | 340 +++++++++++++++++- ansible/molecule/sponsord-onramp/verify.yml | 24 +- ansible/molecule/sponsord/files/sponsord-stub | 17 +- ansible/molecule/validation/converge.yml | 122 ++++++- ansible/molecule/validation/molecule.yml | 5 - ansible/playbooks/sponsord.yml | 3 +- ansible/roles/sponsord_onramp/README.md | 35 +- .../roles/sponsord_onramp/defaults/main.yml | 26 +- ansible/roles/sponsord_onramp/tasks/caddy.yml | 53 ++- ansible/roles/sponsord_onramp/tasks/main.yml | 125 +++++-- .../sponsord_onramp/templates/Caddyfile.j2 | 8 +- .../templates/sponsord-onramp.env.j2 | 8 +- .../templates/sponsord-onramp.service.j2 | 2 +- .../roles/sponsord_onramp/vars/main/caddy.yml | 4 + ansible/tests/alloy-config/validate.sh | 32 +- ansible/tests/firewall-holes/check.sh | 28 +- docs/requirements.md | 7 +- 24 files changed, 748 insertions(+), 164 deletions(-) create mode 100644 ansible/molecule/sponsord-onramp/files/gate.html create mode 100644 ansible/roles/sponsord_onramp/vars/main/caddy.yml diff --git a/AGENTS.md b/AGENTS.md index 1a12a2e5..ba4172b0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -31,12 +31,13 @@ mirror (`vars/main/networks.yml`), never hand-copied into inventory or docs. overrides `node.toml`), keeps the keystore password off the PVC, and refuses secret-bearing keys in `config`. 2. **Localhost-only by default.** Service daemons bind `127.0.0.1` (e.g. the node's metrics - and admin RPC). A service that must accept public traffic declares exactly one hole (the - node's QUIC udp/4433) via `baseline_extra_inbound`; if a service ever needs an HTTP-facing - public path, front it with an explicit reverse proxy that terminates auth + TLS. Never - bind a *backend* to `0.0.0.0` or expose its raw port. The one HTTP-facing service today, - `sponsord-onramp`, binds loopback behind the `sponsord_onramp` role's Caddy (tcp/80 + - tcp/443), and its own gate (Turnstile) is the auth. + and admin RPC). A service that must accept public traffic declares its holes explicitly + via `baseline_extra_inbound` (today: the node's QUIC udp/4433, and tcp/80 + tcp/443 for + the `sponsord_onramp` role's Caddy). An HTTP-facing service is fronted by an explicit + reverse proxy that terminates TLS (and auth, where the service needs it), and the + backend stays on loopback. Never bind a *backend* to `0.0.0.0` or expose its raw port. + `sponsord-onramp` is public by design: its Turnstile gate guards the one action that + spends (`POST /v1/fund`), and per-address rate limits cover the rest. **Kubernetes exception (chart only):** the node's metrics bind `0.0.0.0` inside the pod so kubelet probes and Prometheus can reach them. That is allowed only behind a ClusterIP-only Service and the chart's NetworkPolicy (metrics ingress limited to @@ -163,8 +164,9 @@ addresses) the repo carries, and they carry their upstream commit. - **`ansible/roles/sponsord_onramp`** — `sponsord-onramp`, sponsord's public side (the Turnstile gate, the installers, the `decdn-sponsored` CLI API), as a second play of - `playbooks/sponsord.yml` on `sponsord_onramp_hosts` (asserted to be in `sponsord_hosts` - too: it reads the daemon's `/etc/sponsord/api-token` and calls it on loopback). + `playbooks/sponsord.yml` on `sponsord_onramp_hosts` (which must also be in + `sponsord_hosts`: it reads the daemon's `/etc/sponsord/api-token` and calls it on + loopback; the token gate and `/healthz` fail without it, no group name is checked). - **Install / unit / gate:** the sponsord role's patterns, copied: manual or GPG-verified `sponsord-onramp-v*` release (the KEYS are the sponsord role's file, via `role_path`), `DynamicUser` with the token and the operator-provisioned @@ -255,7 +257,8 @@ make build / galaxy-check # the decdn.node collection `ansible/inventory/hosts.yml` is git-ignored and a real fleet lives in a private overlay. Inventory-adjacent group_vars do not load for an overlay, so anything every host needs regardless of inventory lives in `ansible/playbooks/group_vars/`. The public firewall holes -(`baseline_extra_inbound`, today the node's udp/4433) are built from a host's groups in +(`baseline_extra_inbound`, today the node's udp/4433, plus tcp/80 + tcp/443 on +`sponsord_onramp_hosts` with `sponsord_onramp_proxy: caddy`) are built from a host's groups in `all.yml` (`_baseline_service_inbound`). `decdn_nodes.yml` and `sponsord_hosts.yml` both set `baseline_extra_inbound` to that list, so a co-located host renders one firewall in every play. `ansible/tests/firewall-holes/` (run by `make test-scripts`) pins the result per host shape. diff --git a/README.md b/README.md index cac19dcb..82a09b8f 100644 --- a/README.md +++ b/README.md @@ -83,7 +83,8 @@ This is the canonical statement; the per-path READMEs add only what is specific - **Localhost-only by default.** Backends bind `127.0.0.1`. A service that must accept public traffic declares its port explicitly, and the node declares exactly one: udp/4433. The sponsor's public onramp (Ansible only) stays on loopback too, behind - Caddy on tcp/80 + tcp/443. On Kubernetes, metrics bind `0.0.0.0` in the pod only behind a ClusterIP + Caddy on tcp/80 + tcp/443 (the default; with `sponsord_onramp_proxy: none` you bring + the proxy and open its ports). On Kubernetes, metrics bind `0.0.0.0` in the pod only behind a ClusterIP Service and a NetworkPolicy. - **Default-deny inbound** (Ansible's `baseline`, nftables). SSH is the only universally open port; extra public ports are declared via `baseline_extra_inbound`. diff --git a/ansible/README.md b/ansible/README.md index d611c0a0..870bdc6a 100644 --- a/ansible/README.md +++ b/ansible/README.md @@ -111,7 +111,8 @@ the overlay carries its own: create them next to its `hosts.yml`, starting from `inventory/group_vars/` and `inventory/host_vars/` are **not** loaded for it. Add child groups of your own when hosts differ by group (an origin, an egress cap on metered bandwidth), each with a `group_vars/.yml`. Settings every host needs regardless of -inventory, currently the public firewall holes (the node's udp/4433), live in +inventory, currently the public firewall holes (the node's udp/4433, and tcp/80 + +tcp/443 for Caddy on `sponsord_onramp_hosts`), live in `playbooks/group_vars/`, so an overlay can't drop them: `all.yml` builds the list from the host's groups and `decdn_nodes.yml` / `sponsord_hosts.yml` apply it. Override `baseline_extra_inbound` per host in `host_vars` if you have to. Playbook group_vars beat @@ -314,18 +315,18 @@ make lint-alloy # render grafana_alloy's templates, then `alloy validate` th for every role, must be rejected by their own asserts), `generate-keystore` (opt-in host-side wallet), `host-env` (host-provisioned `/etc/decdn/decdn.env`), `slow-readiness` (advisory `/metrics` probe timeout), `grafana-cloud` (the opt-in -observability wiring — see [Grafana Cloud observability](#grafana-cloud-observability-opt-in)) +observability wiring — see [Grafana Cloud observability](#grafana-cloud-observability-opt-in)), `grafana-cloud-token` (the same wiring with the API token carried through git-ignored inventory instead: role-authored env file + provenance record), `os-matrix` (default's plays on Debian 13, Ubuntu 24.04 and Ubuntu 26.04; `default` itself is Debian 12), `lifecycle` (a `decdn_network` profile with an override, both -backup scopes decrypted and checked, a rejected and a real decommission) and +backup scopes decrypted and checked, a rejected and a real decommission), `sponsord` (`playbooks/sponsord.yml` on a host with no node: generated token, credential rotation restart, the fatal `/healthz` gate, Alloy scraping sponsord and not the node; `grafana-cloud` covers sponsord co-located with a node) and `sponsord-onramp` (the onramp beside its daemon, behind the real distro Caddy with `tls internal`: the https -chain, the client address Caddy passes on, a Turnstile rotation restart, the fatal -`/healthz` gate, a refused foreign Caddyfile). +chain, the client address Caddy passes on, ACME and `none` modes, a Turnstile rotation +restart, the fatal gates, a refused foreign Caddyfile). They are independent, so they run concurrently, and each line of output is prefixed with its scenario name because the runs interleave. `make molecule-serial` is the escape hatch when that interleaving gets in the way of reading a failure. diff --git a/ansible/galaxy/CHANGELOG.md b/ansible/galaxy/CHANGELOG.md index ab40ec02..59eaa17b 100644 --- a/ansible/galaxy/CHANGELOG.md +++ b/ansible/galaxy/CHANGELOG.md @@ -11,7 +11,7 @@ collection adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) - `sponsord_onramp` role: deploys `sponsord-onramp` (decdn/sponsord), the public side of the sponsor: the Cloudflare Turnstile gate, the `decdn.sh` / `decdn.ps1` installers and the API the `decdn-sponsored` CLI polls. It runs on the daemon's - host (it asserts the host is also in `sponsord_hosts`) and reads the daemon's own + host (the token gate and `/healthz` fail without the daemon) and reads the daemon's own `/etc/sponsord/api-token`. - Install: a local binary (`sponsord_onramp_install_method: manual`, the default until upstream tags a release; `sponsord_onramp_release_target_dir` follows diff --git a/ansible/molecule/sponsord-onramp/files/gate.html b/ansible/molecule/sponsord-onramp/files/gate.html new file mode 100644 index 00000000..8f51084f --- /dev/null +++ b/ansible/molecule/sponsord-onramp/files/gate.html @@ -0,0 +1,3 @@ + + +

{{BRAND_NAME}}

{{CLIENT}}

diff --git a/ansible/molecule/sponsord-onramp/files/sponsord-onramp-stub b/ansible/molecule/sponsord-onramp/files/sponsord-onramp-stub index 1e46caef..1ebddc19 100755 --- a/ansible/molecule/sponsord-onramp/files/sponsord-onramp-stub +++ b/ansible/molecule/sponsord-onramp/files/sponsord-onramp-stub @@ -13,8 +13,12 @@ without a release or a Turnstile account: - ONRAMP_TURNSTILE_SECRET_FILE readable and non-empty; - ONRAMP_BRAND_NAME, when set, equal to EXPECTED_BRAND byte for byte (proves the role's EnvironmentFile quoting); - - the daemon at ONRAMP_DAEMON_URL answers /healthz (upstream - reads its /v1/info at start and exits if it can't). + - ONRAMP_BIND and ONRAMP_DAEMON_URL set (no fallback, so a + template that drops either is caught); + - ONRAMP_GATE_TEMPLATE, when set, readable by this unit; + - the daemon at ONRAMP_DAEMON_URL answers /v1/info for the + token in ONRAMP_DAEMON_TOKEN_FILE (upstream reads /v1/info at + start and exits if it can't). Any violation exits 1 with the reason on stderr (the journal). The marker file UNHEALTHY_MARKER makes it exit 1 on start, which is how the scenario proves the readiness gate is fatal. @@ -33,7 +37,7 @@ UNHEALTHY_MARKER = "/etc/sponsord/onramp-stub-unhealthy" # Must match sponsord_onramp_brand_name in the scenario's inventory. EXPECTED_BRAND = 'deCDN "molecule" $HOME' REQUIRED = ( - "ONRAMP_PUBLIC_URL", "ONRAMP_RPC_URL", "ONRAMP_CAPACITY_BOND_ADDR", "ONRAMP_TURNSTILE_SITEKEY", + "ONRAMP_BIND", "ONRAMP_DAEMON_URL", "ONRAMP_PUBLIC_URL", "ONRAMP_RPC_URL", "ONRAMP_CAPACITY_BOND_ADDR", "ONRAMP_TURNSTILE_SITEKEY", "ONRAMP_DECDN_RELEASE", "ONRAMP_DECDN_SUMS_SHA256", "ONRAMP_CLI_RELEASE", "ONRAMP_CLI_SUMS_SHA256", ) @@ -66,16 +70,25 @@ def check_startup(): die(f"{var} is not set") if os.environ.get("ONRAMP_BRAND_NAME", EXPECTED_BRAND) != EXPECTED_BRAND: die("ONRAMP_BRAND_NAME does not match the inventory literal (EnvironmentFile quoting broke it)") - if len(read_secret("ONRAMP_DAEMON_TOKEN_FILE")) < 32: + token = read_secret("ONRAMP_DAEMON_TOKEN_FILE") + if len(token) < 32: die("daemon token is shorter than 32 bytes") read_secret("ONRAMP_TURNSTILE_SECRET_FILE") - daemon = os.environ.get("ONRAMP_DAEMON_URL", "http://127.0.0.1:8090") + template = os.environ.get("ONRAMP_GATE_TEMPLATE") + if template: + try: + with open(template, "rb") as f: + f.read() + except OSError as e: + die(f"ONRAMP_GATE_TEMPLATE {template}: {e}") + daemon = os.environ["ONRAMP_DAEMON_URL"] + req = urllib.request.Request(f"{daemon}/v1/info", headers={"Authorization": "Bearer " + token.decode()}) try: - with urllib.request.urlopen(f"{daemon}/healthz", timeout=5) as r: + with urllib.request.urlopen(req, timeout=5) as r: if r.status != 200: die(f"daemon {daemon} answered {r.status}") except OSError as e: - die(f"daemon {daemon} is unreachable: {e}") + die(f"daemon {daemon}/v1/info failed (unreachable, or the token is not the daemon's): {e}") class Handler(BaseHTTPRequestHandler): @@ -106,7 +119,7 @@ def main(): print(f"sponsord-onramp-stub: unexpected arguments {sys.argv[1:]}", file=sys.stderr) sys.exit(2) check_startup() - host, _, port = os.environ.get("ONRAMP_BIND", "127.0.0.1:8080").rpartition(":") + host, _, port = os.environ["ONRAMP_BIND"].rpartition(":") HTTPServer((host, int(port)), Handler).serve_forever() diff --git a/ansible/molecule/sponsord-onramp/molecule.yml b/ansible/molecule/sponsord-onramp/molecule.yml index 48e53154..4fcc5694 100644 --- a/ansible/molecule/sponsord-onramp/molecule.yml +++ b/ansible/molecule/sponsord-onramp/molecule.yml @@ -22,7 +22,7 @@ # `baseline` is skipped (real-host-only — see ../default/molecule.yml), so the # tcp/80 + tcp/443 holes are covered by tests/firewall-holes instead. Release # mode is covered for the sponsord role (../sponsord), whose install tasks this -# role's mirror. The broken-input matrix lives in ../validation. +# role mirrors. The broken-input matrix lives in ../validation. driver: name: docker platforms: @@ -83,6 +83,8 @@ provisioner: sponsord_onramp_rpc_url: https://sepolia-rollup.arbitrum.io/rpc sponsord_onramp_domain: onramp.molecule.test sponsord_onramp_caddy_tls: internal + # Renders (and so validates) the Caddyfile's global email option. + sponsord_onramp_acme_email: ops@molecule.test sponsord_onramp_turnstile_sitekey: 1x00000000000000000000AA # Must equal EXPECTED_BRAND in files/sponsord-onramp-stub: quotes and a # dollar sign prove the env file's quoting ($$ is molecule's escape). diff --git a/ansible/molecule/sponsord-onramp/side_effect.yml b/ansible/molecule/sponsord-onramp/side_effect.yml index 50db1814..f35e0103 100644 --- a/ansible/molecule/sponsord-onramp/side_effect.yml +++ b/ansible/molecule/sponsord-onramp/side_effect.yml @@ -1,35 +1,61 @@ --- -# Three behaviours a green converge + idempotence cannot prove: -# 1. Rotating the Turnstile secret OUT OF BAND (the operator replaces the file) -# and re-running the role restarts the onramp onto it, and only the onramp: -# the daemon has nothing new to pick up. -# 2. The /healthz readiness gate is FATAL: an onramp that never binds (the stub -# refuses to start while its marker exists) fails the converge at the role's -# report task. -# 3. The role never silently replaces a Caddyfile it did not write: with its -# marker gone, the converge must stop at the refusal, then succeed once -# sponsord_onramp_caddy_overwrite_config confirms the takeover. -# The host is left healthy on the role's Caddyfile for verify.yml. -- name: Side effect — Turnstile rotation, the fatal readiness gate, a foreign Caddyfile +# What a green converge + idempotence cannot prove: +# 1. Rotating the Turnstile secret OUT OF BAND restarts the onramp onto it, and +# only the onramp (a no-op re-run first proves nothing else restarts it). +# 2. The /healthz readiness gate is FATAL, and a failed run records nothing: the +# restart-input record keeps the last healthy inputs until a run succeeds. +# 3. With no record, a running onramp is restarted (it was never confirmed +# healthy on its inputs), and a daemon restart brings the onramp back by +# itself (PartOf=, Restart=on-failure, no start limit). +# 4. A custom gate page is installed, read by the onramp, and removed again. +# 5. proxy "none": no client-IP header, the public URL as given, and a warning +# that the role's Caddy is still installed. +# 6. ACME mode: the Caddyfile has no `tls internal`, Caddy's http redirect +# answers, and a certificate that cannot be issued only warns. With the +# internal CA, a broken upstream behind Caddy fails the run. +# 7. The role never silently replaces a Caddyfile it did not write; the package's +# untouched default it does replace. +# The host is left healthy on the role's own Caddyfile, `tls internal`, for verify. +- name: Side effect — rotation, the fatal gate and the restart-input record hosts: all become: true + vars: + _or_record: /etc/sponsord/.onramp-inputs.sha256 tasks: - - name: Read the unit states before the rotation + - name: Read the unit states before anything changes ansible.builtin.systemd_service: name: "{{ item }}" register: _or_before loop: [sponsord-onramp, sponsord] - - name: Assert the onramp was running cleanly before the rotation + - name: Assert the onramp was running cleanly ansible.builtin.assert: that: - _or_before.results[0].status.ActiveState == 'active' - _or_before.results[0].status.MainPID | int > 0 fail_msg: >- - Expected sponsord-onramp active before the rotation (got + Expected sponsord-onramp active (got ActiveState={{ _or_before.results[0].status.ActiveState }}): journalctl -u sponsord-onramp -e + # include_role here, import_playbook in the converge: if the two contexts + # resolved any input differently, the onramp would restart for that alone, and + # the rotation check below would pass for the wrong reason. + - name: Re-run the onramp with nothing changed + ansible.builtin.include_role: + name: sponsord_onramp + + - name: Read the onramp after the no-op run + ansible.builtin.systemd_service: + name: sponsord-onramp + register: _or_noop + + - name: Require the no-op run to have left the onramp alone + ansible.builtin.assert: + that: + - _or_noop.status.MainPID == _or_before.results[0].status.MainPID + fail_msg: A re-run with no input changed restarted sponsord-onramp. + - name: Rotate the Turnstile secret out of band ansible.builtin.copy: dest: /etc/sponsord/turnstile-secret @@ -58,6 +84,12 @@ (must change) and the daemon's {{ _or_before.results[1].status.MainPID }} -> {{ _or_after.results[1].status.MainPID }} (must not). + - name: Hash the Turnstile secret the onramp is now healthy on + ansible.builtin.stat: + path: /etc/sponsord/turnstile-secret + checksum_algorithm: sha256 + register: _or_secret_b + - name: Stop the onramp, then make it refuse to start ansible.builtin.systemd_service: name: sponsord-onramp @@ -69,6 +101,13 @@ content: "" mode: "0644" + # A changed input on the failing run: the record must not take it. + - name: Rotate the Turnstile secret once more + ansible.builtin.copy: + dest: /etc/sponsord/turnstile-secret + content: "1x0000000000000000000000000000000CC\n" + mode: "0600" + - name: Converge against an onramp that never becomes healthy block: - name: Run the role with a short readiness window @@ -89,6 +128,20 @@ fail_msg: >- Expected the role to fail at its readiness report, but it failed at "{{ ansible_failed_task.name }}". + + - name: Read the restart-input record after the failed run + ansible.builtin.slurp: + src: "{{ _or_record }}" + register: _or_rec_failed + + # Field order: token, Turnstile secret, env file, gate page, unit, binary. + - name: Require the failed run to have left the record on the healthy inputs + ansible.builtin.assert: + that: + - (_or_rec_failed.content | b64decode).split()[1] == _or_secret_b.stat.checksum + fail_msg: >- + The record moved to the inputs of a run whose onramp never became + healthy; the next converge would not restart it onto them. always: - name: Remove the refuse-to-start marker ansible.builtin.file: @@ -99,8 +152,235 @@ ansible.builtin.include_role: name: sponsord_onramp - # A Caddyfile without the marker on a host where Caddy is already installed is - # exactly what an operator's own Caddy looks like to the role. + - name: Hash the Turnstile secret now in place + ansible.builtin.stat: + path: /etc/sponsord/turnstile-secret + checksum_algorithm: sha256 + register: _or_secret_c + + - name: Read the restart-input record after the recovery + ansible.builtin.slurp: + src: "{{ _or_record }}" + register: _or_rec_ok + + - name: Require the record to follow the inputs once the onramp is healthy + ansible.builtin.assert: + that: + - (_or_rec_ok.content | b64decode).split()[1] == _or_secret_c.stat.checksum + fail_msg: The healthy run did not record the Turnstile secret it started on. + +- name: Side effect — a missing record, and a daemon restart + hosts: all + become: true + tasks: + - name: Read the onramp before the record is removed + ansible.builtin.systemd_service: + name: sponsord-onramp + register: _or_norec_before + + - name: Remove the restart-input record + ansible.builtin.file: + path: /etc/sponsord/.onramp-inputs.sha256 + state: absent + + - name: Converge with no record + ansible.builtin.include_role: + name: sponsord_onramp + + - name: Read the onramp and the record after that converge + ansible.builtin.systemd_service: + name: sponsord-onramp + register: _or_norec_after + + - name: Stat the rewritten record + ansible.builtin.stat: + path: /etc/sponsord/.onramp-inputs.sha256 + register: _or_norec_rec + + - name: Require a restart and a fresh record + ansible.builtin.assert: + that: + - _or_norec_after.status.MainPID != _or_norec_before.status.MainPID + - _or_norec_rec.stat.exists + fail_msg: >- + With no record, a running onramp must be restarted (it was never confirmed + healthy on its inputs) and the record written again. + + - name: Restart the daemon + ansible.builtin.systemd_service: + name: sponsord + state: restarted + + # The onramp restarts with the daemon (PartOf=), usually before the daemon + # listens; it must exit, retry and come back on its own, with no converge. + - name: Wait for the onramp to come back by itself + ansible.builtin.uri: + url: http://127.0.0.1:8080/healthz + status_code: 200 + timeout: 5 + register: _or_partof + retries: 30 + delay: 1 + until: _or_partof.status == 200 + + - name: Read the onramp after the daemon restart + ansible.builtin.systemd_service: + name: sponsord-onramp + register: _or_partof_unit + + - name: Require the onramp to have restarted with the daemon + ansible.builtin.assert: + that: + - _or_partof_unit.status.ActiveState == 'active' + - _or_partof_unit.status.MainPID != _or_norec_after.status.MainPID + fail_msg: The onramp did not restart with the daemon (PartOf=sponsord.service). + +- name: Side effect — a custom gate page + hosts: all + become: true + vars: + _or_page: "{{ lookup('ansible.builtin.env', 'MOLECULE_SCENARIO_DIRECTORY') }}/files/gate.html" + tasks: + - name: Converge with a custom gate page + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_gate_template_src: "{{ _or_page }}" + + - name: Stat the installed page and read the env file + ansible.builtin.stat: + path: /etc/sponsord/onramp-gate.html + register: _or_page_stat + + - name: Read the env file with the page set + ansible.builtin.slurp: + src: /etc/sponsord/sponsord-onramp.env + register: _or_page_env + + # The stub reads the page at start, as the onramp's dynamic user: the role's + # /healthz gate passing already proves it was readable. + - name: Require the page installed root-owned, world-readable and configured + ansible.builtin.assert: + that: + - _or_page_stat.stat.isreg + - _or_page_stat.stat.mode == '0644' + - _or_page_stat.stat.pw_name == 'root' + - "'ONRAMP_GATE_TEMPLATE=/etc/sponsord/onramp-gate.html' in (_or_page_env.content | b64decode)" + fail_msg: The custom gate page is not installed, or not configured. + + - name: Converge without the page again + ansible.builtin.include_role: + name: sponsord_onramp + + - name: Stat the page after it was dropped + ansible.builtin.stat: + path: /etc/sponsord/onramp-gate.html + register: _or_page_gone + + - name: Read the env file without the page + ansible.builtin.slurp: + src: /etc/sponsord/sponsord-onramp.env + register: _or_page_env_gone + + - name: Require the page and its setting gone + ansible.builtin.assert: + that: + - not _or_page_gone.stat.exists + - "'ONRAMP_GATE_TEMPLATE' not in (_or_page_env_gone.content | b64decode)" + fail_msg: The dropped gate page is still installed or configured. + +- name: Side effect — bring your own proxy + hosts: all + become: true + tasks: + - name: Converge with proxy "none" + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_proxy: none + sponsord_onramp_public_url: https://downloads.molecule.test + + - name: Read the env file in proxy "none" mode + ansible.builtin.slurp: + src: /etc/sponsord/sponsord-onramp.env + register: _or_none_env + + # With an arbitrary proxy in front, trusting X-Forwarded-For would let clients + # pick the address the rate limits key on. + - name: Require no client-IP header, the given public URL, and the stale-Caddy warning + ansible.builtin.assert: + that: + - "'ONRAMP_CLIENT_IP_HEADER' not in _env" + - "'ONRAMP_PUBLIC_URL=https://downloads.molecule.test\n' in _env" + - sponsord_onramp_stale_caddy.rc == 0 + fail_msg: "{{ _env }}" + vars: + _env: "{{ _or_none_env.content | b64decode }}" + +- name: Side effect — ACME, and a broken upstream behind Caddy + hosts: all + become: true + tasks: + # No public DNS points at a container, so no certificate can be issued: the + # role must still finish, on the redirect probe, with only the ACME warning. + - name: Converge in ACME mode + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_caddy_tls: acme + sponsord_onramp_readiness_retries: 2 + + - name: Read the ACME-mode Caddyfile + ansible.builtin.slurp: + src: /etc/caddy/Caddyfile + register: _or_acme_caddy + + - name: Require ACME's Caddyfile, and the certificate failure judged advisory + ansible.builtin.assert: + that: + - "'tls internal' not in (_or_acme_caddy.content | b64decode)" + - sponsord_onramp_caddy_redirect.status == 308 + - sponsord_onramp_caddy_probe.rc in [35, 60] + fail_msg: >- + ACME mode: tls internal still present, the redirect probe got + {{ sponsord_onramp_caddy_redirect.status | default('none') }}, or curl exit + {{ sponsord_onramp_caddy_probe.rc }} is not a pending-certificate failure. + + - name: Gather package facts for the Caddy tasks run alone + ansible.builtin.package_facts: + + # Caddy pointed at a port nothing listens on: TLS works (internal CA), the + # upstream answers 502, and that must fail the run. + - name: Run the Caddy tasks against a dead upstream + block: + - name: Point Caddy at a port nothing listens on + ansible.builtin.include_role: + name: sponsord_onramp + tasks_from: caddy + vars: + sponsord_onramp_port: 8099 + sponsord_onramp_readiness_retries: 2 + + - name: Fail — the Caddy probe let a dead upstream through + ansible.builtin.fail: + msg: The role finished although Caddy could not reach the onramp. + rescue: + - name: Require the failure to come from the Caddy probe report + ansible.builtin.assert: + that: + - ansible_failed_task.name == 'Report an onramp that is not reachable through Caddy' + fail_msg: >- + Expected the Caddy probe to fail the run, but it failed at + "{{ ansible_failed_task.name }}". + + - name: Converge back onto the internal CA and the real upstream + ansible.builtin.include_role: + name: sponsord_onramp + +- name: Side effect — a Caddyfile someone else wrote, and the package's default + hosts: all + become: true + tasks: - name: Replace the Caddyfile with one this role did not write ansible.builtin.copy: dest: /etc/caddy/Caddyfile @@ -146,3 +426,29 @@ name: sponsord_onramp vars: sponsord_onramp_caddy_overwrite_config: true + + # What a first run leaves when it installs Caddy and stops before templating. + - name: Put the package's own default Caddyfile back + ansible.builtin.shell: + executable: /bin/bash + cmd: | + set -euo pipefail + cd "$(mktemp -d)" + apt-get download caddy >/dev/null + dpkg-deb --fsys-tarfile caddy_*.deb | tar -xO ./etc/caddy/Caddyfile > /etc/caddy/Caddyfile + changed_when: true + + - name: Converge over the package's default Caddyfile, unconfirmed + ansible.builtin.include_role: + name: sponsord_onramp + + - name: Read the Caddyfile after replacing the default + ansible.builtin.slurp: + src: /etc/caddy/Caddyfile + register: _or_after_default + + - name: Require the role's Caddyfile in place of the default + ansible.builtin.assert: + that: + - "'MANAGED BY the sponsord_onramp role' in (_or_after_default.content | b64decode)" + fail_msg: The role did not take over the caddy package's untouched default Caddyfile. diff --git a/ansible/molecule/sponsord-onramp/verify.yml b/ansible/molecule/sponsord-onramp/verify.yml index 4541f2f5..15d833d6 100644 --- a/ansible/molecule/sponsord-onramp/verify.yml +++ b/ansible/molecule/sponsord-onramp/verify.yml @@ -6,6 +6,11 @@ become: true gather_facts: false tasks: + - name: Load the onramp's network profile mirror + ansible.builtin.include_vars: + # Relative to this playbook's directory. + file: ../../roles/sponsord_onramp/vars/main/networks.yml + - name: Stat the onramp's files ansible.builtin.stat: path: "{{ item }}" @@ -41,9 +46,9 @@ - "'ONRAMP_DAEMON_URL=http://127.0.0.1:8090' in _env" - "'ONRAMP_PUBLIC_URL=https://onramp.molecule.test' in _env" - "'ONRAMP_RPC_URL=https://sepolia-rollup.arbitrum.io/rpc' in _env" - # From the network profile mirror (the daemon's sponsord_network). - - "'ONRAMP_CAPACITY_BOND_ADDR=0x' in _env" - - "'ONRAMP_SLASH_JUDGE_ADDR=0x' in _env" + # Exactly the mirror's addresses for the daemon's sponsord_network. + - "('ONRAMP_CAPACITY_BOND_ADDR=' ~ _profile.capacity_bond_address ~ '\n') in _env" + - "('ONRAMP_SLASH_JUDGE_ADDR=' ~ _profile.slash_judge_address ~ '\n') in _env" - "'ONRAMP_CLIENT_IP_HEADER=X-Forwarded-For' in _env" - "'ONRAMP_FUND_RATE_PER_MIN=0' in _env" - "'ONRAMP_TTL_SECS' not in _env" @@ -53,6 +58,7 @@ fail_msg: "{{ _env }}" vars: _env: "{{ _or_text.results[0].content | b64decode }}" + _profile: "{{ _sponsord_onramp_network_profiles['arbitrum-sepolia'] }}" - name: Require the role's Caddyfile, with its admin API off ansible.builtin.assert: @@ -61,6 +67,8 @@ - "'admin off' in _caddy" - "'onramp.molecule.test {' in _caddy" - "'tls internal' in _caddy" + - "'protocols h1 h2' in _caddy" + - "'email ops@molecule.test' in _caddy" - "'reverse_proxy 127.0.0.1:8080' in _caddy" - "'not the onramp' not in _caddy" fail_msg: "{{ _caddy }}" @@ -143,13 +151,15 @@ register: _or_xff changed_when: false - - name: Require the right-most address to be the real client + # Caddy without trusted_proxies REPLACES the header, so the forged address must + # be gone entirely, not just pushed left of the real one. + - name: Require the client address Caddy saw, and nothing else ansible.builtin.assert: that: - - (_or_xff.stdout.split(',') | last | trim) == '127.0.0.1' + - (_or_xff.stdout | trim) == '127.0.0.1' fail_msg: >- - The onramp saw X-Forwarded-For "{{ _or_xff.stdout }}": its right-most address - must be the client Caddy saw (127.0.0.1), not the forged 203.0.113.9. + The onramp saw X-Forwarded-For "{{ _or_xff.stdout }}": it must be exactly the + client Caddy saw (127.0.0.1), without the forged 203.0.113.9. - name: Read the Alloy config ansible.builtin.slurp: diff --git a/ansible/molecule/sponsord/files/sponsord-stub b/ansible/molecule/sponsord/files/sponsord-stub index 5f41da7f..f34538f7 100755 --- a/ansible/molecule/sponsord/files/sponsord-stub +++ b/ansible/molecule/sponsord/files/sponsord-stub @@ -6,7 +6,8 @@ a release, a chain or a funded treasury: * `--version` -> print a version line and exit 0 (the role's version backstop). * no argument -> enforce the startup contract the real daemon enforces, then - serve /healthz and /metrics on SPONSORD_BIND until killed: + serve /healthz, /metrics and a bearer-checked /v1/info on + SPONSORD_BIND until killed: - SPONSORD_RPC_URL, _CHAIN_ID, _PAYMENT_POOL_ADDR, _POOL_ID set (_CHAIN_ID is stricter than upstream, which defaults it; the role always renders it); @@ -41,6 +42,11 @@ UNHEALTHY_MARKER = "/etc/sponsord/stub-unhealthy" EXPECTED_RPC_URL = "https://rpc.example.invalid/?key=$HOME$(touch /tmp/sponsord-rpc-expanded)&q=o'brien \"x\"" +# Set by check_startup; /v1/info answers only a caller presenting it, as the real +# daemon's authenticated routes do (the onramp reads /v1/info at start). +API_TOKEN = b"" + + def die(msg): print(f"sponsord-stub: {msg}", file=sys.stderr) sys.exit(1) @@ -69,7 +75,9 @@ def check_startup(): die(f"{var} is not set") if os.environ.get("SPONSORD_MOLECULE_RPC_LITERAL") == "1" and os.environ["SPONSORD_RPC_URL"] != EXPECTED_RPC_URL: die("SPONSORD_RPC_URL does not match the inventory literal (EnvironmentFile quoting broke it)") - if len(read_secret("SPONSORD_API_TOKEN_FILE")) < 32: + global API_TOKEN + API_TOKEN = read_secret("SPONSORD_API_TOKEN_FILE") + if len(API_TOKEN) < 32: die("API token is shorter than 32 bytes") read_secret("SPONSORD_TREASURY_PASSWORD_FILE") keystore = os.environ.get("SPONSORD_TREASURY_KEYSTORE") @@ -97,6 +105,11 @@ class Handler(BaseHTTPRequestHandler): body, ctype = b'{"ok":true}', "application/json" elif self.path == "/metrics": body, ctype = b"sponsord_pool_topups_total 0\n", "text/plain; version=0.0.4" + elif self.path == "/v1/info": + if self.headers.get("Authorization", "").encode() != b"Bearer " + API_TOKEN: + self.send_error(401) + return + body, ctype = b'{"stub":true}', "application/json" else: self.send_error(404) return diff --git a/ansible/molecule/validation/converge.yml b/ansible/molecule/validation/converge.yml index 618a11f7..70d18c7a 100644 --- a/ansible/molecule/validation/converge.yml +++ b/ansible/molecule/validation/converge.yml @@ -24,8 +24,9 @@ # post-template `decdn config validate` gate still fails the deploy. It therefore # mutates the host, which is why this scenario has a prepare.yml staging wallet # material. It runs after the decdn_node assert-only cases so it cannot interfere -# with them; the grafana_alloy and sponsord cases follow it and clean up after -# themselves. +# with them; the grafana_alloy, sponsord and (in a second play) sponsord_onramp +# cases follow it and clean up after themselves (the onramp section notes its own +# host-state cases). - name: Converge (expect validation failures) hosts: all become: true @@ -2130,8 +2131,8 @@ sponsord_onramp_cli_sums_sha256: "2222222222222222222222222222222222222222222222222222222222222222" tasks: # --- sponsord_onramp: each case breaks ONE input over the valid baseline in play - # vars. All but the last three abort at an assert before any host mutation; those - # stage /etc/sponsord files and remove them again. + # vars. All but the last four abort at an assert before any host mutation; those + # stage /etc/sponsord (and /etc/caddy) files and remove them again. - name: "Case onramp-release-version — release mode with no pinned version" block: - name: Run sponsord_onramp with release mode with no pinned version @@ -2411,6 +2412,84 @@ decdn_rejected: "{{ decdn_rejected + ['onramp-ip-header'] }}" when: ansible_failed_task.name == "Validate the client IP header name" + - name: "Case onramp-network-chain — a daemon chain id other than the onramp's profile" + block: + - name: Run sponsord_onramp with a daemon chain id other than the onramp's profile + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_chain_id: 1 + rescue: + - name: Record onramp-network-chain rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-network-chain'] }}" + when: ansible_failed_task.name == "Require the onramp's network to be the daemon's chain" + + - name: "Case onramp-proxy — a quoted overwrite confirmation" + block: + - name: Run sponsord_onramp with a quoted overwrite confirmation + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_caddy_overwrite_config: "false" + rescue: + - name: Record onramp-proxy rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-proxy'] }}" + when: ansible_failed_task.name == "Validate the proxy settings" + + - name: "Case onramp-proxy — a misspelt TLS mode" + block: + - name: Run sponsord_onramp with a misspelt TLS mode + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_caddy_tls: interal + rescue: + - name: Record onramp-proxy rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-proxy'] }}" + when: ansible_failed_task.name == "Validate the proxy settings" + + - name: "Case onramp-domain — an ACME email that would break the Caddyfile" + block: + - name: Run sponsord_onramp with an ACME email that would break the Caddyfile + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_acme_email: "ops}@example.org" + rescue: + - name: Record onramp-domain rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-domain'] }}" + when: ansible_failed_task.name == "Require a domain for Caddy" + + - name: "Case onramp-rpc — a public RPC URL that injects a second variable" + block: + - name: Run sponsord_onramp with a public RPC URL that injects a second variable + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_rpc_url: "https://rpc.example.invalid/a\nONRAMP_BIND=0.0.0.0:8080" + rescue: + - name: Record onramp-rpc rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-rpc'] }}" + when: ansible_failed_task.name == "Require a public RPC URL with no credentials in it" + + - name: "Case onramp-limit — a rate limit with a unit" + block: + - name: Run sponsord_onramp with a rate limit with a unit + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_poll_rate_per_min: "10m" + rescue: + - name: Record onramp-limit rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-limit'] }}" + when: ansible_failed_task.name == "Validate the onramp's limits" + - name: "Case onramp-token — no daemon API token on the host" block: - name: Run sponsord_onramp with no daemon API token on the host @@ -2504,6 +2583,37 @@ path: /etc/sponsord state: absent + # Caddy run from a static binary leaves no dpkg trace: the file alone decides. + - name: "Case onramp-caddyfile — a foreign Caddyfile with no caddy package" + block: + - name: Stage the API token, the Turnstile secret and a foreign Caddyfile + ansible.builtin.shell: + cmd: | + set -eu + mkdir -p /etc/sponsord /etc/caddy + umask 077 + printf '%064d' 0 > /etc/sponsord/api-token + echo 'not-empty' > /etc/sponsord/turnstile-secret + printf 'other.example {\n respond "theirs"\n}\n' > /etc/caddy/Caddyfile + changed_when: true + + - name: Run sponsord_onramp against the foreign Caddyfile + ansible.builtin.include_role: + name: sponsord_onramp + rescue: + - name: Record onramp-caddyfile rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-caddyfile'] }}" + when: ansible_failed_task.name == "Refuse to replace a Caddyfile this role did not write" + always: + - name: Remove the foreign-Caddyfile fixtures + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - /etc/sponsord + - /etc/caddy + - name: Confirm every bad value was rejected by the role's own validation ansible.builtin.assert: that: @@ -2549,4 +2659,6 @@ "onramp-domain", "onramp-public-url", "onramp-public-url", "onramp-rpc", "onramp-rpc", "onramp-rpc", "onramp-rpc", "onramp-public-url", "onramp-sitekey", "onramp-pin", "onramp-pin", "onramp-limit", "onramp-ip-header", "onramp-token", - "onramp-turnstile", "onramp-turnstile", "onramp-turnstile-empty"] + "onramp-turnstile", "onramp-turnstile", "onramp-turnstile-empty", + "onramp-network-chain", "onramp-proxy", "onramp-proxy", "onramp-domain", + "onramp-rpc", "onramp-limit", "onramp-caddyfile"] diff --git a/ansible/molecule/validation/molecule.yml b/ansible/molecule/validation/molecule.yml index 7e131d14..983c1622 100644 --- a/ansible/molecule/validation/molecule.yml +++ b/ansible/molecule/validation/molecule.yml @@ -26,11 +26,6 @@ platforms: cgroupns_mode: host volumes: - /sys/fs/cgroup:/sys/fs/cgroup:rw - # sponsord_onramp asserts its host also runs the daemon; the cases include the - # roles directly, so membership has no other effect here. - groups: - - sponsord_hosts - - sponsord_onramp_hosts provisioner: name: ansible env: diff --git a/ansible/playbooks/sponsord.yml b/ansible/playbooks/sponsord.yml index 001215bd..9d0af087 100644 --- a/ansible/playbooks/sponsord.yml +++ b/ansible/playbooks/sponsord.yml @@ -26,7 +26,8 @@ # After the daemon's play, whose /healthz gate is fatal, so the onramp starts # against a daemon that is up. Baseline and Alloy already ran for these hosts -# above (the role asserts each one is also in sponsord_hosts). +# above for hosts that are also in sponsord_hosts, which every one must be: the +# onramp uses the daemon on its own host. - name: Provision sponsord-onramp hosts: sponsord_onramp_hosts become: true diff --git a/ansible/roles/sponsord_onramp/README.md b/ansible/roles/sponsord_onramp/README.md index 2719b012..f113e0bc 100644 --- a/ansible/roles/sponsord_onramp/README.md +++ b/ansible/roles/sponsord_onramp/README.md @@ -11,8 +11,10 @@ the public side of the deCDN onboarding sponsor. It serves: It holds no wallet. For each person who passes the gate it asks the [`sponsord`](../sponsord/README.md) daemon on the same host for a capability, using -the daemon's API token. So the host must be in `sponsord_hosts` as well as -`sponsord_onramp_hosts`, and the role checks that it is. +the daemon's API token. So the daemon must run on the same host: in this repo, list +the host in `sponsord_hosts` as well as `sponsord_onramp_hosts`. The role fails +early when the daemon's token file is missing, and its `/healthz` gate fails when +the daemon is unreachable. ## What it does @@ -48,19 +50,24 @@ the daemon's API token. So the host must be in `sponsord_hosts` as well as every restart input (the token, the Turnstile secret, the env file, the gate page, the unit, the binary) restarts the onramp when one changes outside the role. - **Puts Caddy in front** (`sponsord_onramp_proxy: caddy`, the default): - - Installs the distribution's `caddy` package. No third-party apt repository is - added. + - Installs the distribution's `caddy` package (Debian main; on Ubuntu it is in + **universe**, which must be enabled). No third-party apt repository is added. - Writes `/etc/caddy/Caddyfile`: TLS for `sponsord_onramp_domain`, then a reverse proxy to the onramp on loopback. The file is validated with `caddy validate` - before it lands, and the admin API is off, so nothing on the host can - reconfigure the proxy. - - Opens tcp/80 (ACME challenge, https redirect) and tcp/443, through - `playbooks/group_vars/all.yml`. - - Probes `https:///healthz` through Caddy on the host. With ACME a failure - only warns, since the certificate can lag DNS. With `tls internal` it fails the - deploy. - - Refuses to replace a Caddyfile it did not write on a host where Caddy was - already installed, unless you set `sponsord_onramp_caddy_overwrite_config: true`. + before it lands. The admin API is off, so nothing on the host can reconfigure + the running proxy over its admin endpoint (:2019), and HTTP/3 is off, so udp/443 + stays closed. + - Needs tcp/80 (ACME challenge, https redirect) and tcp/443 open. This repo's + playbooks open both (`playbooks/group_vars/all.yml`); from your own playbook, add + them to `baseline_extra_inbound` yourself. + - Probes the domain through Caddy on the host. Plain http must answer with Caddy's + https redirect, and `https:///healthz` must answer through TLS; both + failures are fatal. One exception: with ACME, a certificate that is not issued + yet (the TLS handshake fails, or the certificate cannot be verified) only warns, + since it can lag DNS. + - Refuses to replace a `/etc/caddy/Caddyfile` that it did not write and that is + not the caddy package's untouched default, unless you set + `sponsord_onramp_caddy_overwrite_config: true`. ## Before the first deploy @@ -126,7 +133,7 @@ only way to reach the onramp: `CF-Connecting-IP` behind Cloudflare, or `X-Forwarded-For` if your proxy appends the client address. Otherwise clients can choose their own address and get around the rate limits. Switching from `caddy` to `none` closes the firewall holes but leaves the Caddy package installed and -running; remove it yourself. +running on 80/443; the role warns about it on every run until you remove it. ## Observability diff --git a/ansible/roles/sponsord_onramp/defaults/main.yml b/ansible/roles/sponsord_onramp/defaults/main.yml index 5b93a425..363338cc 100644 --- a/ansible/roles/sponsord_onramp/defaults/main.yml +++ b/ansible/roles/sponsord_onramp/defaults/main.yml @@ -78,14 +78,14 @@ sponsord_onramp_public_url: >- # "internal": Caddy's own local CA, for staging and tests; browsers will not trust it. sponsord_onramp_caddy_tls: acme # "acme" | "internal" sponsord_onramp_acme_email: "" # optional ACME account contact -# The role owns /etc/caddy/Caddyfile. On a host where Caddy was installed before -# this role and that file lacks the role's marker, the role refuses to replace it -# unless this is true. +# The role owns /etc/caddy/Caddyfile. If that file exists without the role's marker +# and is not the package's untouched default, it belongs to someone else, and the +# role refuses to replace it unless this is true. sponsord_onramp_caddy_overwrite_config: false -sponsord_onramp_caddyfile: /etc/caddy/Caddyfile # The header Caddy (or your proxy) puts the client address in; the onramp rate -# limits and gates by it. Caddy without trusted_proxies replaces any incoming -# X-Forwarded-For, so the right-most address is the real client. With your own +# limits and gates by it. Caddy (without trusted_proxies) sets X-Forwarded-For to +# the client it saw, discarding any client-sent value, and the onramp uses the +# right-most entry, so clients cannot choose their address. With your own # proxy, set it only if that proxy is the ONLY way in (CF-Connecting-IP behind # Cloudflare); otherwise clients pick their own address. "" = the TCP peer. sponsord_onramp_client_ip_header: "{{ 'X-Forwarded-For' if sponsord_onramp_proxy == 'caddy' else '' }}" @@ -118,15 +118,17 @@ sponsord_onramp_fund_rate_per_min: "" # [10] POST /v1/fund per address; sponsord_onramp_poll_rate_per_min: "" # [120] GET /v1/capability per address; 0 = off # --- Files on the host ---------------------------------------------------------- -# Under the daemon's directory: the onramp reads the daemon's own API token. +# Beside the daemon's files: the directory and the API token follow the sponsord +# role's sponsord_etc / sponsord_api_token_file when inventory sets them (the +# fallbacks match roles/sponsord/defaults/main.yml). sponsord_onramp_etc: "{{ sponsord_etc | default('/etc/sponsord') }}" -sponsord_onramp_api_token_file: "{{ sponsord_api_token_file | default('/etc/sponsord/api-token') }}" -sponsord_onramp_turnstile_secret_file: /etc/sponsord/turnstile-secret # operator-provisioned (0600) -sponsord_onramp_env_file: /etc/sponsord/sponsord-onramp.env # non-secret (0644, role-owned) -sponsord_onramp_gate_template_file: /etc/sponsord/onramp-gate.html # when _src is set (0644) +sponsord_onramp_api_token_file: "{{ sponsord_api_token_file | default(sponsord_onramp_etc ~ '/api-token') }}" +sponsord_onramp_turnstile_secret_file: "{{ sponsord_onramp_etc }}/turnstile-secret" # operator-provisioned (0600) +sponsord_onramp_env_file: "{{ sponsord_onramp_etc }}/sponsord-onramp.env" # non-secret (0644, role-owned) +sponsord_onramp_gate_template_file: "{{ sponsord_onramp_etc }}/onramp-gate.html" # when _src is set (0644) # sha256 record of every restart input (token, Turnstile secret, env file, gate # page, unit, binary) as of the last converge that saw the onramp healthy. -sponsord_onramp_inputs_checksum_file: /etc/sponsord/.onramp-inputs.sha256 +sponsord_onramp_inputs_checksum_file: "{{ sponsord_onramp_etc }}/.onramp-inputs.sha256" sponsord_onramp_bin: /usr/local/bin/sponsord-onramp diff --git a/ansible/roles/sponsord_onramp/tasks/caddy.yml b/ansible/roles/sponsord_onramp/tasks/caddy.yml index d73bbe6b..79b7cfea 100644 --- a/ansible/roles/sponsord_onramp/tasks/caddy.yml +++ b/ansible/roles/sponsord_onramp/tasks/caddy.yml @@ -1,10 +1,11 @@ --- # Caddy in front of the onramp: the host's only public HTTP listener (tcp/80 for -# the ACME challenge and the https redirect, tcp/443). playbooks/group_vars/all.yml -# opens both in the baseline firewall for hosts in sponsord_onramp_hosts while -# sponsord_onramp_proxy is "caddy". The distro package brings its own hardened unit -# (user caddy, CAP_NET_BIND_SERVICE only); main.yml has already refused a Caddy -# this role does not own. +# the ACME challenge and the https redirect, tcp/443). This repo's +# playbooks/group_vars/all.yml opens both in the baseline firewall for hosts in +# sponsord_onramp_hosts while sponsord_onramp_proxy is "caddy". The distro package +# brings its own unit (user caddy, CAP_NET_BIND_SERVICE as its only ambient +# capability, PrivateTmp, ProtectSystem=full); main.yml has already refused a +# Caddyfile this role does not own. # curl: the probe at the end of this file. - name: Install Caddy from the distribution @@ -15,11 +16,11 @@ cache_valid_time: 3600 # `caddy validate` loads the config the way `caddy run` does (it also provisions -# the TLS automation), so a Caddyfile that would not start never lands. +# the TLS automation), so a Caddyfile that does not parse or provision never lands. - name: Template the Caddyfile ansible.builtin.template: src: Caddyfile.j2 - dest: "{{ sponsord_onramp_caddyfile }}" + dest: "{{ _sponsord_onramp_caddyfile }}" owner: root group: root mode: "0644" @@ -38,10 +39,28 @@ state: started when: not ansible_check_mode or 'caddy' in ansible_facts.packages +# Caddy serves the site at all: plain http answers with its https redirect. No +# certificate is involved, so this holds from the first run, ACME or not. +- name: Probe Caddy's http redirect for the domain + ansible.builtin.uri: + url: http://127.0.0.1/healthz + headers: + Host: "{{ sponsord_onramp_domain }}" + follow_redirects: none + status_code: 308 + timeout: 5 + register: sponsord_onramp_caddy_redirect + retries: "{{ sponsord_onramp_readiness_retries }}" + delay: "{{ sponsord_onramp_readiness_delay }}" + until: sponsord_onramp_caddy_redirect.status | default(-1) == 308 + changed_when: false + when: not ansible_check_mode + # The whole chain, as a user would reach it but pinned to this host: TLS for the -# domain, the proxy, the onramp. With ACME the certificate can lag (DNS not yet -# pointed here, a rate limit), so that probe only warns; with Caddy's internal CA -# nothing outside the host is involved, so it is fatal. +# domain, the proxy, the onramp. Fatal, except that with ACME a certificate that is +# not issued yet (DNS not pointed here, a rate limit) only warns: Caddy then aborts +# the handshake (curl 35) or serves one curl cannot verify (60). Any other failure +# (a 502 from a dead upstream, nothing on 443, a hang) is a fault either way. - name: Probe the onramp through Caddy ansible.builtin.command: argv: >- @@ -66,20 +85,20 @@ sudo journalctl -u caddy -e when: - not ansible_check_mode - - sponsord_onramp_caddy_tls == 'internal' - sponsord_onramp_caddy_probe.rc != 0 + - sponsord_onramp_caddy_tls == 'internal' or sponsord_onramp_caddy_probe.rc not in [35, 60] - name: Warn that the public certificate is not in place yet ansible.builtin.debug: msg: |- - WARNING: https://{{ sponsord_onramp_domain }}/healthz did not answer through Caddy - on this host yet (curl exit {{ sponsord_onramp_caddy_probe.rc }}: - {{ sponsord_onramp_caddy_probe.stderr | trim }}). With ACME that usually means the - certificate is not issued yet: check that the domain's A/AAAA records point at - this host and that tcp/80 and tcp/443 reach it, then read + WARNING: https://{{ sponsord_onramp_domain }} has no trusted certificate yet + (curl exit {{ sponsord_onramp_caddy_probe.rc }}: {{ sponsord_onramp_caddy_probe.stderr | trim }}). + Caddy serves the site (its http redirect answered), so ACME has not finished: + check that the domain's A/AAAA records point at this host and that tcp/80 and + tcp/443 reach it, then read sudo journalctl -u caddy -e Users cannot reach the onramp until this answers. when: - not ansible_check_mode - sponsord_onramp_caddy_tls == 'acme' - - sponsord_onramp_caddy_probe.rc != 0 + - sponsord_onramp_caddy_probe.rc in [35, 60] diff --git a/ansible/roles/sponsord_onramp/tasks/main.yml b/ansible/roles/sponsord_onramp/tasks/main.yml index d94950f5..d14841f0 100644 --- a/ansible/roles/sponsord_onramp/tasks/main.yml +++ b/ansible/roles/sponsord_onramp/tasks/main.yml @@ -5,17 +5,10 @@ # before the binary is installed. The Turnstile widget and the release pins are # operator inputs — see roles/sponsord_onramp/README.md. -# --- Placement --------------------------------------------------------------------- -# The onramp calls the daemon on loopback with the daemon's own API token, and the -# sponsord play that runs before this one is what makes the daemon healthy. -- name: Require the host to run the sponsord daemon too - ansible.builtin.assert: - that: - - "'sponsord_hosts' in group_names" - fail_msg: >- - {{ inventory_hostname }} is in sponsord_onramp_hosts but not in sponsord_hosts. - The onramp needs the sponsord daemon on the same host (it reads the daemon's - API token and calls it on loopback). Add the host to sponsord_hosts too. +# The onramp needs the sponsord daemon on the same host: it reads the daemon's API +# token file and calls it on loopback. Nothing here names an inventory group; the +# token gate below and the /healthz gate (the onramp binds only after reaching the +# daemon) are what prove the daemon is there. # --- Install method + target ---------------------------------------------------- - name: Require a valid install method @@ -181,6 +174,22 @@ - sponsord_onramp_network | length > 0 - sponsord_network | default('') | length > 0 +# Also when the daemon has no profile but an explicit chain id: the onramp's +# profile must be that chain's. +- name: Require the onramp's network to be the daemon's chain + ansible.builtin.assert: + that: + - _sponsord_onramp_profile.chain_id | string == sponsord_chain_id | string + fail_msg: >- + sponsord_onramp_network "{{ sponsord_onramp_network }}" is chain + {{ _sponsord_onramp_profile.chain_id }}, but the daemon's sponsord_chain_id is + {{ sponsord_chain_id }}. Its CapacityBond and SlashJudge would belong to another + chain. Set the onramp's contract addresses for chain {{ sponsord_chain_id }} + explicitly and clear sponsord_onramp_network. + when: + - sponsord_onramp_network | length > 0 + - sponsord_chain_id | default('') | string | length > 0 + - name: Report a contract address overridden in inventory ansible.builtin.debug: msg: >- @@ -242,7 +251,9 @@ that: # A DNS name: labels of letters, digits and inner hyphens, at least one dot. - sponsord_onramp_domain is match('^([A-Za-z0-9]([A-Za-z0-9-]{0,61}[A-Za-z0-9])?\.)+[A-Za-z]{2,63}$') - - sponsord_onramp_acme_email == '' or sponsord_onramp_acme_email is match('^[^@\s]+@[^@\s]+$') + - >- + sponsord_onramp_acme_email == '' + or sponsord_onramp_acme_email is match('^[A-Za-z0-9._%+-]+@([A-Za-z0-9-]+\.)+[A-Za-z]{2,63}$') fail_msg: >- sponsord_onramp_proxy "caddy" needs sponsord_onramp_domain, the DNS name users reach (got "{{ sponsord_onramp_domain }}"), and sponsord_onramp_acme_email must be @@ -361,8 +372,9 @@ fail_msg: >- {{ sponsord_onramp_api_token_file }} is {{ 'missing' if not sponsord_onramp_secret_stat.results[0].stat.exists else 'not a regular file' }}. - The sponsord role creates it; run the sponsord play on this host first - (`make deploy-sponsord` runs both). + The onramp runs beside the sponsord daemon and uses its token, which the sponsord + role creates: deploy sponsord on this host first (in this repo, list the host in + sponsord_hosts too; `make deploy-sponsord` runs both). when: not (ansible_check_mode and not sponsord_onramp_secret_stat.results[0].stat.exists) - name: Require the Turnstile secret as a regular file @@ -399,46 +411,103 @@ register: sponsord_onramp_turnstile_grep changed_when: false check_mode: false - failed_when: false + # 1 = no non-blank character (reported below); 2 = grep could not read the file. + failed_when: sponsord_onramp_turnstile_grep.rc not in [0, 1] - name: Report an empty Turnstile secret ansible.builtin.fail: msg: >- {{ sponsord_onramp_turnstile_secret_file }} is empty. Write the widget's secret - key into it (see the previous task's message for the recipe). - when: sponsord_onramp_turnstile_grep.rc != 0 + key into it, as root: umask 077; printf '%s' '' | sudo tee + {{ sponsord_onramp_turnstile_secret_file }} >/dev/null + when: sponsord_onramp_turnstile_grep.rc == 1 - name: Read whether Caddy is already installed ansible.builtin.package_facts: when: sponsord_onramp_proxy == 'caddy' # Before any mutation: a Caddyfile someone else wrote is never replaced silently. +# It is judged by the file, not by whether the package is installed: a Caddy run +# from a static binary leaves no dpkg trace, and apt keeps an existing conffile. +- name: Check the Caddyfile + ansible.builtin.stat: + path: "{{ _sponsord_onramp_caddyfile }}" + follow: false + get_checksum: true + checksum_algorithm: md5 + register: sponsord_onramp_caddyfile_stat + when: sponsord_onramp_proxy == 'caddy' + - name: Check the Caddyfile for this role's marker ansible.builtin.command: - argv: [grep, -qF, "MANAGED BY the sponsord_onramp role", "{{ sponsord_onramp_caddyfile }}"] + argv: [grep, -qF, "MANAGED BY the sponsord_onramp role", "{{ _sponsord_onramp_caddyfile }}"] register: sponsord_onramp_caddyfile_marker changed_when: false check_mode: false - failed_when: false + # 1 = no marker; 2 = unreadable, which must not pass for "no marker". + failed_when: sponsord_onramp_caddyfile_marker.rc not in [0, 1] + when: + - sponsord_onramp_proxy == 'caddy' + - sponsord_onramp_caddyfile_stat.stat.isreg | default(false) + +# The package's own default Caddyfile (a first run that installed Caddy and then +# stopped before templating leaves exactly that) is safe to replace. dpkg records +# its md5 among the package's conffiles; empty when the package is not installed. +- name: Read the packaged Caddyfile's checksum + ansible.builtin.shell: + executable: /bin/bash + cmd: | + set -o pipefail + { dpkg-query -W -f='${Conffiles}\n' caddy 2>/dev/null || true; } \ + | awk -v f={{ _sponsord_onramp_caddyfile | quote }} '$1 == f { print $2 }' + register: sponsord_onramp_caddyfile_pristine + changed_when: false + check_mode: false when: - sponsord_onramp_proxy == 'caddy' - - "'caddy' in ansible_facts.packages" + - sponsord_onramp_caddyfile_marker.rc | default(0) == 1 - name: Refuse to replace a Caddyfile this role did not write ansible.builtin.assert: that: - - sponsord_onramp_caddy_overwrite_config + - sponsord_onramp_caddyfile_stat.stat.isreg + - >- + sponsord_onramp_caddy_overwrite_config + or sponsord_onramp_caddyfile_marker.rc | default(0) == 0 + or (sponsord_onramp_caddyfile_pristine.stdout | default('') | trim | length > 0 + and sponsord_onramp_caddyfile_pristine.stdout | trim == sponsord_onramp_caddyfile_stat.stat.checksum) fail_msg: >- - Caddy was installed on this host before this role and - {{ sponsord_onramp_caddyfile }} is not the role's (no "MANAGED BY the - sponsord_onramp role" line). The role owns that whole file, so it would discard - whatever else this Caddy serves. Move the other sites elsewhere (or onto your own - proxy, with sponsord_onramp_proxy "none"), or set + {{ _sponsord_onramp_caddyfile }} exists and is not the role's (no "MANAGED BY the + sponsord_onramp role" line, and not the caddy package's untouched default), or + is not a regular file. The role owns that whole file, so it would discard + whatever else this Caddy serves. Move the other sites elsewhere (or onto your + own proxy, with sponsord_onramp_proxy "none"), or set sponsord_onramp_caddy_overwrite_config: true to let the role replace it. when: - sponsord_onramp_proxy == 'caddy' - - "'caddy' in ansible_facts.packages" - - sponsord_onramp_caddyfile_marker.rc != 0 + - sponsord_onramp_caddyfile_stat.stat.exists + +# Switching to "none" closes the firewall holes but leaves the role's Caddy running +# on 80/443 (your own proxy could not bind them). The role removes nothing. +- name: Check for a Caddyfile this role wrote while proxy was "caddy" + ansible.builtin.command: + argv: [grep, -qF, "MANAGED BY the sponsord_onramp role", "{{ _sponsord_onramp_caddyfile }}"] + register: sponsord_onramp_stale_caddy + changed_when: false + check_mode: false + failed_when: false + when: sponsord_onramp_proxy == 'none' + +- name: Warn that the role's Caddy is still installed + ansible.builtin.debug: + msg: >- + WARNING: sponsord_onramp_proxy is "none", but {{ _sponsord_onramp_caddyfile }} is + this role's, so its Caddy may still be running and holding tcp/80 and tcp/443 + (the firewall no longer opens them). Remove it: sudo systemctl disable --now + caddy && sudo apt-get remove caddy. + when: + - sponsord_onramp_proxy == 'none' + - sponsord_onramp_stale_caddy.rc | default(1) == 0 # --- Install, configure, start ------------------------------------------------------ - name: Install the sponsord-onramp binary diff --git a/ansible/roles/sponsord_onramp/templates/Caddyfile.j2 b/ansible/roles/sponsord_onramp/templates/Caddyfile.j2 index ebae5ded..63652361 100644 --- a/ansible/roles/sponsord_onramp/templates/Caddyfile.j2 +++ b/ansible/roles/sponsord_onramp/templates/Caddyfile.j2 @@ -5,9 +5,15 @@ # with no trusted_proxies, replaces any client-sent X-Forwarded-For with the real # client address, which the onramp rate limits by. { - # No admin API: nothing on this host may reconfigure the proxy over loopback. + # No admin API: nothing on this host can reconfigure the running proxy over + # the admin endpoint (:2019). # Config changes restart Caddy instead of reloading it. admin off + # HTTP/1.1 and HTTP/2 only: HTTP/3 would need udp/443, which the firewall + # does not open. + servers { + protocols h1 h2 + } {% if sponsord_onramp_acme_email | length > 0 %} email {{ sponsord_onramp_acme_email }} {% endif %} diff --git a/ansible/roles/sponsord_onramp/templates/sponsord-onramp.env.j2 b/ansible/roles/sponsord_onramp/templates/sponsord-onramp.env.j2 index db4803eb..5904d9bf 100644 --- a/ansible/roles/sponsord_onramp/templates/sponsord-onramp.env.j2 +++ b/ansible/roles/sponsord_onramp/templates/sponsord-onramp.env.j2 @@ -1,7 +1,7 @@ # MANAGED BY the sponsord_onramp role — do not edit by hand. NO secrets: the # daemon's API token and the Turnstile secret reach the onramp as systemd -# credentials (see sponsord-onramp.service). Every value here is public: the RPC -# URL and the contract addresses are served to every user. +# credentials (see sponsord-onramp.service). Every value here is non-secret; the +# RPC URL and the contract addresses are even served to every user. ONRAMP_BIND={{ sponsord_onramp_bind_address }}:{{ sponsord_onramp_port }} ONRAMP_DAEMON_URL={{ sponsord_onramp_daemon_url }} ONRAMP_PUBLIC_URL={{ sponsord_onramp_public_url }} @@ -14,7 +14,9 @@ ONRAMP_DECDN_SUMS_SHA256={{ sponsord_onramp_decdn_sums_sha256 }} ONRAMP_CLI_RELEASE={{ sponsord_onramp_cli_release }} ONRAMP_CLI_SUMS_SHA256={{ sponsord_onramp_cli_sums_sha256 }} {% if sponsord_onramp_brand_name | length > 0 %} -{# to_json: the double-quoted, C-escaped form systemd's EnvironmentFile parser reads. #} +{# to_json: double-quoted with \" and \\ escaped, which systemd's EnvironmentFile + parser unescapes (preflight refuses control characters, so no other escape is + emitted). #} ONRAMP_BRAND_NAME={{ sponsord_onramp_brand_name | string | to_json(ensure_ascii=false) }} {% endif %} {% if sponsord_onramp_gate_template_src | length > 0 %} diff --git a/ansible/roles/sponsord_onramp/templates/sponsord-onramp.service.j2 b/ansible/roles/sponsord_onramp/templates/sponsord-onramp.service.j2 index 2e72fa60..4b37832d 100644 --- a/ansible/roles/sponsord_onramp/templates/sponsord-onramp.service.j2 +++ b/ansible/roles/sponsord_onramp/templates/sponsord-onramp.service.j2 @@ -35,7 +35,7 @@ Restart=on-failure RestartSec=5 # --- Hardening ------------------------------------------------------------- -# Stateless: no StateDirectory, nothing on disk is writable. +# Stateless: no StateDirectory; nothing on disk is writable beyond its private /tmp. DynamicUser=true NoNewPrivileges=true ProtectSystem=strict diff --git a/ansible/roles/sponsord_onramp/vars/main/caddy.yml b/ansible/roles/sponsord_onramp/vars/main/caddy.yml new file mode 100644 index 00000000..ed3e16e1 --- /dev/null +++ b/ansible/roles/sponsord_onramp/vars/main/caddy.yml @@ -0,0 +1,4 @@ +--- +# The Caddyfile the distribution's caddy.service loads (its ExecStart names this +# path), so it is a fact of the package, not a knob. +_sponsord_onramp_caddyfile: /etc/caddy/Caddyfile diff --git a/ansible/tests/alloy-config/validate.sh b/ansible/tests/alloy-config/validate.sh index 55bce9e3..1ae823be 100755 --- a/ansible/tests/alloy-config/validate.sh +++ b/ansible/tests/alloy-config/validate.sh @@ -316,11 +316,11 @@ samples_json() { # nanosecond timestamp ["$1","s-plain an unlevelled line"], ["$1","Error: s-rpcurl read PaymentPool.usdc(): error sending request for url (https://arb-sepolia.example-provider.io/v2/SECRET-KEY?x=SECRET-Q)"]]}, {"stream":{"journal__systemd_unit":"sponsord-onramp.service","journal_priority_keyword":"info"},"values":[ - ["$1","2026-10-05T07:59:11.000001Z DEBUG sponsord_onramp::http: r-debug"], - ["$1","2026-10-05T07:59:11.000002Z INFO sponsord_onramp: r-info"], - ["$1","2026-10-05T07:59:11.000003Z WARN sponsord_onramp::gate::turnstile: r-warn"], - ["$1","Error: r-raw the daemon at http://127.0.0.1:8090 is unreachable"], - ["$1","r-plain an unlevelled line"]]}, + ["$1","2026-10-05T07:59:11.000001Z DEBUG sponsord_onramp::http: ro-debug"], + ["$1","2026-10-05T07:59:11.000002Z INFO sponsord_onramp: ro-info"], + ["$1","2026-10-05T07:59:11.000003Z WARN sponsord_onramp::gate::turnstile: ro-warn"], + ["$1","Error: ro-raw the daemon at http://127.0.0.1:8090 is unreachable"], + ["$1","ro-plain an unlevelled line"]]}, {"stream":{"journal__systemd_unit":"other.service","journal_priority_keyword":"info"},"values":[ ["$1","{\"level\":\"ERROR\",\"fields\":{\"message\":\"m-other-info\"}}"]]}, {"stream":{"journal__systemd_unit":"other.service","journal_priority_keyword":"error"},"values":[ @@ -463,8 +463,8 @@ kept m-weird info decdn-node.service # a non-string level is not a label valu kept m-text info decdn-node.service # not JSON: journald's stands kept m-other-info info other.service # another unit's JSON is never re-levelled kept m-other-error error other.service -kept r-warn info sponsord-onramp.service # onramp not enabled: journald's level -kept r-debug info sponsord-onramp.service +kept ro-warn info sponsord-onramp.service # onramp not enabled: journald's level +kept ro-debug info sponsord-onramp.service dropped m-trace # TRACE is not a journald keyword, so listed dropped m-debug # padded + lower-cased before the drop dropped m-other-debug # the journald-priority drop, other units @@ -523,7 +523,7 @@ kept m-nolevel info decdn-node.service kept m-weird info decdn-node.service kept m-text info decdn-node.service kept m-other-error error other.service -dropped r-warn # onramp not enabled: judged at journald's info +dropped ro-warn # onramp not enabled: judged at journald's info dropped m-info dropped m-trace dropped m-debug @@ -544,7 +544,7 @@ kept s-plain info sponsord.service # any other unlevelled line: journald's dropped s-trace dropped s-debug kept m-error error decdn-node.service # the node's JSON stage is unaffected -kept r-warn info sponsord-onramp.service # the onramp is not enabled here +kept ro-warn info sponsord-onramp.service # the onramp is not enabled here echo "ok: sponsord log level follows its text line (defaults)" # Co-located, guardrail "info|debug|trace": both daemons exempt from the @@ -563,20 +563,20 @@ dropped s-info dropped s-trace dropped m-info dropped m-other-info -dropped r-warn # the onramp is not enabled: not exempt -dropped r-raw +dropped ro-warn # the onramp is not enabled: not exempt +dropped ro-raw echo "ok: both daemons judged by their own level (info in the guardrail)" # The onramp too: exempt, and levelled from its plain-text line like sponsord. run_level_harness colocatedonramp.alloy 32 no_secrets -kept r-warn warning sponsord-onramp.service -kept r-raw error sponsord-onramp.service # its exit error, with the daemon URL redacted -kept r-plain info sponsord-onramp.service # exempt: no level of its own to judge by +kept ro-warn warning sponsord-onramp.service +kept ro-raw error sponsord-onramp.service # its exit error: an error +kept ro-plain info sponsord-onramp.service # exempt: no level of its own to judge by kept s-warn warning sponsord.service # sponsord's stage is unaffected kept m-error error decdn-node.service -dropped r-info -dropped r-debug +dropped ro-info +dropped ro-debug echo "ok: the onramp judged by its own level (info in the guardrail)" # --- Gate 2: every ExecStart flag exists ------------------------------------- diff --git a/ansible/tests/firewall-holes/check.sh b/ansible/tests/firewall-holes/check.sh index 8f760ccd..2928421d 100755 --- a/ansible/tests/firewall-holes/check.sh +++ b/ansible/tests/firewall-holes/check.sh @@ -10,14 +10,21 @@ set -euo pipefail here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" playbooks="$here/../../playbooks" -got="$( - # From this directory, so ansible/ansible.cfg (inventory, become) is not read. +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT + +# Each host writes its resolved list to .json with the core copy module (the +# json stdout callback lives in ansible.posix, which a core-only install lacks). +# From this directory, so ansible/ansible.cfg (inventory, become) is not read. +( cd "$here" - ANSIBLE_LOAD_CALLBACK_PLUGINS=1 ANSIBLE_STDOUT_CALLBACK=json ANSIBLE_DEPRECATION_WARNINGS=0 \ - ansible all -i "$here/inventory.yml" --playbook-dir "$playbooks" \ - -m ansible.builtin.debug -a var=baseline_extra_inbound "$work/ansible.log" 2>&1 +) || { cat "$work/ansible.log" >&2; echo "ansible could not resolve the inventory" >&2; exit 1; } +got="$(cd "$work" && jq -S -n '[inputs | {key: (input_filename | rtrimstr(".json")), value: .}] | from_entries' ./*.json \ + | jq -S 'with_entries(.key |= ltrimstr("./"))')" want="$(jq -S . "$here/expected.json")" if [[ "$got" != "$want" ]]; then @@ -25,4 +32,11 @@ if [[ "$got" != "$want" ]]; then diff <(echo "$want") <(echo "$got") >&2 || true exit 1 fi +# all.yml falls back to two role defaults outside the roles' plays; they must agree. +roles="$here/../../roles" +grep -qE '^decdn_bind_port: 4433([[:space:]]|$)' "$roles/decdn_node/defaults/main.yml" \ + || { echo "decdn_bind_port's default is not 4433: update the fallback in playbooks/group_vars/all.yml" >&2; exit 1; } +grep -qE '^sponsord_onramp_proxy: caddy([[:space:]]|$)' "$roles/sponsord_onramp/defaults/main.yml" \ + || { echo "sponsord_onramp_proxy's default is not caddy: update the fallback in playbooks/group_vars/all.yml" >&2; exit 1; } + echo "baseline_extra_inbound matches for $(jq -r 'keys | join(", ")' <<<"$want")" diff --git a/docs/requirements.md b/docs/requirements.md index 6a7ac82e..d9ec9e23 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -44,9 +44,10 @@ On Ubuntu 25.10 and later, `sudo` is sudo-rs; see the `ansible_become_exe` note provider's security group or firewall, which the host cannot see. - **A sponsord-onramp host (Ansible only) also needs inbound tcp/80 and tcp/443** for Caddy, from anywhere: 443 serves the onramp, 80 answers the ACME challenge and - redirects to https. `baseline` opens both for hosts in `sponsord_onramp_hosts`; open - them in the provider's firewall too, and point the domain's A/AAAA records at the - host. + redirects to https. `baseline` opens both for hosts in `sponsord_onramp_hosts` while + `sponsord_onramp_proxy` is `caddy` (the default; `none` opens neither); open them in + the provider's firewall too, and point the domain's A/AAAA records at the host. + Caddy comes from the distribution: on Ubuntu, enable the **universe** component. - **Outbound:** HTTPS to your RPC provider and to any cache origin; QUIC/UDP to peers; HTTPS to the iroh relays (upstream's defaults unless you set your own). - **NAT:** a node without direct inbound reachability still works through iroh relays, From 9438ced5a5c337d0e8a048912060a304e67f4465 Mon Sep 17 00:00:00 2001 From: Ant Somers Date: Mon, 5 Oct 2026 17:04:40 +0300 Subject: [PATCH 4/4] fix(sponsord_onramp): keep stock Caddy down, require sponsord_hosts in the playbook From the PR review: - Install Caddy with policy_rc_d 101, so the package's maintainer scripts never start it on its stock config while 80/443 are already open. A run that failed after the install could otherwise leave the default server public. Caddy first starts from the restart handler, after the managed Caddyfile has validated. - playbooks/sponsord.yml gains a fact-free first play that refuses an onramp host outside sponsord_hosts: such a host would skip baseline and get no Caddy firewall holes. The role still checks no group name, so collection users keep their own groups. tests/scripts-test.sh lifts that play out with yq and runs it against a fixture, both ways. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 4 ++- ansible/playbooks/sponsord.yml | 21 +++++++++++++-- ansible/roles/sponsord_onramp/README.md | 9 ++++--- ansible/roles/sponsord_onramp/tasks/caddy.yml | 7 ++++- ansible/tests/playbook-guards/inventory.yml | 14 ++++++++++ tests/scripts-test.sh | 26 +++++++++++++++++-- 6 files changed, 72 insertions(+), 9 deletions(-) create mode 100644 ansible/tests/playbook-guards/inventory.yml diff --git a/AGENTS.md b/AGENTS.md index ba4172b0..66490fa7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -166,7 +166,9 @@ addresses) the repo carries, and they carry their upstream commit. Turnstile gate, the installers, the `decdn-sponsored` CLI API), as a second play of `playbooks/sponsord.yml` on `sponsord_onramp_hosts` (which must also be in `sponsord_hosts`: it reads the daemon's `/etc/sponsord/api-token` and calls it on - loopback; the token gate and `/healthz` fail without it, no group name is checked). + loopback). The playbook's first play enforces that membership (tested by + `make test-scripts`); the role checks no group name, so collection users keep their + own groups, and its token gate and `/healthz` fail without a daemon. - **Install / unit / gate:** the sponsord role's patterns, copied: manual or GPG-verified `sponsord-onramp-v*` release (the KEYS are the sponsord role's file, via `role_path`), `DynamicUser` with the token and the operator-provisioned diff --git a/ansible/playbooks/sponsord.yml b/ansible/playbooks/sponsord.yml index 9d0af087..a93c3b59 100644 --- a/ansible/playbooks/sponsord.yml +++ b/ansible/playbooks/sponsord.yml @@ -13,6 +13,24 @@ # Opening and funding the treasury's pool is an operator action — see # roles/sponsord/README.md; the Turnstile widget and the release pins are too — # see roles/sponsord_onramp/README.md. +# Before anything runs: an onramp host outside sponsord_hosts would skip the play +# below (baseline hardening included) and get no firewall holes for Caddy, which +# playbooks/group_vars/ derive from sponsord_hosts. The role itself checks no group +# name, so the decdn.node collection does not impose this repo's groups on others. +# tests/scripts-test.sh runs this play by its name. +- name: Require every sponsord-onramp host to be a sponsord host + hosts: sponsord_onramp_hosts + gather_facts: false + tasks: + - name: Require the host in sponsord_hosts too + ansible.builtin.assert: + that: + - "'sponsord_hosts' in group_names" + fail_msg: >- + {{ inventory_hostname }} is in sponsord_onramp_hosts but not in sponsord_hosts. + The onramp runs beside the sponsord daemon and needs this playbook's baseline + and firewall holes, which come with sponsord_hosts: list the host there too. + - name: Provision sponsord hosts: sponsord_hosts become: true @@ -26,8 +44,7 @@ # After the daemon's play, whose /healthz gate is fatal, so the onramp starts # against a daemon that is up. Baseline and Alloy already ran for these hosts -# above for hosts that are also in sponsord_hosts, which every one must be: the -# onramp uses the daemon on its own host. +# above: the first play holds every one of them to sponsord_hosts. - name: Provision sponsord-onramp hosts: sponsord_onramp_hosts become: true diff --git a/ansible/roles/sponsord_onramp/README.md b/ansible/roles/sponsord_onramp/README.md index f113e0bc..5d90401b 100644 --- a/ansible/roles/sponsord_onramp/README.md +++ b/ansible/roles/sponsord_onramp/README.md @@ -12,9 +12,10 @@ the public side of the deCDN onboarding sponsor. It serves: It holds no wallet. For each person who passes the gate it asks the [`sponsord`](../sponsord/README.md) daemon on the same host for a capability, using the daemon's API token. So the daemon must run on the same host: in this repo, list -the host in `sponsord_hosts` as well as `sponsord_onramp_hosts`. The role fails -early when the daemon's token file is missing, and its `/healthz` gate fails when -the daemon is unreachable. +the host in `sponsord_hosts` as well as `sponsord_onramp_hosts` (`playbooks/sponsord.yml` +refuses it otherwise). The role itself checks no group name: it fails early when +the daemon's token file is missing, and its `/healthz` gate fails when the daemon +is unreachable. ## What it does @@ -52,6 +53,8 @@ the daemon is unreachable. - **Puts Caddy in front** (`sponsord_onramp_proxy: caddy`, the default): - Installs the distribution's `caddy` package (Debian main; on Ubuntu it is in **universe**, which must be enabled). No third-party apt repository is added. + The package is not allowed to start Caddy on its stock config: Caddy first + starts once the role's Caddyfile has validated. - Writes `/etc/caddy/Caddyfile`: TLS for `sponsord_onramp_domain`, then a reverse proxy to the onramp on loopback. The file is validated with `caddy validate` before it lands. The admin API is off, so nothing on the host can reconfigure diff --git a/ansible/roles/sponsord_onramp/tasks/caddy.yml b/ansible/roles/sponsord_onramp/tasks/caddy.yml index 79b7cfea..e09f3109 100644 --- a/ansible/roles/sponsord_onramp/tasks/caddy.yml +++ b/ansible/roles/sponsord_onramp/tasks/caddy.yml @@ -7,13 +7,18 @@ # capability, PrivateTmp, ProtectSystem=full); main.yml has already refused a # Caddyfile this role does not own. -# curl: the probe at the end of this file. +# curl: the probe at the end of this file. policy_rc_d 101 stops the package's +# maintainer scripts from starting Caddy on its stock config: the firewall already +# has 80/443 open, so a run that failed after the install would leave that default +# server public. Caddy first starts from the Restart caddy handler, once the +# managed Caddyfile below has validated. - name: Install Caddy from the distribution ansible.builtin.apt: name: [caddy, curl] state: present update_cache: true cache_valid_time: 3600 + policy_rc_d: 101 # `caddy validate` loads the config the way `caddy run` does (it also provisions # the TLS automation), so a Caddyfile that does not parse or provision never lands. diff --git a/ansible/tests/playbook-guards/inventory.yml b/ansible/tests/playbook-guards/inventory.yml new file mode 100644 index 00000000..fbec66cc --- /dev/null +++ b/ansible/tests/playbook-guards/inventory.yml @@ -0,0 +1,14 @@ +--- +# Fixture for tests/scripts-test.sh's playbook-guard cases. Nothing here is ever +# contacted: the cases run only the guard play, which gathers no facts. +all: + vars: + ansible_connection: local + children: + sponsord_hosts: + hosts: + both: {} + sponsord_onramp_hosts: + hosts: + both: {} + onramp-only: {} diff --git a/tests/scripts-test.sh b/tests/scripts-test.sh index 1a966f7d..f5ec4c0f 100755 --- a/tests/scripts-test.sh +++ b/tests/scripts-test.sh @@ -163,14 +163,36 @@ else skipped+=("lint-cloud-init negatives (needs cloud-init on PATH; CI installs it)") fi -# --- baseline firewall holes per host shape (playbooks/group_vars) ------------------ +# --- baseline firewall holes per host shape, and playbook guards --------------------- if command -v ansible >/dev/null; then expect 0 "firewall holes resolve per host shape, co-located included" \ "$repo/ansible/tests/firewall-holes/check.sh" + # playbooks/sponsord.yml refuses an onramp host outside sponsord_hosts. Its guard + # play is lifted out with yq and run alone (running the whole playbook would need + # the roles' collections just to parse), from a directory without ansible.cfg, + # against a fixture nothing connects to. + if command -v yq >/dev/null; then + yq '[.[] | select(.name == "Require every sponsord-onramp host to be a sponsord host")]' \ + "$repo/ansible/playbooks/sponsord.yml" > "$work/guard.yml" + [[ "$(yq 'length' "$work/guard.yml")" == 1 ]] \ + || fail "playbooks/sponsord.yml has no single sponsord-onramp placement play" + guard() { # + (cd "$repo/ansible/tests/playbook-guards" && ANSIBLE_DEPRECATION_WARNINGS=0 \ + ansible-playbook -i inventory.yml "$work/guard.yml" --limit "$1" &2; fail "the onramp-only refusal did not come from the placement check"; } + elif [[ -n ${CI:-} ]]; then + fail "yq is not on PATH in CI; the playbook-guard cases would be skipped" + else + skipped+=("playbook guards (needs yq)") + fi elif [[ -n ${CI:-} ]]; then fail "ansible is not on PATH in CI; the firewall-holes check would be skipped" else - skipped+=("firewall holes (needs ansible-core on PATH)") + skipped+=("firewall holes and playbook guards (need ansible-core on PATH)") fi # --- upstream-mirror generators (optional: needs a decdn/decdn checkout) --------------