diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 42feb08..76599fb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -381,6 +381,9 @@ jobs: # Without cloud-init on PATH the harness SKIPS the lint-cloud-init cases. - name: Install cloud-init run: command -v cloud-init || { sudo apt-get update -qq && sudo apt-get install -y -qq cloud-init; } + # Without ansible-core on PATH the harness FAILS the firewall-holes case in CI. + - name: Install ansible-core + run: command -v ansible || pipx install ansible-core - name: Script and guard tests run: make test-scripts diff --git a/.github/workflows/molecule.yml b/.github/workflows/molecule.yml index dcd8b89..233e0db 100644 --- a/.github/workflows/molecule.yml +++ b/.github/workflows/molecule.yml @@ -27,7 +27,8 @@ jobs: runs-on: ubuntu-latest # Docker is preinstalled # ~4m41s at JOBS=3 with 8 scenarios. os-matrix (3 containers) and lifecycle took the # suite to ~10m at JOBS=3 on a local 16-thread box; re-measure from CI's first run. - # sponsord added a 12th scenario (2 containers, Debian 12 + Ubuntu 24.04). + # sponsord added a 12th scenario (2 containers, Debian 12 + Ubuntu 24.04), and + # sponsord-onramp a 13th (the same 2, plus an apt install of Caddy each). # A bound, not a target: without one a wedged privileged systemd container burns # the 360-minute default, and with cancel-in-progress above, some branch-protection # setups read the resulting cancelled check as "not failed" rather than as a failure. diff --git a/.github/workflows/upstream-drift.yml b/.github/workflows/upstream-drift.yml index aa249bb..2bdd886 100644 --- a/.github/workflows/upstream-drift.yml +++ b/.github/workflows/upstream-drift.yml @@ -3,8 +3,9 @@ name: Upstream drift # Three things in this repo mirror decdn/decdn and go stale when upstream moves: # - roles/decdn_node/vars/main/networks.yml (contract addresses; upstream -# redeploys change every one of them at once) and its subset -# roles/sponsord/vars/main/networks.yml (the same script and check) +# redeploys change every one of them at once) and its subsets +# roles/sponsord/vars/main/networks.yml and +# roles/sponsord_onramp/vars/main/networks.yml (the same script and check) # - charts/decdn-node/files/monitoring/ (dashboards + alert rules) # - ansible/molecule/schema/files/schema-keys.txt (the node.toml key inventory) # This job regenerates each from upstream main and fails when a committed copy diff --git a/AGENTS.md b/AGENTS.md index 3c19e15..66490fa 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -31,10 +31,13 @@ mirror (`vars/main/networks.yml`), never hand-copied into inventory or docs. overrides `node.toml`), keeps the keystore password off the PVC, and refuses secret-bearing keys in `config`. 2. **Localhost-only by default.** Service daemons bind `127.0.0.1` (e.g. the node's metrics - and admin RPC). A service that must accept public traffic declares exactly one hole (the - node's QUIC udp/4433) via `baseline_extra_inbound`; if a service ever needs an HTTP-facing - public path, front it with an explicit reverse proxy that terminates auth + TLS. Never - bind a *backend* to `0.0.0.0` or expose its raw port. + and admin RPC). A service that must accept public traffic declares its holes explicitly + via `baseline_extra_inbound` (today: the node's QUIC udp/4433, and tcp/80 + tcp/443 for + the `sponsord_onramp` role's Caddy). An HTTP-facing service is fronted by an explicit + reverse proxy that terminates TLS (and auth, where the service needs it), and the + backend stays on loopback. Never bind a *backend* to `0.0.0.0` or expose its raw port. + `sponsord-onramp` is public by design: its Turnstile gate guards the one action that + spends (`POST /v1/fund`), and per-address rate limits cover the rest. **Kubernetes exception (chart only):** the node's metrics bind `0.0.0.0` inside the pod so kubelet probes and Prometheus can reach them. That is allowed only behind a ClusterIP-only Service and the chart's NetworkPolicy (metrics ingress limited to @@ -54,8 +57,8 @@ mirror (`vars/main/networks.yml`), never hand-copied into inventory or docs. ``` ansible/ # the deployment project (DevSec-hardened, lean roles) - playbooks/ # site.yml (decdn node + sponsord), sponsord.yml, backup.yml, decommission.yml - roles/ # baseline, decdn_node, grafana_alloy, sponsord + playbooks/ # site.yml (decdn node + sponsord), sponsord.yml (+ onramp), backup.yml, decommission.yml + roles/ # baseline, decdn_node, grafana_alloy, sponsord, sponsord_onramp inventory/ galaxy/ molecule/ # see ansible/README.md cloud-init/ # user-data.yaml + on-host bootstrap.sh; pinned ansible-core/collections (see its README.md) compose/ # Docker Compose deploy path for a single host (see its README.md) @@ -69,8 +72,9 @@ scripts/ # upstream-mirror generators + the release gate ``` **Generated mirrors of upstream — regenerate, never hand-edit:** -`ansible/roles/decdn_node/vars/main/networks.yml` and its subset -`ansible/roles/sponsord/vars/main/networks.yml` (both `scripts/sync-network-profiles.py`), +`ansible/roles/decdn_node/vars/main/networks.yml` and its subsets +`ansible/roles/sponsord/vars/main/networks.yml` and +`ansible/roles/sponsord_onramp/vars/main/networks.yml` (all `scripts/sync-network-profiles.py`), `charts/decdn-node/files/monitoring/` (`scripts/sync-monitoring.sh`) and `ansible/molecule/schema/files/schema-keys.txt` (`gen-schema-keys.py`). The weekly `upstream-drift` workflow flags staleness. These are the only protocol facts (contract @@ -157,7 +161,29 @@ addresses) the repo carries, and they carry their upstream commit. - **Molecule:** under docker, `/` must be `--make-rshared` (see `molecule/sponsord/prepare.yml`) or every credential directory is empty. - **Not in cloud-init:** its lint only allows `decdn_nodes`. - - The public `sponsord-onramp` is not deployed yet. + +- **`ansible/roles/sponsord_onramp`** — `sponsord-onramp`, sponsord's public side (the + Turnstile gate, the installers, the `decdn-sponsored` CLI API), as a second play of + `playbooks/sponsord.yml` on `sponsord_onramp_hosts` (which must also be in + `sponsord_hosts`: it reads the daemon's `/etc/sponsord/api-token` and calls it on + loopback). The playbook's first play enforces that membership (tested by + `make test-scripts`); the role checks no group name, so collection users keep their + own groups, and its token gate and `/healthz` fail without a daemon. + - **Install / unit / gate:** the sponsord role's patterns, copied: manual or + GPG-verified `sponsord-onramp-v*` release (the KEYS are the sponsord role's file, + via `role_path`), `DynamicUser` with the token and the operator-provisioned + Turnstile secret as `LoadCredential=`, `PartOf=sponsord.service`, a restart-inputs + record, and a fatal `/healthz` gate on a loopback listener. + - **Public inputs:** `sponsord_onramp_rpc_url` is served to every user, so preflight + refuses userinfo, a query and the characters upstream's installers cannot quote. + CapacityBond/SlashJudge come from the generated `vars/main/networks.yml`. + - **Caddy (default) or none:** the distro package, a role-owned `/etc/caddy/Caddyfile` + (marker-gated: a foreign one needs `sponsord_onramp_caddy_overwrite_config`), + `admin off`, so the handler restarts. tcp/80 + tcp/443 come from + `playbooks/group_vars/all.yml`, which reads `sponsord_onramp_proxy` from inventory. + - **Molecule:** `molecule/sponsord-onramp` runs the real Caddy with `tls internal` + and checks the https chain and that a forged `X-Forwarded-For` cannot pick the + client address. - **`cloud-init/`** — the Ansible path with no control machine. `user-data.yaml` carries only public material (the lint refuses secret-looking keys, credentials in URLs and @@ -231,13 +257,18 @@ make build / galaxy-check # the decdn.node collection **Inventory is private; the firewall hole is not.** This repo is public, so `ansible/inventory/hosts.yml` is git-ignored and a real fleet lives in a private overlay. -Inventory-adjacent group_vars do not load for an overlay, so anything every node needs regardless of inventory (today the udp/4433 -`baseline_extra_inbound` hole) lives in `ansible/playbooks/group_vars/decdn_nodes.yml`, and -the Alloy per-daemon toggles every host needs in `ansible/playbooks/group_vars/all.yml`. -Don't move it back under `inventory/`. +Inventory-adjacent group_vars do not load for an overlay, so anything every host needs +regardless of inventory lives in `ansible/playbooks/group_vars/`. The public firewall holes +(`baseline_extra_inbound`, today the node's udp/4433, plus tcp/80 + tcp/443 on +`sponsord_onramp_hosts` with `sponsord_onramp_proxy: caddy`) are built from a host's groups in +`all.yml` (`_baseline_service_inbound`). `decdn_nodes.yml` and `sponsord_hosts.yml` both set +`baseline_extra_inbound` to that list, so a co-located host renders one firewall in every play. +`ansible/tests/firewall-holes/` (run by `make test-scripts`) pins the result per host shape. +The Alloy per-daemon toggles also live in `all.yml`. Don't move any of it back under +`inventory/`. -**Galaxy collection (`decdn.node`).** The four roles (`baseline` + `decdn_node` + -`grafana_alloy` + `sponsord`) ship as a +**Galaxy collection (`decdn.node`).** The five roles (`baseline` + `decdn_node` + +`grafana_alloy` + `sponsord` + `sponsord_onramp`) ship as a distributable collection. The overlay lives in `ansible/galaxy/` and is staged into a clean collection tree by `galaxy/build.sh` — there is **no** `galaxy.yml` at the `ansible/` root (that would make ansible-lint treat the deploy project as a collection). Build/validate with diff --git a/README.md b/README.md index bf28628..82a09b8 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,9 @@ This is the canonical statement; the per-path READMEs add only what is specific you choose. - **Localhost-only by default.** Backends bind `127.0.0.1`. A service that must accept public traffic declares its port explicitly, and the node declares exactly one: - udp/4433. On Kubernetes, metrics bind `0.0.0.0` in the pod only behind a ClusterIP + udp/4433. The sponsor's public onramp (Ansible only) stays on loopback too, behind + Caddy on tcp/80 + tcp/443 (the default; with `sponsord_onramp_proxy: none` you bring + the proxy and open its ports). On Kubernetes, metrics bind `0.0.0.0` in the pod only behind a ClusterIP Service and a NetworkPolicy. - **Default-deny inbound** (Ansible's `baseline`, nftables). SSH is the only universally open port; extra public ports are declared via `baseline_extra_inbound`. @@ -97,7 +99,7 @@ This is the canonical statement; the per-path READMEs add only what is specific | Path | What it is | |------|------------| -| [`ansible/`](ansible/README.md) | The Ansible project: `inventory/`, `playbooks/` (`site.yml`, `sponsord.yml`, `backup.yml`, `decommission.yml`), `roles/` (`baseline`, `decdn_node`, `grafana_alloy`, `sponsord`), `galaxy/` (the `decdn.node` collection), `molecule/`. | +| [`ansible/`](ansible/README.md) | The Ansible project: `inventory/`, `playbooks/` (`site.yml`, `sponsord.yml`, `backup.yml`, `decommission.yml`), `roles/` (`baseline`, `decdn_node`, `grafana_alloy`, `sponsord`, `sponsord_onramp`), `galaxy/` (the `decdn.node` collection), `molecule/`. | | [`cloud-init/`](cloud-init/README.md) | The cloud-init deploy path: `user-data.yaml`, the on-host `bootstrap.sh`, and the pinned ansible-core and collections it installs. | | [`compose/`](compose/README.md) | The Docker Compose deploy path. | | [`charts/decdn-node/`](charts/decdn-node/README.md) | The Helm chart, with vendored dashboards and alert rules in `files/monitoring/`. | diff --git a/ansible/Makefile b/ansible/Makefile index c2ef7ee..1e5ded8 100644 --- a/ansible/Makefile +++ b/ansible/Makefile @@ -121,8 +121,10 @@ decommission: # /metrics probe timeout), grafana-cloud and grafana-cloud-token (opt-in Grafana # Cloud observability, host-provisioned vs inventory token), os-matrix (default's # plays on Debian 13, Ubuntu 24.04 and 26.04), lifecycle (network profile, -# backup + decommission) and sponsord (the sponsord role on a host with no node, -# through playbooks/sponsord.yml, on Debian 12 and Ubuntu 24.04). The glob below is the authority; this list is a guide. +# backup + decommission), sponsord (the sponsord role on a host with no node, +# through playbooks/sponsord.yml, on Debian 12 and Ubuntu 24.04) and sponsord-onramp +# (the onramp beside its daemon, behind the real distro Caddy). The glob below is the +# authority; this list is a guide. # # The scenarios run concurrently: distinct container names, no published host ports, # per-scenario ephemeral dirs, and what they share on the control machine (the @@ -209,8 +211,8 @@ molecule-all: deps molecule test --all # --- Galaxy collection (decdn.node) ------------------------------------------ -# Stage baseline + decdn_node + grafana_alloy + sponsord into a clean collection tree and -# build the artifact under build/. Only the four roles ship; see galaxy/README.md. Publishing +# Stage baseline + decdn_node + grafana_alloy + sponsord + sponsord_onramp into a clean collection tree and +# build the artifact under build/. Only the five roles ship; see galaxy/README.md. Publishing # runs from a vX.Y.Z tag in .github/workflows/release.yml (see RELEASING.md). build: ./galaxy/build.sh diff --git a/ansible/README.md b/ansible/README.md index 9c4057b..870bdc6 100644 --- a/ansible/README.md +++ b/ansible/README.md @@ -8,7 +8,7 @@ hardened host baseline. The roles also ship as the `decdn.node` Galaxy collectio | **`site.yml`** | Harden the host and deploy the public **deCDN node** (`decdn-node`). | `make check` / `make deploy` | | `backup.yml` | Encrypted backup of a node's identity or full state. | `make backup` | | `decommission.yml` | Stop a node and remove its service (keeps the identity; no on-chain steps). | `make decommission` | -| `sponsord.yml` | Harden the host and deploy **sponsord**, the onboarding sponsor, on hosts in `sponsord_hosts`, with or without a node. Also run by `site.yml`. | `make check-sponsord` / `make deploy-sponsord` | +| `sponsord.yml` | Harden the host and deploy **sponsord**, the onboarding sponsor, on hosts in `sponsord_hosts`, with or without a node; then its public **sponsord-onramp** (behind Caddy) on hosts in `sponsord_onramp_hosts`. Also run by `site.yml`. | `make check-sponsord` / `make deploy-sponsord` | ``` baseline host hardening: DevSec os/ssh, nftables default-deny inbound, @@ -17,7 +17,9 @@ baseline host hardening: DevSec os/ssh, nftables default-deny inbound, ├─ decdn_node public QUIC udp/4433; metrics + admin loopback; signed-tarball or │ local-build install; hardened systemd unit; backup/decommission └─ sponsord treasury signer + PaymentPool keeper; loopback API; DynamicUser - unit with systemd credentials; standalone or beside a node + │ unit with systemd credentials; standalone or beside a node + └─ sponsord_onramp the public gate + installers on sponsord's host; loopback + listener behind Caddy (ACME TLS, tcp/80 + tcp/443) ``` ## Security model @@ -26,8 +28,9 @@ The repo-wide model is in the [root README](../README.md#security-model). What i specific to this path: - **nftables is the firewall.** SSH plus the node's **udp/4433** (via - `baseline_extra_inbound`) are the only holes; metrics 9090 and the admin RPC 9191 stay - loopback. Baseline turns off `os_hardening`'s ufw config template + `baseline_extra_inbound`) are the only holes, plus **tcp/80 + tcp/443** for Caddy on a + `sponsord_onramp_hosts` host; metrics 9090, the admin RPC 9191, sponsord 8090 and the + onramp 8080 stay loopback. Baseline turns off `os_hardening`'s ufw config template (`ufw_manage_defaults: false`) so a misleading DROP-policy `/etc/default/ufw` is never written. Do not install or enable ufw: it would replace the nftables ruleset and drop QUIC and SSH. @@ -107,10 +110,13 @@ the overlay carries its own: create them next to its `hosts.yml`, starting from `host_vars/*/secret.*` in the private repo, so it never tracks a per-node secret. The public `inventory/group_vars/` and `inventory/host_vars/` are **not** loaded for it. Add child groups of your own when hosts differ by group (an origin, an egress cap on metered bandwidth), each -with a `group_vars/.yml`. Settings every node needs regardless of -inventory, currently only the udp/4433 QUIC firewall hole, live in -`playbooks/group_vars/decdn_nodes.yml`, so an overlay can't drop them. Override that per node -in `host_vars` if you have to. Playbook group_vars beat inventory group_vars. +with a `group_vars/.yml`. Settings every host needs regardless of +inventory, currently the public firewall holes (the node's udp/4433, and tcp/80 + +tcp/443 for Caddy on `sponsord_onramp_hosts`), live in +`playbooks/group_vars/`, so an overlay can't drop them: `all.yml` builds the list from the +host's groups and `decdn_nodes.yml` / `sponsord_hosts.yml` apply it. Override +`baseline_extra_inbound` per host in `host_vars` if you have to. Playbook group_vars beat +inventory group_vars. By default baseline **deploys you as yourself**: the runner (your control-machine `$USER` + its autodetected `~/.ssh` key, `id_ed25519` > `ecdsa` > `rsa`) is prepended as the head of @@ -264,8 +270,19 @@ a dedicated host or a node host listed in both groups. Then run `sponsord_hosts` and the node on hosts in `decdn_nodes` (`playbooks/group_vars/all.yml`). -Setup and variables: [`roles/sponsord/README.md`](roles/sponsord/README.md). The public -`sponsord-onramp` is not deployed by this repo yet. +Setup and variables: [`roles/sponsord/README.md`](roles/sponsord/README.md). + +Its public side, `sponsord-onramp` (the Turnstile gate, the installers, the CLI API), +is the fifth role, `sponsord_onramp`. List the sponsord host in `sponsord_onramp_hosts` +as well; `make deploy-sponsord` runs it after the daemon. + +- **You provision** the Turnstile secret on the host. The onramp reads the daemon's own + API token. +- **Caddy** (the default) terminates TLS for `sponsord_onramp_domain` and opens tcp/80 + and tcp/443. Set `sponsord_onramp_proxy: none` to bring your own proxy. +- **The RPC URL it gets is public**: it is handed to every user, so it must hold no key. + +Setup and variables: [`roles/sponsord_onramp/README.md`](roles/sponsord_onramp/README.md). ### Backup and decommission @@ -298,15 +315,18 @@ make lint-alloy # render grafana_alloy's templates, then `alloy validate` th for every role, must be rejected by their own asserts), `generate-keystore` (opt-in host-side wallet), `host-env` (host-provisioned `/etc/decdn/decdn.env`), `slow-readiness` (advisory `/metrics` probe timeout), `grafana-cloud` (the opt-in -observability wiring — see [Grafana Cloud observability](#grafana-cloud-observability-opt-in)) +observability wiring — see [Grafana Cloud observability](#grafana-cloud-observability-opt-in)), `grafana-cloud-token` (the same wiring with the API token carried through git-ignored inventory instead: role-authored env file + provenance record), `os-matrix` (default's plays on Debian 13, Ubuntu 24.04 and Ubuntu 26.04; `default` itself is Debian 12), `lifecycle` (a `decdn_network` profile with an override, both -backup scopes decrypted and checked, a rejected and a real decommission) and +backup scopes decrypted and checked, a rejected and a real decommission), `sponsord` (`playbooks/sponsord.yml` on a host with no node: generated token, credential rotation restart, the fatal `/healthz` gate, Alloy scraping sponsord and not the node; -`grafana-cloud` covers sponsord co-located with a node). +`grafana-cloud` covers sponsord co-located with a node) and `sponsord-onramp` (the +onramp beside its daemon, behind the real distro Caddy with `tls internal`: the https +chain, the client address Caddy passes on, ACME and `none` modes, a Turnstile rotation +restart, the fatal gates, a refused foreign Caddyfile). They are independent, so they run concurrently, and each line of output is prefixed with its scenario name because the runs interleave. `make molecule-serial` is the escape hatch when that interleaving gets in the way of reading a failure. @@ -360,12 +380,12 @@ the RPC URL when it is not provisioned on the host instead). Highlights: ## Packaging as a Galaxy collection (`decdn.node`) -The four roles (`baseline`, `decdn_node`, `grafana_alloy`, `sponsord`) are also packaged as the +The five roles (`baseline`, `decdn_node`, `grafana_alloy`, `sponsord`, `sponsord_onramp`) are also packaged as the distributable **`decdn.node`** collection, for operators who bring their own playbooks. The collection overlay lives in [`galaxy/`](galaxy/) (`galaxy.yml`, the collection `README.md`/`CHANGELOG.md`, `meta/runtime.yml`, `build.sh`). It is deliberately **not** -a `galaxy.yml` at the project root: `galaxy/build.sh` stages only the four roles into a +a `galaxy.yml` at the project root: `galaxy/build.sh` stages only the five roles into a clean `ansible_collections/decdn/node/` tree and builds the artifact, so this project stays a plain Ansible project (the `make deploy`/`lint` flow is unchanged). diff --git a/ansible/galaxy/CHANGELOG.md b/ansible/galaxy/CHANGELOG.md index cc80314..59eaa17 100644 --- a/ansible/galaxy/CHANGELOG.md +++ b/ansible/galaxy/CHANGELOG.md @@ -8,6 +8,44 @@ collection adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) ### Added +- `sponsord_onramp` role: deploys `sponsord-onramp` (decdn/sponsord), the public + side of the sponsor: the Cloudflare Turnstile gate, the `decdn.sh` / `decdn.ps1` + installers and the API the `decdn-sponsored` CLI polls. It runs on the daemon's + host (the token gate and `/healthz` fail without the daemon) and reads the daemon's own + `/etc/sponsord/api-token`. + - Install: a local binary (`sponsord_onramp_install_method: manual`, the default + until upstream tags a release; `sponsord_onramp_release_target_dir` follows + `sponsord_release_target_dir`) or a `sponsord-onramp-v` release + verified against its GPG-signed `SHA256SUMS`, with the keys vendored in the + `sponsord` role. + - Unit: `DynamicUser`, the daemon token and the host-provisioned Turnstile secret + (`0600 /etc/sponsord/turnstile-secret`, never handled by the role) as + `LoadCredential=` credentials, `PartOf=sponsord.service` so it re-reads the + daemon's limits when the daemon restarts. + - Inputs: `sponsord_onramp_rpc_url` is published to every user, so the role + refuses credentials, a query or any character the installers cannot quote. + The release pins (`sponsord_onramp_decdn_release` / `_cli_release` and their + `SHA256SUMS` digests) and the Turnstile sitekey are checked against upstream's + shapes. CapacityBond and SlashJudge come from + `roles/sponsord_onramp/vars/main/networks.yml`, generated by + `scripts/sync-network-profiles.py`, through `sponsord_onramp_network` (which + follows `sponsord_network` and must agree with it). + - Listener: loopback only (asserted). The deploy fails unless `/healthz` answers + from a listener owned by the running unit. A restart-input record restarts the + onramp after an out-of-band rotation of the token or the Turnstile secret. + - TLS front: `sponsord_onramp_proxy: caddy` (default) installs the distro's Caddy + and owns `/etc/caddy/Caddyfile` (validated before it lands, admin API off, + `tls internal` for staging). A Caddyfile without the role's marker on a host + that already had Caddy is refused unless + `sponsord_onramp_caddy_overwrite_config: true`. The client address is the + right-most `X-Forwarded-For` Caddy sets. `none` leaves the proxy, the public + URL and its ports to you. +- `grafana_alloy_sponsord_onramp_enabled` (default `false`): label the onramp's + journald stream with `service_name="sponsord-onramp"` and a `level` parsed from + each line, as for sponsord. It has no `/metrics`, so there is no scrape. The + systemd collector's default unit list now includes `sponsord-onramp.service` and + `caddy.service`, which changes `config.alloy` once on every Grafana-enabled host. + - `sponsord` role: deploys the deCDN onboarding sponsor (decdn/sponsord) on its own host or beside a node. It installs a local binary (`sponsord_install_method: manual`, the default until upstream tags a release) or a `sponsord-v` release @@ -125,6 +163,13 @@ collection adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) ### Fixed +- On a host in both `decdn_nodes` and `sponsord_hosts`, the sponsord play's + baseline could not render the node's udp/4433 hole: it read `decdn_bind_port`, a + `decdn_node` default that play does not load. The public holes are now built + from the host's groups in `playbooks/group_vars/all.yml`, so every play renders + the same firewall. With `sponsord_onramp_hosts` they include tcp/80 and tcp/443 + for Caddy. + - `grafana_alloy` no longer ships URL credentials in journald lines to Grafana Cloud Loki (#84). Every line was forwarded verbatim, so a daemon error that quoted the RPC URL (sponsord's `error sending request for url (…)`) put its diff --git a/ansible/galaxy/README.md b/ansible/galaxy/README.md index 7e35a2d..18ee927 100644 --- a/ansible/galaxy/README.md +++ b/ansible/galaxy/README.md @@ -2,14 +2,15 @@ Deploy and harden a **public [deCDN](https://decdn.org) node**. This collection is the public, reusable slice of the [`decdn/devops`](https://github.com/decdn/devops) -repository — four roles and nothing else: +repository — five roles and nothing else: | Role | Purpose | |------|---------| | `decdn.node.baseline` | Debian/Ubuntu host baseline — nftables default-deny inbound, fail2ban, unattended-upgrades, chrony, an admin sudo user, then DevSec OS + SSH hardening (applied last). | | `decdn.node.decdn_node` | The `decdn-node` daemon under a hardened systemd unit — from locally built binaries (the default until upstream tags a release) or a GPG-verified release tarball; public QUIC udp/4433, loopback metrics + admin RPC. `decdn_network` sets the chain from upstream's manifest; `tasks_from: backup` / `decommission` for day 2. | -| `decdn.node.grafana_alloy` | Opt-in Grafana Cloud observability agent — loopback-only Alloy receiver and hardened telemetry export; scrapes the node, `sponsord`, or both (`grafana_alloy_node_enabled` / `grafana_alloy_sponsord_enabled`). | +| `decdn.node.grafana_alloy` | Opt-in Grafana Cloud observability agent — loopback-only Alloy receiver and hardened telemetry export; scrapes the node, `sponsord`, or both (`grafana_alloy_node_enabled` / `grafana_alloy_sponsord_enabled`), and labels the onramp's journal (`grafana_alloy_sponsord_onramp_enabled`). | | `decdn.node.sponsord` | The `sponsord` onboarding sponsor (treasury signer + PaymentPool keeper), standalone or beside a node — local binary or GPG-verified `sponsord-v*` release, `DynamicUser` unit with the API token and treasury wallet as systemd credentials, loopback-only API, `/healthz` deploy gate. `sponsord_network` sets the chain and PaymentPool from upstream's manifest. | +| `decdn.node.sponsord_onramp` | `sponsord-onramp`, sponsord's public side (Turnstile gate, installers, CLI API), on the daemon's host — local binary or GPG-verified `sponsord-onramp-v*` release, `DynamicUser` unit with the daemon token and Turnstile secret as systemd credentials, loopback listener behind Caddy (distro package, ACME TLS, admin API off; or your own proxy), `/healthz` deploy gate. | ## Requirements @@ -81,10 +82,15 @@ full variable list, the eth-keystore prerequisite, and day-2 ops: - [`roles/decdn_node`](https://github.com/decdn/devops/tree/main/ansible/roles/decdn_node) - [`roles/grafana_alloy`](https://github.com/decdn/devops/tree/main/ansible/roles/grafana_alloy) - [`roles/sponsord`](https://github.com/decdn/devops/tree/main/ansible/roles/sponsord) +- [`roles/sponsord_onramp`](https://github.com/decdn/devops/tree/main/ansible/roles/sponsord_onramp) ## Security model Backends bind `127.0.0.1`; the node opens exactly one public hole (QUIC udp/4433). +`sponsord_onramp` also listens on loopback: Caddy is its public side, so a host that +runs it opens tcp/80 and tcp/443 for Caddy in `baseline_extra_inbound` (this repo's +playbooks derive that from `sponsord_onramp_hosts`; from your own playbook, add them +yourself). No secrets ship in the collection or are committed — the eth keystore is operator-provisioned on the host, and `rpc_url` (which may embed an API key) renders to a `0600` file. SSH hardening is applied last, after the admin key is in place, so diff --git a/ansible/galaxy/build.sh b/ansible/galaxy/build.sh index f81f88b..d640ff8 100755 --- a/ansible/galaxy/build.sh +++ b/ansible/galaxy/build.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash # Stage and build the public `decdn.node` Galaxy collection. # -# Only the roles/baseline, roles/decdn_node, roles/grafana_alloy and roles/sponsord sources ship. All deploy machinery +# Only the roles/baseline, roles/decdn_node, roles/grafana_alloy, roles/sponsord and +# roles/sponsord_onramp sources ship. All deploy machinery # (inventory, Makefile, ansible.cfg) is excluded BY CONSTRUCTION — it is simply # never copied into the staging tree. This keeps the artifact clean and leaves the # internal project untouched (no galaxy.yml at the project root, so ansible-lint @@ -16,7 +17,7 @@ repo_root="$(cd "$ansible_dir/.." && pwd)" # repo root build_dir="$ansible_dir/build" stage="$build_dir/ansible_collections/decdn/node" -roles=(baseline decdn_node grafana_alloy sponsord) +roles=(baseline decdn_node grafana_alloy sponsord sponsord_onramp) echo "staging decdn.node -> $stage" rm -rf "$stage" diff --git a/ansible/galaxy/galaxy.yml b/ansible/galaxy/galaxy.yml index c54cfcf..3e63d97 100644 --- a/ansible/galaxy/galaxy.yml +++ b/ansible/galaxy/galaxy.yml @@ -3,7 +3,8 @@ # slice of this DevOps repo: deploy + harden a deCDN node. # # This file is an OVERLAY, not the project root manifest. It is assembled into a -# clean collection tree (with the baseline, decdn_node, grafana_alloy and sponsord roles) by galaxy/build.sh; +# clean collection tree (with the baseline, decdn_node, grafana_alloy, sponsord and +# sponsord_onramp roles) by galaxy/build.sh; # it is deliberately NOT placed at ansible/ root, so the deploy project stays a # plain Ansible project (bare role names, roles_path, production-profile # ansible-lint) rather than being reinterpreted as a collection. See galaxy/README.md. @@ -15,8 +16,8 @@ authors: - deCDN Contributors description: >- Deploy and harden a public deCDN node: DevSec host baseline, the decdn_node - daemon (with backup and decommission), the sponsord onboarding sponsor, and - opt-in Grafana Cloud observability. + daemon (with backup and decommission), the sponsord onboarding sponsor and its + public onramp, and opt-in Grafana Cloud observability. license: - MIT # Galaxy tags must match ^[a-z0-9]+$ (no hyphens/underscores). @@ -36,6 +37,7 @@ tags: # decdn_node -> ansible.builtin only # grafana_alloy -> ansible.builtin only # sponsord -> ansible.builtin only +# sponsord_onramp -> ansible.builtin only (Caddy comes from the distro's apt) # community.general is NOT used by any role, so it is intentionally absent here. dependencies: devsec.hardening: ">=10.0.0" diff --git a/ansible/inventory/group_vars/decdn_nodes.yml b/ansible/inventory/group_vars/decdn_nodes.yml index 2b3fc5d..bf1a7c6 100644 --- a/ansible/inventory/group_vars/decdn_nodes.yml +++ b/ansible/inventory/group_vars/decdn_nodes.yml @@ -5,8 +5,8 @@ # you would rather the role authored that file) in a git-ignored secret.yml. # The node's public QUIC firewall hole (udp/4433) is NOT set here. It lives in -# playbooks/group_vars/decdn_nodes.yml so it applies to every inventory, including a -# private fleet overlay passed with INVENTORY=. +# playbooks/group_vars/ (all.yml builds it) so it applies to every inventory, +# including a private fleet overlay passed with INVENTORY=. # # Settings every node shares (decdn_network, decdn_node_install_method, …) can move # here from host_vars//main.yml once you run more than one node. host_vars diff --git a/ansible/inventory/group_vars/sponsord_onramp_hosts.yml b/ansible/inventory/group_vars/sponsord_onramp_hosts.yml new file mode 100644 index 0000000..75d9268 --- /dev/null +++ b/ansible/inventory/group_vars/sponsord_onramp_hosts.yml @@ -0,0 +1,35 @@ +--- +# Shared config for sponsord-onramp hosts (the second play of playbooks/sponsord.yml, +# also run by site.yml). Each host must be in sponsord_hosts too. Uncomment and fill +# in; per-host values can move to host_vars//main.yml. +# Full variable list: roles/sponsord_onramp/defaults/main.yml and its README.md. +# +# Before the first deploy, on the host as root (the role never handles it): +# /etc/sponsord/turnstile-secret the Turnstile widget's secret key, 0600 +# The daemon's API token (/etc/sponsord/api-token) is the sponsord role's. + +# --- Binary ------------------------------------------------------------------------ +# Follows sponsord_release_target_dir (`cargo build --release` in a decdn/sponsord +# checkout builds both binaries). Once a release exists: +# sponsord_onramp_install_method: release +# sponsord_onramp_version: "0.1.0" + +# --- Public address ---------------------------------------------------------------- +# Caddy terminates TLS for this name (ACME). Its A/AAAA records must point at the +# host; the playbooks open tcp/80 + tcp/443. To bring your own proxy instead, set +# sponsord_onramp_proxy: none (HERE, not in a play: the firewall holes read it) and +# sponsord_onramp_public_url. +# sponsord_onramp_domain: downloads.example.org +# sponsord_onramp_acme_email: ops@example.org + +# --- What users get (REQUIRED) ----------------------------------------------------- +# PUBLIC: served to every user. A keyless endpoint, never sponsord_rpc_url. +# sponsord_onramp_rpc_url: https://sepolia-rollup.arbitrum.io/rpc +# sponsord_onramp_turnstile_sitekey: "0x…" +# The releases the installers install: tags and the SHA-256 of each SHA256SUMS +# (decdn-sponsored's release notes print both of its values). +# sponsord_onramp_decdn_release: v0.1.0 +# sponsord_onramp_decdn_sums_sha256: "…" +# sponsord_onramp_cli_release: decdn-sponsored-v0.1.0 +# sponsord_onramp_cli_sums_sha256: "…" +# CapacityBond and SlashJudge come from sponsord_network (group_vars/sponsord_hosts.yml). diff --git a/ansible/inventory/hosts.yml.example b/ansible/inventory/hosts.yml.example index e4e97d5..10e99c3 100644 --- a/ansible/inventory/hosts.yml.example +++ b/ansible/inventory/hosts.yml.example @@ -60,3 +60,12 @@ decdn_nodes: # hosts: # sponsord-1: # ansible_host: REPLACE_WITH_SPONSORD_VPS_IP_OR_DNS +# +# OPTIONAL: sponsord-onramp, sponsord's public side (Turnstile gate, installers, the +# decdn-sponsored CLI API), behind Caddy on tcp/80 + tcp/443. List a sponsord host +# again here; it must be in sponsord_hosts too. Its knobs are in +# group_vars/sponsord_onramp_hosts.yml; the Turnstile secret is provisioned on the +# host (roles/sponsord_onramp/README.md). +# sponsord_onramp_hosts: +# hosts: +# sponsord-1: diff --git a/ansible/molecule/sponsord-onramp/converge.yml b/ansible/molecule/sponsord-onramp/converge.yml new file mode 100644 index 0000000..2e0611e --- /dev/null +++ b/ansible/molecule/sponsord-onramp/converge.yml @@ -0,0 +1,7 @@ +--- +# The real entry point: both plays of playbooks/sponsord.yml (the daemon, then the +# onramp), so playbooks/group_vars/all.yml derives the Alloy toggles exactly as a +# deploy would. Variables come from molecule.yml's inventory group_vars; baseline +# is skipped there. +- name: Converge + ansible.builtin.import_playbook: ../../playbooks/sponsord.yml diff --git a/ansible/molecule/sponsord-onramp/files/gate.html b/ansible/molecule/sponsord-onramp/files/gate.html new file mode 100644 index 0000000..8f51084 --- /dev/null +++ b/ansible/molecule/sponsord-onramp/files/gate.html @@ -0,0 +1,3 @@ + + +

{{BRAND_NAME}}

{{CLIENT}}

diff --git a/ansible/molecule/sponsord-onramp/files/sponsord-onramp-stub b/ansible/molecule/sponsord-onramp/files/sponsord-onramp-stub new file mode 100755 index 0000000..1ebddc1 --- /dev/null +++ b/ansible/molecule/sponsord-onramp/files/sponsord-onramp-stub @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +"""Stub sponsord-onramp binary for the Molecule scenarios. + +Stands in for the real `sponsord-onramp` so the role can be exercised end to end +without a release or a Turnstile account: + + * `--version` -> print a version line and exit 0 (the role's version backstop). + * no argument -> enforce the startup contract the real onramp enforces, then + serve on ONRAMP_BIND until killed: + - ONRAMP_PUBLIC_URL, _RPC_URL, _CAPACITY_BOND_ADDR, + _TURNSTILE_SITEKEY and the four release-pin variables set; + - ONRAMP_DAEMON_TOKEN_FILE readable and >= 32 bytes; + - ONRAMP_TURNSTILE_SECRET_FILE readable and non-empty; + - ONRAMP_BRAND_NAME, when set, equal to EXPECTED_BRAND byte for + byte (proves the role's EnvironmentFile quoting); + - ONRAMP_BIND and ONRAMP_DAEMON_URL set (no fallback, so a + template that drops either is caught); + - ONRAMP_GATE_TEMPLATE, when set, readable by this unit; + - the daemon at ONRAMP_DAEMON_URL answers /v1/info for the + token in ONRAMP_DAEMON_TOKEN_FILE (upstream reads /v1/info at + start and exits if it can't). + Any violation exits 1 with the reason on stderr (the journal). + The marker file UNHEALTHY_MARKER makes it exit 1 on start, + which is how the scenario proves the readiness gate is fatal. + Routes: /healthz, and /molecule/client-ip, which echoes the + header named by ONRAMP_CLIENT_IP_HEADER, so the scenario can + check what Caddy passes on as the client address. + * anything else -> exit 2. +""" +import os +import sys +import urllib.request +from http.server import BaseHTTPRequestHandler, HTTPServer + +VERSION = "sponsord-onramp 0.0.0-molecule-stub" +UNHEALTHY_MARKER = "/etc/sponsord/onramp-stub-unhealthy" +# Must match sponsord_onramp_brand_name in the scenario's inventory. +EXPECTED_BRAND = 'deCDN "molecule" $HOME' +REQUIRED = ( + "ONRAMP_BIND", "ONRAMP_DAEMON_URL", "ONRAMP_PUBLIC_URL", "ONRAMP_RPC_URL", "ONRAMP_CAPACITY_BOND_ADDR", "ONRAMP_TURNSTILE_SITEKEY", + "ONRAMP_DECDN_RELEASE", "ONRAMP_DECDN_SUMS_SHA256", "ONRAMP_CLI_RELEASE", "ONRAMP_CLI_SUMS_SHA256", +) + + +def die(msg): + print(f"sponsord-onramp-stub: {msg}", file=sys.stderr) + sys.exit(1) + + +def read_secret(var): + path = os.environ.get(var) + if not path: + die(f"{var} is not set") + try: + with open(path, "rb") as f: + value = f.read() + except OSError as e: + die(f"{var}: cannot read {path}: {e}") + value = value.removesuffix(b"\n").removesuffix(b"\r") + if not value: + die(f"{var}: {path} is empty") + return value + + +def check_startup(): + if os.path.exists(UNHEALTHY_MARKER): + die(f"{UNHEALTHY_MARKER} exists: refusing to start") + for var in REQUIRED: + if not os.environ.get(var): + die(f"{var} is not set") + if os.environ.get("ONRAMP_BRAND_NAME", EXPECTED_BRAND) != EXPECTED_BRAND: + die("ONRAMP_BRAND_NAME does not match the inventory literal (EnvironmentFile quoting broke it)") + token = read_secret("ONRAMP_DAEMON_TOKEN_FILE") + if len(token) < 32: + die("daemon token is shorter than 32 bytes") + read_secret("ONRAMP_TURNSTILE_SECRET_FILE") + template = os.environ.get("ONRAMP_GATE_TEMPLATE") + if template: + try: + with open(template, "rb") as f: + f.read() + except OSError as e: + die(f"ONRAMP_GATE_TEMPLATE {template}: {e}") + daemon = os.environ["ONRAMP_DAEMON_URL"] + req = urllib.request.Request(f"{daemon}/v1/info", headers={"Authorization": "Bearer " + token.decode()}) + try: + with urllib.request.urlopen(req, timeout=5) as r: + if r.status != 200: + die(f"daemon {daemon} answered {r.status}") + except OSError as e: + die(f"daemon {daemon}/v1/info failed (unreachable, or the token is not the daemon's): {e}") + + +class Handler(BaseHTTPRequestHandler): + def do_GET(self): + if self.path == "/healthz": + body = b'{"ok":true}' + elif self.path == "/molecule/client-ip": + header = os.environ.get("ONRAMP_CLIENT_IP_HEADER", "") + body = (self.headers.get(header, "") if header else self.client_address[0]).encode() + else: + self.send_error(404) + return + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args): + pass + + +def main(): + if sys.argv[1:] == ["--version"]: + print(VERSION) + return + if sys.argv[1:]: + print(f"sponsord-onramp-stub: unexpected arguments {sys.argv[1:]}", file=sys.stderr) + sys.exit(2) + check_startup() + host, _, port = os.environ["ONRAMP_BIND"].rpartition(":") + HTTPServer((host, int(port)), Handler).serve_forever() + + +if __name__ == "__main__": + main() diff --git a/ansible/molecule/sponsord-onramp/molecule.yml b/ansible/molecule/sponsord-onramp/molecule.yml new file mode 100644 index 0000000..4fcc569 --- /dev/null +++ b/ansible/molecule/sponsord-onramp/molecule.yml @@ -0,0 +1,111 @@ +--- +# sponsord-onramp beside its daemon: a host in sponsord_hosts AND +# sponsord_onramp_hosts, converged through the real playbooks/sponsord.yml (both +# plays) against the stub daemon (../sponsord/files/sponsord-stub), a stub onramp +# (files/sponsord-onramp-stub) and the REAL distro Caddy, with Caddy's internal CA +# in place of ACME (no public DNS in a container). +# +# converge the daemon (as ../sponsord, minus its RPC literal check), then the +# onramp: the host-provisioned Turnstile secret locked to 0600, the +# daemon's token as a credential, loopback listener, /healthz gate, +# Caddy terminating TLS for the domain with the role's Caddyfile; +# Alloy labelling the onramp's journal stream. +# idempotence an unchanged host must not restart anything. +# side_effect rotate the Turnstile secret out of band -> the next converge must +# restart the onramp (and only it); make the onramp refuse to start +# -> the /healthz gate must FAIL the converge; strip the role's +# marker from the Caddyfile -> the role must refuse to replace it +# until sponsord_onramp_caddy_overwrite_config is set. +# verify files, modes, unit hardening, credentials, listeners, the whole +# https chain through Caddy, the client address Caddy passes on. +# +# `baseline` is skipped (real-host-only — see ../default/molecule.yml), so the +# tcp/80 + tcp/443 holes are covered by tests/firewall-holes instead. Release +# mode is covered for the sponsord role (../sponsord), whose install tasks this +# role mirrors. The broken-input matrix lives in ../validation. +driver: + name: docker +platforms: + - name: onramp-molecule + # Same digest-pinned images as ../sponsord (repo convention). + image: geerlingguy/docker-debian12-ansible@sha256:a131e3b7a9a38dda40e98b49b39d6200ffab0ad853c714a513f5eea41bab9a25 + pre_build_image: true + command: /usr/lib/systemd/systemd + privileged: true + cgroupns_mode: host + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + groups: + - sponsord_hosts + - sponsord_onramp_hosts + # systemd 255 writes LoadCredential= files 0440 and Caddy comes from universe. + - name: onramp-molecule-u2404 + image: geerlingguy/docker-ubuntu2404-ansible@sha256:3394e335df58c961c00b78ae601ef30cdd7463b112f21b0c5f78674a70526302 + pre_build_image: true + command: /usr/lib/systemd/systemd + privileged: true + cgroupns_mode: host + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + groups: + - sponsord_hosts + - sponsord_onramp_hosts +provisioner: + name: ansible + options: + skip-tags: baseline + env: + ANSIBLE_ROLES_PATH: "${MOLECULE_PROJECT_DIRECTORY}/roles" + ANSIBLE_COLLECTIONS_PATH: "${MOLECULE_PROJECT_DIRECTORY}/collections" + # Same pipelining note as ../default/molecule.yml. + ANSIBLE_PIPELINING: "true" + inventory: + group_vars: + sponsord_hosts: + sponsord_install_method: manual + sponsord_manual_bin_src: "${MOLECULE_PROJECT_DIRECTORY}/molecule/sponsord/files/sponsord-stub" + sponsord_version: "0.0.0-molecule-stub" + sponsord_network: arbitrum-sepolia + sponsord_pool_id: "0x00000000000000000000000000000000000000000000000000000000000000aa" + sponsord_rpc_url: "https://rpc.example.invalid/v2/molecule" + sponsord_readiness_retries: 10 + sponsord_readiness_delay: 1 + decdn_grafana_cloud_enabled: true + grafana_alloy_install_method: manual + grafana_alloy_manual_bin_src: "${MOLECULE_PROJECT_DIRECTORY}/molecule/grafana-cloud/files/alloy-stub" + grafana_alloy_region: US + grafana_alloy_loki_url: https://logs-prod-xx.molecule.invalid/loki/api/v1/push + sponsord_onramp_hosts: + sponsord_onramp_install_method: manual + sponsord_onramp_manual_bin_src: "${MOLECULE_SCENARIO_DIRECTORY}/files/sponsord-onramp-stub" + sponsord_onramp_version: "0.0.0-molecule-stub" + # sponsord_onramp_network follows sponsord_network (arbitrum-sepolia). + sponsord_onramp_rpc_url: https://sepolia-rollup.arbitrum.io/rpc + sponsord_onramp_domain: onramp.molecule.test + sponsord_onramp_caddy_tls: internal + # Renders (and so validates) the Caddyfile's global email option. + sponsord_onramp_acme_email: ops@molecule.test + sponsord_onramp_turnstile_sitekey: 1x00000000000000000000AA + # Must equal EXPECTED_BRAND in files/sponsord-onramp-stub: quotes and a + # dollar sign prove the env file's quoting ($$ is molecule's escape). + sponsord_onramp_brand_name: 'deCDN "molecule" $$HOME' + sponsord_onramp_decdn_release: v0.1.0 + sponsord_onramp_decdn_sums_sha256: "1111111111111111111111111111111111111111111111111111111111111111" + sponsord_onramp_cli_release: decdn-sponsored-v0.1.0 + sponsord_onramp_cli_sums_sha256: "2222222222222222222222222222222222222222222222222222222222222222" + sponsord_onramp_fund_rate_per_min: 0 + sponsord_onramp_readiness_retries: 10 + sponsord_onramp_readiness_delay: 1 +verifier: + name: ansible +scenario: + test_sequence: + # Leading destroy: see ../default/molecule.yml. + - destroy + - create + - prepare + - converge + - idempotence + - side_effect + - verify + - destroy diff --git a/ansible/molecule/sponsord-onramp/prepare.yml b/ansible/molecule/sponsord-onramp/prepare.yml new file mode 100644 index 0000000..ff10630 --- /dev/null +++ b/ansible/molecule/sponsord-onramp/prepare.yml @@ -0,0 +1,51 @@ +--- +# Stage what an operator provisions before the first deploy: +# 1. the daemon's treasury keystore + password (as ../sponsord/prepare.yml); +# 2. the Turnstile secret, at 0644 so verify.yml can attribute the 0600 +# lock-down to the role; +# 3. the Grafana Cloud credential env file (as ../grafana-cloud/prepare.yml). +# The API token is NOT staged: the sponsord role generates it, and the onramp +# reads that same file. +# +# CONTAINER-ONLY step first, as ../sponsord/prepare.yml explains: without shared +# propagation on / every LoadCredential= directory comes up empty under docker. +- name: Prepare + hosts: all + become: true + tasks: + - name: Give / shared mount propagation, as systemd does on a real host # noqa: command-instead-of-module + ansible.builtin.command: + cmd: mount --make-rshared / + changed_when: true + + - name: Create the sponsord config directory + ansible.builtin.file: + path: /etc/sponsord + state: directory + mode: "0755" + + - name: Stage the placeholder treasury files and the Turnstile secret + ansible.builtin.copy: + dest: "/etc/sponsord/{{ item.name }}" + content: "{{ item.content }}" + mode: "0644" + loop: + - {name: treasury-keystore.json, content: "{{ '{}' }}\n"} + - {name: treasury-password, content: "molecule-placeholder\n"} + - {name: turnstile-secret, content: "1x0000000000000000000000000000000AA\n"} + loop_control: + label: "{{ item.name }}" + + - name: Stage the Grafana Cloud credential fixture on the host + ansible.builtin.copy: + dest: /etc/grafana-alloy.env + owner: root + group: root + mode: "0600" + content: | + GC_PROM_REMOTE_WRITE_URL=https://prometheus-prod-xx.molecule.invalid/api/prom/push + GC_OTLP_ENDPOINT=https://otlp-gateway-prod-xx.molecule.invalid/otlp + GC_PROM_USERNAME=999888777 + GC_OTLP_USERNAME=999888779 + GC_LOKI_USERNAME=999888778 + GC_API_TOKEN=molecule-test-token diff --git a/ansible/molecule/sponsord-onramp/side_effect.yml b/ansible/molecule/sponsord-onramp/side_effect.yml new file mode 100644 index 0000000..f35e010 --- /dev/null +++ b/ansible/molecule/sponsord-onramp/side_effect.yml @@ -0,0 +1,454 @@ +--- +# What a green converge + idempotence cannot prove: +# 1. Rotating the Turnstile secret OUT OF BAND restarts the onramp onto it, and +# only the onramp (a no-op re-run first proves nothing else restarts it). +# 2. The /healthz readiness gate is FATAL, and a failed run records nothing: the +# restart-input record keeps the last healthy inputs until a run succeeds. +# 3. With no record, a running onramp is restarted (it was never confirmed +# healthy on its inputs), and a daemon restart brings the onramp back by +# itself (PartOf=, Restart=on-failure, no start limit). +# 4. A custom gate page is installed, read by the onramp, and removed again. +# 5. proxy "none": no client-IP header, the public URL as given, and a warning +# that the role's Caddy is still installed. +# 6. ACME mode: the Caddyfile has no `tls internal`, Caddy's http redirect +# answers, and a certificate that cannot be issued only warns. With the +# internal CA, a broken upstream behind Caddy fails the run. +# 7. The role never silently replaces a Caddyfile it did not write; the package's +# untouched default it does replace. +# The host is left healthy on the role's own Caddyfile, `tls internal`, for verify. +- name: Side effect — rotation, the fatal gate and the restart-input record + hosts: all + become: true + vars: + _or_record: /etc/sponsord/.onramp-inputs.sha256 + tasks: + - name: Read the unit states before anything changes + ansible.builtin.systemd_service: + name: "{{ item }}" + register: _or_before + loop: [sponsord-onramp, sponsord] + + - name: Assert the onramp was running cleanly + ansible.builtin.assert: + that: + - _or_before.results[0].status.ActiveState == 'active' + - _or_before.results[0].status.MainPID | int > 0 + fail_msg: >- + Expected sponsord-onramp active (got + ActiveState={{ _or_before.results[0].status.ActiveState }}): + journalctl -u sponsord-onramp -e + + # include_role here, import_playbook in the converge: if the two contexts + # resolved any input differently, the onramp would restart for that alone, and + # the rotation check below would pass for the wrong reason. + - name: Re-run the onramp with nothing changed + ansible.builtin.include_role: + name: sponsord_onramp + + - name: Read the onramp after the no-op run + ansible.builtin.systemd_service: + name: sponsord-onramp + register: _or_noop + + - name: Require the no-op run to have left the onramp alone + ansible.builtin.assert: + that: + - _or_noop.status.MainPID == _or_before.results[0].status.MainPID + fail_msg: A re-run with no input changed restarted sponsord-onramp. + + - name: Rotate the Turnstile secret out of band + ansible.builtin.copy: + dest: /etc/sponsord/turnstile-secret + content: "1x0000000000000000000000000000000BB\n" + mode: "0600" + + - name: Converge the onramp again after the rotation + ansible.builtin.include_role: + name: sponsord_onramp + + - name: Read the unit states after the rotation converge + ansible.builtin.systemd_service: + name: "{{ item }}" + register: _or_after + loop: [sponsord-onramp, sponsord] + + - name: Require the rotation to have restarted the onramp, and only the onramp + ansible.builtin.assert: + that: + - _or_after.results[0].status.ActiveState == 'active' + - _or_after.results[0].status.MainPID != _or_before.results[0].status.MainPID + - _or_after.results[1].status.MainPID == _or_before.results[1].status.MainPID + fail_msg: >- + After the Turnstile secret was rotated, the onramp's PID went + {{ _or_before.results[0].status.MainPID }} -> {{ _or_after.results[0].status.MainPID }} + (must change) and the daemon's {{ _or_before.results[1].status.MainPID }} -> + {{ _or_after.results[1].status.MainPID }} (must not). + + - name: Hash the Turnstile secret the onramp is now healthy on + ansible.builtin.stat: + path: /etc/sponsord/turnstile-secret + checksum_algorithm: sha256 + register: _or_secret_b + + - name: Stop the onramp, then make it refuse to start + ansible.builtin.systemd_service: + name: sponsord-onramp + state: stopped + + - name: Stage the stub's refuse-to-start marker + ansible.builtin.copy: + dest: /etc/sponsord/onramp-stub-unhealthy + content: "" + mode: "0644" + + # A changed input on the failing run: the record must not take it. + - name: Rotate the Turnstile secret once more + ansible.builtin.copy: + dest: /etc/sponsord/turnstile-secret + content: "1x0000000000000000000000000000000CC\n" + mode: "0600" + + - name: Converge against an onramp that never becomes healthy + block: + - name: Run the role with a short readiness window + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_readiness_retries: 3 + sponsord_onramp_readiness_delay: 1 + + - name: Fail — the readiness gate let an unhealthy onramp through + ansible.builtin.fail: + msg: The role finished although sponsord-onramp never answered /healthz. + rescue: + - name: Require the failure to come from the readiness gate itself + ansible.builtin.assert: + that: + - ansible_failed_task.name == 'Report a sponsord-onramp that did not become healthy' + fail_msg: >- + Expected the role to fail at its readiness report, but it failed at + "{{ ansible_failed_task.name }}". + + - name: Read the restart-input record after the failed run + ansible.builtin.slurp: + src: "{{ _or_record }}" + register: _or_rec_failed + + # Field order: token, Turnstile secret, env file, gate page, unit, binary. + - name: Require the failed run to have left the record on the healthy inputs + ansible.builtin.assert: + that: + - (_or_rec_failed.content | b64decode).split()[1] == _or_secret_b.stat.checksum + fail_msg: >- + The record moved to the inputs of a run whose onramp never became + healthy; the next converge would not restart it onto them. + always: + - name: Remove the refuse-to-start marker + ansible.builtin.file: + path: /etc/sponsord/onramp-stub-unhealthy + state: absent + + - name: Converge once more to bring the onramp back + ansible.builtin.include_role: + name: sponsord_onramp + + - name: Hash the Turnstile secret now in place + ansible.builtin.stat: + path: /etc/sponsord/turnstile-secret + checksum_algorithm: sha256 + register: _or_secret_c + + - name: Read the restart-input record after the recovery + ansible.builtin.slurp: + src: "{{ _or_record }}" + register: _or_rec_ok + + - name: Require the record to follow the inputs once the onramp is healthy + ansible.builtin.assert: + that: + - (_or_rec_ok.content | b64decode).split()[1] == _or_secret_c.stat.checksum + fail_msg: The healthy run did not record the Turnstile secret it started on. + +- name: Side effect — a missing record, and a daemon restart + hosts: all + become: true + tasks: + - name: Read the onramp before the record is removed + ansible.builtin.systemd_service: + name: sponsord-onramp + register: _or_norec_before + + - name: Remove the restart-input record + ansible.builtin.file: + path: /etc/sponsord/.onramp-inputs.sha256 + state: absent + + - name: Converge with no record + ansible.builtin.include_role: + name: sponsord_onramp + + - name: Read the onramp and the record after that converge + ansible.builtin.systemd_service: + name: sponsord-onramp + register: _or_norec_after + + - name: Stat the rewritten record + ansible.builtin.stat: + path: /etc/sponsord/.onramp-inputs.sha256 + register: _or_norec_rec + + - name: Require a restart and a fresh record + ansible.builtin.assert: + that: + - _or_norec_after.status.MainPID != _or_norec_before.status.MainPID + - _or_norec_rec.stat.exists + fail_msg: >- + With no record, a running onramp must be restarted (it was never confirmed + healthy on its inputs) and the record written again. + + - name: Restart the daemon + ansible.builtin.systemd_service: + name: sponsord + state: restarted + + # The onramp restarts with the daemon (PartOf=), usually before the daemon + # listens; it must exit, retry and come back on its own, with no converge. + - name: Wait for the onramp to come back by itself + ansible.builtin.uri: + url: http://127.0.0.1:8080/healthz + status_code: 200 + timeout: 5 + register: _or_partof + retries: 30 + delay: 1 + until: _or_partof.status == 200 + + - name: Read the onramp after the daemon restart + ansible.builtin.systemd_service: + name: sponsord-onramp + register: _or_partof_unit + + - name: Require the onramp to have restarted with the daemon + ansible.builtin.assert: + that: + - _or_partof_unit.status.ActiveState == 'active' + - _or_partof_unit.status.MainPID != _or_norec_after.status.MainPID + fail_msg: The onramp did not restart with the daemon (PartOf=sponsord.service). + +- name: Side effect — a custom gate page + hosts: all + become: true + vars: + _or_page: "{{ lookup('ansible.builtin.env', 'MOLECULE_SCENARIO_DIRECTORY') }}/files/gate.html" + tasks: + - name: Converge with a custom gate page + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_gate_template_src: "{{ _or_page }}" + + - name: Stat the installed page and read the env file + ansible.builtin.stat: + path: /etc/sponsord/onramp-gate.html + register: _or_page_stat + + - name: Read the env file with the page set + ansible.builtin.slurp: + src: /etc/sponsord/sponsord-onramp.env + register: _or_page_env + + # The stub reads the page at start, as the onramp's dynamic user: the role's + # /healthz gate passing already proves it was readable. + - name: Require the page installed root-owned, world-readable and configured + ansible.builtin.assert: + that: + - _or_page_stat.stat.isreg + - _or_page_stat.stat.mode == '0644' + - _or_page_stat.stat.pw_name == 'root' + - "'ONRAMP_GATE_TEMPLATE=/etc/sponsord/onramp-gate.html' in (_or_page_env.content | b64decode)" + fail_msg: The custom gate page is not installed, or not configured. + + - name: Converge without the page again + ansible.builtin.include_role: + name: sponsord_onramp + + - name: Stat the page after it was dropped + ansible.builtin.stat: + path: /etc/sponsord/onramp-gate.html + register: _or_page_gone + + - name: Read the env file without the page + ansible.builtin.slurp: + src: /etc/sponsord/sponsord-onramp.env + register: _or_page_env_gone + + - name: Require the page and its setting gone + ansible.builtin.assert: + that: + - not _or_page_gone.stat.exists + - "'ONRAMP_GATE_TEMPLATE' not in (_or_page_env_gone.content | b64decode)" + fail_msg: The dropped gate page is still installed or configured. + +- name: Side effect — bring your own proxy + hosts: all + become: true + tasks: + - name: Converge with proxy "none" + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_proxy: none + sponsord_onramp_public_url: https://downloads.molecule.test + + - name: Read the env file in proxy "none" mode + ansible.builtin.slurp: + src: /etc/sponsord/sponsord-onramp.env + register: _or_none_env + + # With an arbitrary proxy in front, trusting X-Forwarded-For would let clients + # pick the address the rate limits key on. + - name: Require no client-IP header, the given public URL, and the stale-Caddy warning + ansible.builtin.assert: + that: + - "'ONRAMP_CLIENT_IP_HEADER' not in _env" + - "'ONRAMP_PUBLIC_URL=https://downloads.molecule.test\n' in _env" + - sponsord_onramp_stale_caddy.rc == 0 + fail_msg: "{{ _env }}" + vars: + _env: "{{ _or_none_env.content | b64decode }}" + +- name: Side effect — ACME, and a broken upstream behind Caddy + hosts: all + become: true + tasks: + # No public DNS points at a container, so no certificate can be issued: the + # role must still finish, on the redirect probe, with only the ACME warning. + - name: Converge in ACME mode + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_caddy_tls: acme + sponsord_onramp_readiness_retries: 2 + + - name: Read the ACME-mode Caddyfile + ansible.builtin.slurp: + src: /etc/caddy/Caddyfile + register: _or_acme_caddy + + - name: Require ACME's Caddyfile, and the certificate failure judged advisory + ansible.builtin.assert: + that: + - "'tls internal' not in (_or_acme_caddy.content | b64decode)" + - sponsord_onramp_caddy_redirect.status == 308 + - sponsord_onramp_caddy_probe.rc in [35, 60] + fail_msg: >- + ACME mode: tls internal still present, the redirect probe got + {{ sponsord_onramp_caddy_redirect.status | default('none') }}, or curl exit + {{ sponsord_onramp_caddy_probe.rc }} is not a pending-certificate failure. + + - name: Gather package facts for the Caddy tasks run alone + ansible.builtin.package_facts: + + # Caddy pointed at a port nothing listens on: TLS works (internal CA), the + # upstream answers 502, and that must fail the run. + - name: Run the Caddy tasks against a dead upstream + block: + - name: Point Caddy at a port nothing listens on + ansible.builtin.include_role: + name: sponsord_onramp + tasks_from: caddy + vars: + sponsord_onramp_port: 8099 + sponsord_onramp_readiness_retries: 2 + + - name: Fail — the Caddy probe let a dead upstream through + ansible.builtin.fail: + msg: The role finished although Caddy could not reach the onramp. + rescue: + - name: Require the failure to come from the Caddy probe report + ansible.builtin.assert: + that: + - ansible_failed_task.name == 'Report an onramp that is not reachable through Caddy' + fail_msg: >- + Expected the Caddy probe to fail the run, but it failed at + "{{ ansible_failed_task.name }}". + + - name: Converge back onto the internal CA and the real upstream + ansible.builtin.include_role: + name: sponsord_onramp + +- name: Side effect — a Caddyfile someone else wrote, and the package's default + hosts: all + become: true + tasks: + - name: Replace the Caddyfile with one this role did not write + ansible.builtin.copy: + dest: /etc/caddy/Caddyfile + mode: "0644" + content: | + # the operator's own site + other.molecule.test { + tls internal + respond "not the onramp" + } + + - name: Converge against the foreign Caddyfile + block: + - name: Run the role without the overwrite confirmation + ansible.builtin.include_role: + name: sponsord_onramp + + - name: Fail — the role replaced a Caddyfile it did not write + ansible.builtin.fail: + msg: The role finished although /etc/caddy/Caddyfile carried no marker. + rescue: + - name: Require the failure to come from the Caddyfile refusal + ansible.builtin.assert: + that: + - ansible_failed_task.name == 'Refuse to replace a Caddyfile this role did not write' + fail_msg: >- + Expected the role to refuse the foreign Caddyfile, but it failed at + "{{ ansible_failed_task.name }}". + + - name: Read the Caddyfile after the refusal + ansible.builtin.slurp: + src: /etc/caddy/Caddyfile + register: _or_foreign + + - name: Require the foreign Caddyfile to be untouched + ansible.builtin.assert: + that: + - "'not the onramp' in (_or_foreign.content | b64decode)" + fail_msg: The refused run still changed /etc/caddy/Caddyfile. + + - name: Converge with the takeover confirmed + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_caddy_overwrite_config: true + + # What a first run leaves when it installs Caddy and stops before templating. + - name: Put the package's own default Caddyfile back + ansible.builtin.shell: + executable: /bin/bash + cmd: | + set -euo pipefail + cd "$(mktemp -d)" + apt-get download caddy >/dev/null + dpkg-deb --fsys-tarfile caddy_*.deb | tar -xO ./etc/caddy/Caddyfile > /etc/caddy/Caddyfile + changed_when: true + + - name: Converge over the package's default Caddyfile, unconfirmed + ansible.builtin.include_role: + name: sponsord_onramp + + - name: Read the Caddyfile after replacing the default + ansible.builtin.slurp: + src: /etc/caddy/Caddyfile + register: _or_after_default + + - name: Require the role's Caddyfile in place of the default + ansible.builtin.assert: + that: + - "'MANAGED BY the sponsord_onramp role' in (_or_after_default.content | b64decode)" + fail_msg: The role did not take over the caddy package's untouched default Caddyfile. diff --git a/ansible/molecule/sponsord-onramp/verify.yml b/ansible/molecule/sponsord-onramp/verify.yml new file mode 100644 index 0000000..15d833d --- /dev/null +++ b/ansible/molecule/sponsord-onramp/verify.yml @@ -0,0 +1,179 @@ +--- +# What a converged onramp host must look like. Runs after side_effect.yml, which +# leaves the host healthy on the role's own Caddyfile. +- name: Verify + hosts: all + become: true + gather_facts: false + tasks: + - name: Load the onramp's network profile mirror + ansible.builtin.include_vars: + # Relative to this playbook's directory. + file: ../../roles/sponsord_onramp/vars/main/networks.yml + + - name: Stat the onramp's files + ansible.builtin.stat: + path: "{{ item }}" + register: _or_files + loop: + - /etc/sponsord/turnstile-secret + - /etc/sponsord/sponsord-onramp.env + - /etc/sponsord/.onramp-inputs.sha256 + - /etc/caddy/Caddyfile + + - name: Require the secret and records root-only, the config world-readable + ansible.builtin.assert: + that: + - _or_files.results[0].stat.isreg and _or_files.results[0].stat.mode == '0600' + - _or_files.results[0].stat.pw_name == 'root' + - _or_files.results[1].stat.isreg and _or_files.results[1].stat.mode == '0644' + - _or_files.results[2].stat.isreg and _or_files.results[2].stat.mode == '0600' + - _or_files.results[3].stat.isreg and _or_files.results[3].stat.mode == '0644' + fail_msg: "{{ _or_files.results | map(attribute='stat') | map(attribute='mode', default='absent') | list }}" + + - name: Read the env file and the Caddyfile + ansible.builtin.slurp: + src: "{{ item }}" + register: _or_text + loop: + - /etc/sponsord/sponsord-onramp.env + - /etc/caddy/Caddyfile + + - name: Require the env file to carry the settings and no secret + ansible.builtin.assert: + that: + - "'ONRAMP_BIND=127.0.0.1:8080' in _env" + - "'ONRAMP_DAEMON_URL=http://127.0.0.1:8090' in _env" + - "'ONRAMP_PUBLIC_URL=https://onramp.molecule.test' in _env" + - "'ONRAMP_RPC_URL=https://sepolia-rollup.arbitrum.io/rpc' in _env" + # Exactly the mirror's addresses for the daemon's sponsord_network. + - "('ONRAMP_CAPACITY_BOND_ADDR=' ~ _profile.capacity_bond_address ~ '\n') in _env" + - "('ONRAMP_SLASH_JUDGE_ADDR=' ~ _profile.slash_judge_address ~ '\n') in _env" + - "'ONRAMP_CLIENT_IP_HEADER=X-Forwarded-For' in _env" + - "'ONRAMP_FUND_RATE_PER_MIN=0' in _env" + - "'ONRAMP_TTL_SECS' not in _env" + - "'TOKEN' not in _env" + - "'TURNSTILE_SECRET' not in _env" + - "'1x0000000000000000000000000000000' not in _env" + fail_msg: "{{ _env }}" + vars: + _env: "{{ _or_text.results[0].content | b64decode }}" + _profile: "{{ _sponsord_onramp_network_profiles['arbitrum-sepolia'] }}" + + - name: Require the role's Caddyfile, with its admin API off + ansible.builtin.assert: + that: + - "'MANAGED BY the sponsord_onramp role' in _caddy" + - "'admin off' in _caddy" + - "'onramp.molecule.test {' in _caddy" + - "'tls internal' in _caddy" + - "'protocols h1 h2' in _caddy" + - "'email ops@molecule.test' in _caddy" + - "'reverse_proxy 127.0.0.1:8080' in _caddy" + - "'not the onramp' not in _caddy" + fail_msg: "{{ _caddy }}" + vars: + _caddy: "{{ _or_text.results[1].content | b64decode }}" + + - name: Read the onramp unit + ansible.builtin.command: + argv: [systemctl, cat, sponsord-onramp] + register: _or_unit + changed_when: false + + - name: Require credentials, hardening and the tie to the daemon + ansible.builtin.assert: + that: + # kics-scan ignore-line (a credential file PATH in the unit, not a secret) + - "'LoadCredential=api-token:/etc/sponsord/api-token' in _or_unit.stdout" + # kics-scan ignore-line (a credential file PATH in the unit, not a secret) + - "'LoadCredential=turnstile-secret:/etc/sponsord/turnstile-secret' in _or_unit.stdout" + - "'Environment=ONRAMP_DAEMON_TOKEN_FILE=%d/api-token' in _or_unit.stdout" + - "'Environment=ONRAMP_TURNSTILE_SECRET_FILE=%d/turnstile-secret' in _or_unit.stdout" + - "'DynamicUser=true' in _or_unit.stdout" + - "'ProtectSystem=strict' in _or_unit.stdout" + - "'CapabilityBoundingSet=\n' in _or_unit.stdout" + - "'PartOf=sponsord.service' in _or_unit.stdout" + fail_msg: "{{ _or_unit.stdout }}" + + - name: Read the TCP listeners + ansible.builtin.command: + argv: [ss, -Hltn] + register: _or_ss + changed_when: false + + - name: Require the onramp on loopback only, Caddy on 80 and 443 + ansible.builtin.assert: + that: + - _or_ss.stdout is search('127\.0\.0\.1:8080\s') + - _or_ss.stdout is not search('(0\.0\.0\.0|\*|\[::\]):8080\s') + - _or_ss.stdout is search('(\*|0\.0\.0\.0|\[::\]):443\s') + - _or_ss.stdout is search('(\*|0\.0\.0\.0|\[::\]):80\s') + # The admin API is off: nothing on Caddy's default 2019. + - _or_ss.stdout is not search(':2019\s') + fail_msg: "{{ _or_ss.stdout }}" + + - name: Reach the onramp through Caddy, as a user would + ansible.builtin.command: + argv: + - curl + - --silent + - --show-error + - --fail + - --insecure + - --resolve + - onramp.molecule.test:443:127.0.0.1 + - https://onramp.molecule.test/healthz + register: _or_https + changed_when: false + + - name: Require the https chain to answer + ansible.builtin.assert: + that: + - (_or_https.stdout | from_json).ok is true + fail_msg: "{{ _or_https.stdout }}" + + # The onramp rate-limits and gates by the right-most X-Forwarded-For address. + # A client that sends its own must not be able to choose it. + - name: Send a forged X-Forwarded-For through Caddy + ansible.builtin.command: + argv: + - curl + - --silent + - --show-error + - --fail + - --insecure + - --header + - "X-Forwarded-For: 203.0.113.9" + - --resolve + - onramp.molecule.test:443:127.0.0.1 + - https://onramp.molecule.test/molecule/client-ip + register: _or_xff + changed_when: false + + # Caddy without trusted_proxies REPLACES the header, so the forged address must + # be gone entirely, not just pushed left of the real one. + - name: Require the client address Caddy saw, and nothing else + ansible.builtin.assert: + that: + - (_or_xff.stdout | trim) == '127.0.0.1' + fail_msg: >- + The onramp saw X-Forwarded-For "{{ _or_xff.stdout }}": it must be exactly the + client Caddy saw (127.0.0.1), without the forged 203.0.113.9. + + - name: Read the Alloy config + ansible.builtin.slurp: + src: /etc/alloy/config.alloy + register: _or_alloy + + - name: Require Alloy to label the onramp's journal stream + ansible.builtin.assert: + that: + # The journal service_name rule, and the unit in the level selector. + - "'replacement = \"sponsord-onramp\"' in _cfg" + - "'(sponsord|sponsord-onramp)' in _cfg" + - "'prometheus.scrape \"sponsord\"' in _cfg" + - "'prometheus.scrape \"decdn_node\"' not in _cfg" + fail_msg: Alloy's config lacks the onramp's journal labels (or scrapes a node). + vars: + _cfg: "{{ _or_alloy.content | b64decode }}" diff --git a/ansible/molecule/sponsord/files/sponsord-stub b/ansible/molecule/sponsord/files/sponsord-stub index 5f41da7..f34538f 100755 --- a/ansible/molecule/sponsord/files/sponsord-stub +++ b/ansible/molecule/sponsord/files/sponsord-stub @@ -6,7 +6,8 @@ a release, a chain or a funded treasury: * `--version` -> print a version line and exit 0 (the role's version backstop). * no argument -> enforce the startup contract the real daemon enforces, then - serve /healthz and /metrics on SPONSORD_BIND until killed: + serve /healthz, /metrics and a bearer-checked /v1/info on + SPONSORD_BIND until killed: - SPONSORD_RPC_URL, _CHAIN_ID, _PAYMENT_POOL_ADDR, _POOL_ID set (_CHAIN_ID is stricter than upstream, which defaults it; the role always renders it); @@ -41,6 +42,11 @@ UNHEALTHY_MARKER = "/etc/sponsord/stub-unhealthy" EXPECTED_RPC_URL = "https://rpc.example.invalid/?key=$HOME$(touch /tmp/sponsord-rpc-expanded)&q=o'brien \"x\"" +# Set by check_startup; /v1/info answers only a caller presenting it, as the real +# daemon's authenticated routes do (the onramp reads /v1/info at start). +API_TOKEN = b"" + + def die(msg): print(f"sponsord-stub: {msg}", file=sys.stderr) sys.exit(1) @@ -69,7 +75,9 @@ def check_startup(): die(f"{var} is not set") if os.environ.get("SPONSORD_MOLECULE_RPC_LITERAL") == "1" and os.environ["SPONSORD_RPC_URL"] != EXPECTED_RPC_URL: die("SPONSORD_RPC_URL does not match the inventory literal (EnvironmentFile quoting broke it)") - if len(read_secret("SPONSORD_API_TOKEN_FILE")) < 32: + global API_TOKEN + API_TOKEN = read_secret("SPONSORD_API_TOKEN_FILE") + if len(API_TOKEN) < 32: die("API token is shorter than 32 bytes") read_secret("SPONSORD_TREASURY_PASSWORD_FILE") keystore = os.environ.get("SPONSORD_TREASURY_KEYSTORE") @@ -97,6 +105,11 @@ class Handler(BaseHTTPRequestHandler): body, ctype = b'{"ok":true}', "application/json" elif self.path == "/metrics": body, ctype = b"sponsord_pool_topups_total 0\n", "text/plain; version=0.0.4" + elif self.path == "/v1/info": + if self.headers.get("Authorization", "").encode() != b"Bearer " + API_TOKEN: + self.send_error(401) + return + body, ctype = b'{"stub":true}', "application/json" else: self.send_error(404) return diff --git a/ansible/molecule/validation/converge.yml b/ansible/molecule/validation/converge.yml index 948c41b..70d18c7 100644 --- a/ansible/molecule/validation/converge.yml +++ b/ansible/molecule/validation/converge.yml @@ -1,7 +1,8 @@ --- # NEGATIVE-path scenario: proves the role's fail-loud tuning-knob validation actually # REJECTS bad values (AGENTS.md hard-rule 4), so a regression that weakens an assert -# fails CI. Each case runs one role (decdn_node, grafana_alloy or sponsord) with a +# fails CI. Each case runs one role (decdn_node, grafana_alloy, sponsord or +# sponsord_onramp) with a # VALID required config plus ONE bad input, wrapped in block/rescue. The role aborts at its validation asserts # (which for the tuning-knob cases run before any host mutation), so those cases # don't interfere and no service starts. The env-file cases below are the @@ -23,8 +24,9 @@ # post-template `decdn config validate` gate still fails the deploy. It therefore # mutates the host, which is why this scenario has a prepare.yml staging wallet # material. It runs after the decdn_node assert-only cases so it cannot interfere -# with them; the grafana_alloy and sponsord cases follow it and clean up after -# themselves. +# with them; the grafana_alloy, sponsord and (in a second play) sponsord_onramp +# cases follow it and clean up after themselves (the onramp section notes its own +# host-state cases). - name: Converge (expect validation failures) hosts: all become: true @@ -2108,6 +2110,510 @@ - /etc/sponsord - /root/sponsord-secret-target +- name: Converge sponsord_onramp (expect validation failures) + hosts: all + become: true + # A second play only to stay under ansible-lint's per-play task cap. The tracker + # is a host fact, so it carries over from the play above. + vars: + # The daemon's network, which the onramp's follows. + sponsord_network: arbitrum-sepolia + # Valid sponsord_onramp baseline for the onramp-* cases. + sponsord_onramp_manual_bin_src: >- + {{ lookup('ansible.builtin.env', 'MOLECULE_PROJECT_DIRECTORY') + }}/molecule/sponsord-onramp/files/sponsord-onramp-stub + sponsord_onramp_rpc_url: https://sepolia-rollup.arbitrum.io/rpc + sponsord_onramp_domain: onramp.validation.test + sponsord_onramp_turnstile_sitekey: 1x00000000000000000000AA + sponsord_onramp_decdn_release: v0.1.0 + sponsord_onramp_decdn_sums_sha256: "1111111111111111111111111111111111111111111111111111111111111111" + sponsord_onramp_cli_release: decdn-sponsored-v0.1.0 + sponsord_onramp_cli_sums_sha256: "2222222222222222222222222222222222222222222222222222222222222222" + tasks: + # --- sponsord_onramp: each case breaks ONE input over the valid baseline in play + # vars. All but the last four abort at an assert before any host mutation; those + # stage /etc/sponsord (and /etc/caddy) files and remove them again. + - name: "Case onramp-release-version — release mode with no pinned version" + block: + - name: Run sponsord_onramp with release mode with no pinned version + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_install_method: release + sponsord_onramp_version: "" + rescue: + - name: Record onramp-release-version rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-release-version'] }}" + when: ansible_failed_task.name == "Require a pinned release version" + + - name: "Case onramp-manual-src — manual mode with no binary source" + block: + - name: Run sponsord_onramp with manual mode with no binary source + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_manual_bin_src: "" + sponsord_onramp_release_target_dir: "" + rescue: + - name: Record onramp-manual-src rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-manual-src'] }}" + when: ansible_failed_task.name == "Require a manual binary source" + + - name: "Case onramp-network-unknown — an unknown network profile" + block: + - name: Run sponsord_onramp with an unknown network profile + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_network: nope + rescue: + - name: Record onramp-network-unknown rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-network-unknown'] }}" + when: ansible_failed_task.name == "Require a known sponsord_onramp_network" + + - name: "Case onramp-network-mismatch — a network other than the daemon's" + block: + - name: Run sponsord_onramp with a network other than the daemon's + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_network: some-other-chain + sponsord_onramp_network: arbitrum-sepolia + rescue: + - name: Record onramp-network-mismatch rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-network-mismatch'] }}" + when: ansible_failed_task.name == "Require the onramp and the daemon to name the same network" + + - name: "Case onramp-zero-address — the zero CapacityBond address" + block: + - name: Run sponsord_onramp with the zero CapacityBond address + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_capacity_bond_address: "0x0000000000000000000000000000000000000000" + rescue: + - name: Record onramp-zero-address rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-zero-address'] }}" + when: ansible_failed_task.name == "Require the CapacityBond (and a well-formed SlashJudge) address" + + - name: "Case onramp-bind — a wildcard listener" + block: + - name: Run sponsord_onramp with a wildcard listener + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_bind_address: "0.0.0.0" + rescue: + - name: Record onramp-bind rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-bind'] }}" + when: ansible_failed_task.name == "Require the listener to stay on loopback" + + - name: "Case onramp-bind — a daemon URL off the host" + block: + - name: Run sponsord_onramp with a daemon URL off the host + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_daemon_url: "http://10.0.0.5:8090" + rescue: + - name: Record onramp-bind rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-bind'] }}" + when: ansible_failed_task.name == "Require the listener to stay on loopback" + + - name: "Case onramp-proxy — an unknown proxy" + block: + - name: Run sponsord_onramp with an unknown proxy + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_proxy: nginx + rescue: + - name: Record onramp-proxy rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-proxy'] }}" + when: ansible_failed_task.name == "Validate the proxy settings" + + - name: "Case onramp-domain — Caddy with no domain" + block: + - name: Run sponsord_onramp with Caddy with no domain + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_domain: "" + rescue: + - name: Record onramp-domain rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-domain'] }}" + when: ansible_failed_task.name == "Require a domain for Caddy" + + - name: "Case onramp-public-url — an http public URL behind your own proxy" + block: + - name: Run sponsord_onramp with an http public URL behind your own proxy + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_proxy: none + sponsord_onramp_public_url: "http://onramp.validation.test" + rescue: + - name: Record onramp-public-url rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-public-url'] }}" + when: ansible_failed_task.name == "Require an https public base URL the installers can hold" + + - name: "Case onramp-public-url — no public URL behind your own proxy" + block: + - name: Run sponsord_onramp with no public URL behind your own proxy + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_proxy: none + rescue: + - name: Record onramp-public-url rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-public-url'] }}" + when: ansible_failed_task.name == "Require an https public base URL the installers can hold" + + - name: "Case onramp-rpc — a public RPC URL with credentials" + block: + - name: Run sponsord_onramp with a public RPC URL with credentials + ansible.builtin.include_role: + name: sponsord_onramp + vars: + # kics-scan ignore-line (the bad input this case must reject, not a secret) + sponsord_onramp_rpc_url: "https://user:SECRET@rpc.example.invalid/" + rescue: + - name: Record onramp-rpc rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-rpc'] }}" + when: ansible_failed_task.name == "Require a public RPC URL with no credentials in it" + + - name: "Case onramp-rpc — a public RPC URL with an API key in its query" + block: + - name: Run sponsord_onramp with a public RPC URL with an API key in its query + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_rpc_url: "https://rpc.example.invalid/?apikey=SECRET" + rescue: + - name: Record onramp-rpc rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-rpc'] }}" + when: ansible_failed_task.name == "Require a public RPC URL with no credentials in it" + + - name: "Case onramp-rpc — a public RPC URL the installers cannot quote" + block: + - name: Run sponsord_onramp with a public RPC URL the installers cannot quote + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_rpc_url: "https://rpc.example.invalid/$(id)" + rescue: + - name: Record onramp-rpc rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-rpc'] }}" + when: ansible_failed_task.name == "Require a public RPC URL with no credentials in it" + + - name: "Case onramp-rpc — a public RPC URL with a single quote" + block: + - name: Run sponsord_onramp with a public RPC URL with a single quote + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_rpc_url: "https://rpc.example.invalid/o'brien" + rescue: + - name: Record onramp-rpc rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-rpc'] }}" + when: ansible_failed_task.name == "Require a public RPC URL with no credentials in it" + + - name: "Case onramp-public-url — a public URL with a backslash" + block: + - name: Run sponsord_onramp with a public URL with a backslash + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_proxy: none + sponsord_onramp_public_url: "https://onramp.validation.test/a\\b" + rescue: + - name: Record onramp-public-url rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-public-url'] }}" + when: ansible_failed_task.name == "Require an https public base URL the installers can hold" + + - name: "Case onramp-sitekey — a sitekey with a space" + block: + - name: Run sponsord_onramp with a sitekey with a space + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_turnstile_sitekey: "1x00 AA" + rescue: + - name: Record onramp-sitekey rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-sitekey'] }}" + when: ansible_failed_task.name == "Validate the Turnstile sitekey and gate settings" + + - name: "Case onramp-pin — a decdn release tag without its v" + block: + - name: Run sponsord_onramp with a decdn release tag without its v + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_decdn_release: "0.1.0" + rescue: + - name: Record onramp-pin rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-pin'] }}" + when: ansible_failed_task.name == "Validate the release pins the installers use" + + - name: "Case onramp-pin — an upper-case SHA256SUMS digest" + block: + - name: Run sponsord_onramp with an upper-case SHA256SUMS digest + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_cli_sums_sha256: "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + rescue: + - name: Record onramp-pin rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-pin'] }}" + when: ansible_failed_task.name == "Validate the release pins the installers use" + + - name: "Case onramp-limit — a zero TTL" + block: + - name: Run sponsord_onramp with a zero TTL + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_ttl_secs: 0 + rescue: + - name: Record onramp-limit rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-limit'] }}" + when: ansible_failed_task.name == "Validate the onramp's limits" + + - name: "Case onramp-ip-header — a header name with a space" + block: + - name: Run sponsord_onramp with a header name with a space + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_client_ip_header: "X Forwarded For" + rescue: + - name: Record onramp-ip-header rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-ip-header'] }}" + when: ansible_failed_task.name == "Validate the client IP header name" + + - name: "Case onramp-network-chain — a daemon chain id other than the onramp's profile" + block: + - name: Run sponsord_onramp with a daemon chain id other than the onramp's profile + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_chain_id: 1 + rescue: + - name: Record onramp-network-chain rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-network-chain'] }}" + when: ansible_failed_task.name == "Require the onramp's network to be the daemon's chain" + + - name: "Case onramp-proxy — a quoted overwrite confirmation" + block: + - name: Run sponsord_onramp with a quoted overwrite confirmation + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_caddy_overwrite_config: "false" + rescue: + - name: Record onramp-proxy rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-proxy'] }}" + when: ansible_failed_task.name == "Validate the proxy settings" + + - name: "Case onramp-proxy — a misspelt TLS mode" + block: + - name: Run sponsord_onramp with a misspelt TLS mode + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_caddy_tls: interal + rescue: + - name: Record onramp-proxy rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-proxy'] }}" + when: ansible_failed_task.name == "Validate the proxy settings" + + - name: "Case onramp-domain — an ACME email that would break the Caddyfile" + block: + - name: Run sponsord_onramp with an ACME email that would break the Caddyfile + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_acme_email: "ops}@example.org" + rescue: + - name: Record onramp-domain rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-domain'] }}" + when: ansible_failed_task.name == "Require a domain for Caddy" + + - name: "Case onramp-rpc — a public RPC URL that injects a second variable" + block: + - name: Run sponsord_onramp with a public RPC URL that injects a second variable + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_rpc_url: "https://rpc.example.invalid/a\nONRAMP_BIND=0.0.0.0:8080" + rescue: + - name: Record onramp-rpc rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-rpc'] }}" + when: ansible_failed_task.name == "Require a public RPC URL with no credentials in it" + + - name: "Case onramp-limit — a rate limit with a unit" + block: + - name: Run sponsord_onramp with a rate limit with a unit + ansible.builtin.include_role: + name: sponsord_onramp + vars: + sponsord_onramp_poll_rate_per_min: "10m" + rescue: + - name: Record onramp-limit rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-limit'] }}" + when: ansible_failed_task.name == "Validate the onramp's limits" + + - name: "Case onramp-token — no daemon API token on the host" + block: + - name: Run sponsord_onramp with no daemon API token on the host + ansible.builtin.include_role: + name: sponsord_onramp + rescue: + - name: Record onramp-token rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-token'] }}" + when: ansible_failed_task.name == "Require the daemon API token as a regular file" + + - name: "Case onramp-turnstile — no Turnstile secret on the host" + block: + - name: Stage the daemon's API token only + ansible.builtin.shell: + cmd: | + set -eu + mkdir -p /etc/sponsord + umask 077 + printf '%064d' 0 > /etc/sponsord/api-token + changed_when: true + + - name: Run sponsord_onramp with no Turnstile secret + ansible.builtin.include_role: + name: sponsord_onramp + rescue: + - name: Record onramp-turnstile rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-turnstile'] }}" + when: ansible_failed_task.name == "Require the Turnstile secret as a regular file" + always: + - name: Remove the onramp fixtures + ansible.builtin.file: + path: /etc/sponsord + state: absent + + # The lock-down must not chmod a symlink's target as root. + - name: "Case onramp-turnstile — the Turnstile secret is a symlink" + block: + - name: Stage the API token and a symlinked Turnstile secret + ansible.builtin.shell: + cmd: | + set -eu + mkdir -p /etc/sponsord + umask 077 + printf '%064d' 0 > /etc/sponsord/api-token + echo 'not-a-secret' > /root/onramp-turnstile-target + ln -sfn /root/onramp-turnstile-target /etc/sponsord/turnstile-secret + changed_when: true + + - name: Run sponsord_onramp against the symlinked Turnstile secret + ansible.builtin.include_role: + name: sponsord_onramp + rescue: + - name: Record onramp-turnstile symlink rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-turnstile'] }}" + when: ansible_failed_task.name == "Require the Turnstile secret as a regular file" + always: + - name: Remove the onramp symlink fixtures + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - /etc/sponsord + - /root/onramp-turnstile-target + + - name: "Case onramp-turnstile-empty — an empty Turnstile secret" + block: + - name: Stage the API token and an empty Turnstile secret + ansible.builtin.shell: + cmd: | + set -eu + mkdir -p /etc/sponsord + umask 077 + printf '%064d' 0 > /etc/sponsord/api-token + printf '\n' > /etc/sponsord/turnstile-secret + changed_when: true + + - name: Run sponsord_onramp with an empty Turnstile secret + ansible.builtin.include_role: + name: sponsord_onramp + rescue: + - name: Record onramp-turnstile-empty rejection (only if its check failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-turnstile-empty'] }}" + when: ansible_failed_task.name == "Report an empty Turnstile secret" + always: + - name: Remove the empty-secret fixtures + ansible.builtin.file: + path: /etc/sponsord + state: absent + + # Caddy run from a static binary leaves no dpkg trace: the file alone decides. + - name: "Case onramp-caddyfile — a foreign Caddyfile with no caddy package" + block: + - name: Stage the API token, the Turnstile secret and a foreign Caddyfile + ansible.builtin.shell: + cmd: | + set -eu + mkdir -p /etc/sponsord /etc/caddy + umask 077 + printf '%064d' 0 > /etc/sponsord/api-token + echo 'not-empty' > /etc/sponsord/turnstile-secret + printf 'other.example {\n respond "theirs"\n}\n' > /etc/caddy/Caddyfile + changed_when: true + + - name: Run sponsord_onramp against the foreign Caddyfile + ansible.builtin.include_role: + name: sponsord_onramp + rescue: + - name: Record onramp-caddyfile rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['onramp-caddyfile'] }}" + when: ansible_failed_task.name == "Refuse to replace a Caddyfile this role did not write" + always: + - name: Remove the foreign-Caddyfile fixtures + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - /etc/sponsord + - /etc/caddy + - name: Confirm every bad value was rejected by the role's own validation ansible.builtin.assert: that: @@ -2115,7 +2621,7 @@ # lets one broken sibling hide behind another case that uses the same gate. - decdn_rejected | sort == _decdn_expected | sort fail_msg: >- - decdn_node, grafana_alloy and sponsord did not reject every bad value at + decdn_node, grafana_alloy, sponsord and sponsord_onramp did not reject every bad value at their validation asserts. Expected {{ _decdn_expected | sort }}, got {{ decdn_rejected | sort }} — a missing tag means that bad value slipped past validation (it would reach @@ -2147,4 +2653,12 @@ "sponsord-dir-missing", "sponsord-dir-format", "sponsord-dir-cross", "sponsord-bind", "sponsord-overwrite-untracked", "sponsord-token", "sponsord-rpc-unusable", "sponsord-secret-env-keys", "sponsord-secret-env-keys", - "sponsord-secret-env-symlink"] + "sponsord-secret-env-symlink", + "onramp-release-version", "onramp-manual-src", "onramp-network-unknown", "onramp-network-mismatch", + "onramp-zero-address", "onramp-bind", "onramp-bind", "onramp-proxy", + "onramp-domain", "onramp-public-url", "onramp-public-url", "onramp-rpc", + "onramp-rpc", "onramp-rpc", "onramp-rpc", "onramp-public-url", "onramp-sitekey", "onramp-pin", + "onramp-pin", "onramp-limit", "onramp-ip-header", "onramp-token", + "onramp-turnstile", "onramp-turnstile", "onramp-turnstile-empty", + "onramp-network-chain", "onramp-proxy", "onramp-proxy", "onramp-domain", + "onramp-rpc", "onramp-limit", "onramp-caddyfile"] diff --git a/ansible/playbooks/group_vars/all.yml b/ansible/playbooks/group_vars/all.yml index e003d87..ac450b4 100644 --- a/ansible/playbooks/group_vars/all.yml +++ b/ansible/playbooks/group_vars/all.yml @@ -2,9 +2,10 @@ # Playbook-adjacent group_vars for every host (see decdn_nodes.yml beside this for # why playbook-adjacent: they load for any INVENTORY=, an overlay included). # -# Which daemons the opt-in Grafana Alloy agent scrapes, derived from the groups a -# host is in, so a sponsord-only host does not scrape a decdn-node /metrics that -# does not exist, and a co-located host gets both. Host-scoped on purpose: site.yml +# Which daemons the opt-in Grafana Alloy agent scrapes (the onramp has no /metrics: +# its toggle only labels its journal), derived from the groups a host is in, so a +# sponsord-only host does not scrape a decdn-node /metrics that does not exist, and +# a co-located host gets both. Host-scoped on purpose: site.yml # runs grafana_alloy once per play on a co-located host, and play vars would make # the two runs render different configs. # @@ -15,3 +16,24 @@ # host_vars, so override them there. grafana_alloy_node_enabled: "{{ 'decdn_nodes' in group_names }}" grafana_alloy_sponsord_enabled: "{{ 'sponsord_hosts' in group_names }}" +grafana_alloy_sponsord_onramp_enabled: "{{ 'sponsord_onramp_hosts' in group_names }}" + +# The public holes in the baseline firewall, from the groups a host is in. Each +# group that runs baseline (decdn_nodes.yml, sponsord_hosts.yml) sets +# baseline_extra_inbound to this, so every play on a host renders the same rules. +# Only genuinely public listeners belong here: metrics, admin RPC and the sponsord +# API stay on loopback and get no hole. +# decdn_nodes: the node's QUIC listener. decdn_bind_port is a decdn_node role +# default, which a play without that role (sponsord.yml) does not load; the +# fallback must match roles/decdn_node/defaults/main.yml. +# sponsord_onramp_hosts: Caddy, the onramp's TLS front (80 for the ACME challenge +# and the https redirect, 443), unless sponsord_onramp_proxy is "none" (your +# own proxy, which opens its own ports). Same fallback rule: "caddy" is +# roles/sponsord_onramp/defaults/main.yml's default. +_baseline_service_inbound: >- + {{ ([{'proto': 'udp', 'port': decdn_bind_port | default(4433), 'comment': 'decdn QUIC'}] + if 'decdn_nodes' in group_names else []) + + ([{'proto': 'tcp', 'port': 80, 'comment': 'sponsord-onramp (Caddy, http)'}, + {'proto': 'tcp', 'port': 443, 'comment': 'sponsord-onramp (Caddy, https)'}] + if 'sponsord_onramp_hosts' in group_names + and sponsord_onramp_proxy | default('caddy') == 'caddy' else []) }} diff --git a/ansible/playbooks/group_vars/decdn_nodes.yml b/ansible/playbooks/group_vars/decdn_nodes.yml index 68b95e8..591a7b9 100644 --- a/ansible/playbooks/group_vars/decdn_nodes.yml +++ b/ansible/playbooks/group_vars/decdn_nodes.yml @@ -9,9 +9,8 @@ # Precedence: playbook group_vars beat inventory group_vars, so to change this # for a node, override it in that node's host_vars. -# Open the node's public QUIC listener in the baseline firewall. Metrics (9090) -# and admin RPC (9191) stay loopback and get NO hole. -baseline_extra_inbound: - - proto: udp - port: "{{ decdn_bind_port }}" - comment: "decdn QUIC" +# The public holes in the baseline firewall: the node's QUIC listener, plus those of +# any other service the host also runs (all.yml builds the list from the host's +# groups). sponsord_hosts.yml sets the same value, so a host in both groups renders +# one firewall in both plays. +baseline_extra_inbound: "{{ _baseline_service_inbound }}" diff --git a/ansible/playbooks/group_vars/sponsord_hosts.yml b/ansible/playbooks/group_vars/sponsord_hosts.yml new file mode 100644 index 0000000..737c9f7 --- /dev/null +++ b/ansible/playbooks/group_vars/sponsord_hosts.yml @@ -0,0 +1,8 @@ +--- +# Playbook-adjacent group_vars for sponsord_hosts (see decdn_nodes.yml beside this +# for why playbook-adjacent, and for the precedence). +# +# The same list as decdn_nodes.yml, built in all.yml from the host's groups. It +# must be identical in both files: a host in both groups runs baseline once per +# play, and two different lists would flip its firewall on every run. +baseline_extra_inbound: "{{ _baseline_service_inbound }}" diff --git a/ansible/playbooks/sponsord.yml b/ansible/playbooks/sponsord.yml index befb125..a93c3b5 100644 --- a/ansible/playbooks/sponsord.yml +++ b/ansible/playbooks/sponsord.yml @@ -1,14 +1,36 @@ --- -# sponsord: the deCDN onboarding sponsor (treasury signer + PaymentPool keeper). -# baseline -> grafana_alloy (opt-in, off by default) -> sponsord -# Standalone entry point (`make deploy-sponsord`), also imported by site.yml. It -# needs no decdn-node on the host and opens no firewall hole: its only listener is -# loopback, for the onramp on the same host. A host in both decdn_nodes and -# sponsord_hosts runs baseline and Alloy once per play; both converge to the same -# state, because Alloy's per-daemon toggles are host-scoped -# (playbooks/group_vars/all.yml), not play-scoped. +# sponsord: the deCDN onboarding sponsor (treasury signer + PaymentPool keeper), +# and its public onramp. +# sponsord_hosts: baseline -> grafana_alloy (opt-in, off by default) -> sponsord +# sponsord_onramp_hosts: sponsord_onramp (+ Caddy in front, unless turned off) +# Standalone entry point (`make deploy-sponsord`), also imported by site.yml. The +# daemon needs no decdn-node on the host and listens on loopback only, for the +# onramp on the same host. The onramp listens on loopback too; Caddy is its public +# side (tcp/80 + tcp/443, opened by playbooks/group_vars/all.yml). A host in both +# decdn_nodes and sponsord_hosts runs baseline and Alloy once per play; both +# converge to the same state, because the firewall holes and Alloy's per-daemon +# toggles are host-scoped (playbooks/group_vars/), not play-scoped. # Opening and funding the treasury's pool is an operator action — see -# roles/sponsord/README.md. +# roles/sponsord/README.md; the Turnstile widget and the release pins are too — +# see roles/sponsord_onramp/README.md. +# Before anything runs: an onramp host outside sponsord_hosts would skip the play +# below (baseline hardening included) and get no firewall holes for Caddy, which +# playbooks/group_vars/ derive from sponsord_hosts. The role itself checks no group +# name, so the decdn.node collection does not impose this repo's groups on others. +# tests/scripts-test.sh runs this play by its name. +- name: Require every sponsord-onramp host to be a sponsord host + hosts: sponsord_onramp_hosts + gather_facts: false + tasks: + - name: Require the host in sponsord_hosts too + ansible.builtin.assert: + that: + - "'sponsord_hosts' in group_names" + fail_msg: >- + {{ inventory_hostname }} is in sponsord_onramp_hosts but not in sponsord_hosts. + The onramp runs beside the sponsord daemon and needs this playbook's baseline + and firewall holes, which come with sponsord_hosts: list the host there too. + - name: Provision sponsord hosts: sponsord_hosts become: true @@ -19,3 +41,13 @@ tags: [observability, sponsord] - role: sponsord tags: [sponsord] + +# After the daemon's play, whose /healthz gate is fatal, so the onramp starts +# against a daemon that is up. Baseline and Alloy already ran for these hosts +# above: the first play holds every one of them to sponsord_hosts. +- name: Provision sponsord-onramp + hosts: sponsord_onramp_hosts + become: true + roles: + - role: sponsord_onramp + tags: [sponsord_onramp, sponsord] diff --git a/ansible/roles/grafana_alloy/README.md b/ansible/roles/grafana_alloy/README.md index f8d02cc..789e893 100644 --- a/ansible/roles/grafana_alloy/README.md +++ b/ansible/roles/grafana_alloy/README.md @@ -198,7 +198,7 @@ Binary/package removal is manual. | Node `/metrics` | `decdn-node` | your own dashboards/queries (`grafana_alloy_node_enabled`) | | sponsord `/metrics` | `sponsord` | pool balance, top-ups, keeper failures, issued capabilities (`grafana_alloy_sponsord_enabled`); `service_name=sponsord` | | Machine metrics | `integrations/node_exporter` | Grafana Cloud's **Linux Server** integration dashboards + alerts, unmodified | -| journald | `integrations/node_exporter` | the same integration's logs dashboards (correlated by `instance`); the daemon's own unit (`decdn-node.service`) also carries `service_name=` (default `decdn-node`), and `sponsord.service` carries `service_name=sponsord` when that scrape is on | +| journald | `integrations/node_exporter` | the same integration's logs dashboards (correlated by `instance`); the daemon's own unit (`decdn-node.service`) also carries `service_name=` (default `decdn-node`), `sponsord.service` carries `service_name=sponsord` when that scrape is on, and `sponsord-onramp.service` carries `service_name=sponsord-onramp` when `grafana_alloy_sponsord_onramp_enabled` is | | Alloy self-metrics | `integrations/alloy` | agent health | | Traces | — | Application Observability | @@ -268,6 +268,9 @@ deliberately lean and every lever is a variable: `Caused by: …`, a panic) are labelled `error`. Other unlevelled lines keep journald's level. With the toggle off, nothing re-levels its stream, so an info-dropping priority regex would drop its warnings and errors too. +- sponsord-onramp, when `grafana_alloy_sponsord_onramp_enabled` is on, logs the same + plain-text format and is treated exactly like sponsord. It has no `/metrics`, so + that toggle adds no scrape. ## Hardening: the two relaxations machine monitoring requires @@ -303,7 +306,7 @@ upstream version/sha256). Highlights: | `grafana_alloy_host_metrics_enabled` | `true` | Machine metrics (Alloy's in-process `node_exporter`) | | `grafana_alloy_host_collectors` | 13 collectors | Replacement set; `_host_extra_collectors` adds, `_host_disable_collectors` vetoes | | `grafana_alloy_host_scrape_interval` | `60s` | Host metrics only; the node keeps `30s` | -| `grafana_alloy_host_systemd_unit_include` | `(decdn-node\|sponsord\|alloy\|ssh\|sshd)\.service` | Scope of the per-unit `systemd` collector | +| `grafana_alloy_host_systemd_unit_include` | `(decdn-node\|sponsord\|sponsord-onramp\|caddy\|alloy\|ssh\|sshd)\.service` | Scope of the per-unit `systemd` collector | | `grafana_alloy_logs_enabled` | `true` | journald → Grafana Cloud Loki | | `grafana_alloy_logs_max_age` | `12h` | Bounds the catch-up burst after an outage | | `grafana_alloy_self_metrics_enabled` | `true` | Alloy's own health | @@ -315,6 +318,8 @@ upstream version/sha256). Highlights: | `grafana_alloy_node_enabled` | `true` | Scrape decdn-node's `/metrics`. This repo's playbooks set it from membership of `decdn_nodes` (`playbooks/group_vars/all.yml`) | | `grafana_alloy_sponsord_enabled` | `false` | Scrape sponsord's `/metrics`, label and re-level its journald stream. Set from membership of `sponsord_hosts` by the same file | | `grafana_alloy_sponsord_job` / `_sponsord_service_name` | `sponsord` / `sponsord` | sponsord's job label and `service_name` | +| `grafana_alloy_sponsord_onramp_enabled` | `false` | Label and re-level sponsord-onramp's journald stream (no scrape: it has no `/metrics`). Set from membership of `sponsord_onramp_hosts` by the same file | +| `grafana_alloy_sponsord_onramp_service_name` | `sponsord-onramp` | The onramp stream's `service_name` | Label variables (`service_name`, `service_namespace`, `instance_id`, `deployment_environment`, `region`) ship on most series, spans and log lines — @@ -365,10 +370,11 @@ Three layers, because the first one cannot prove correctness on its own: container with no D-Bus and no real journal, so the `systemd` collector and journald collect nothing there — expected, and not what this layer proves. 2. **`make lint-alloy`** (CI job `alloy-config`) — renders these templates in - twelve variable combinations (defaults, minimal identity, fully overridden, + thirteen variable combinations (defaults, minimal identity, fully overridden, host-metrics-only, logs-only, every sub-knob off, inventory-supplied endpoints, OTLP username fallback, vetoed collector + blanked node job, - info-dropping guardrail, sponsord-only, node + sponsord co-located) and feeds them to the REAL pinned Alloy binary: `alloy validate` + info-dropping guardrail, sponsord-only, node + sponsord co-located, and the + same with the onramp) and feeds them to the REAL pinned Alloy binary: `alloy validate` (component graph, not just syntax), a check that every `ExecStart` flag exists in `alloy run --help`, and greps proving each Jinja branch actually switched — including that "every sub-knob off" reproduces the pre-machine-monitoring diff --git a/ansible/roles/grafana_alloy/defaults/main.yml b/ansible/roles/grafana_alloy/defaults/main.yml index 574d8e3..67e6435 100644 --- a/ansible/roles/grafana_alloy/defaults/main.yml +++ b/ansible/roles/grafana_alloy/defaults/main.yml @@ -149,6 +149,12 @@ grafana_alloy_node_enabled: true grafana_alloy_sponsord_enabled: false grafana_alloy_sponsord_job: sponsord grafana_alloy_sponsord_service_name: sponsord +# sponsord-onramp (roles/sponsord_onramp) has no /metrics: this labels its journald +# stream only, with a service_name and a `level` parsed from each line, exactly as +# for sponsord above. playbooks/group_vars/all.yml derives it from +# sponsord_onramp_hosts. +grafana_alloy_sponsord_onramp_enabled: false +grafana_alloy_sponsord_onramp_service_name: sponsord-onramp # --- Host (machine) metrics --------------------------------------------------- # Alloy's built-in node_exporter. Runs IN-PROCESS: no extra listener, no extra @@ -211,7 +217,7 @@ grafana_alloy_host_disk_device_exclude: '^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n # Per-unit state, deliberately scoped: one series set per unit on the box is a # real cost, and only these units matter. Units must match include AND NOT match # exclude. -grafana_alloy_host_systemd_unit_include: '(decdn-node|sponsord|alloy|ssh|sshd)\.service' +grafana_alloy_host_systemd_unit_include: '(decdn-node|sponsord|sponsord-onramp|caddy|alloy|ssh|sshd)\.service' grafana_alloy_host_systemd_unit_exclude: '.+\.(automount|device|mount|scope|slice)' # --- Agent self-monitoring ---------------------------------------------------- diff --git a/ansible/roles/grafana_alloy/tasks/preflight.yml b/ansible/roles/grafana_alloy/tasks/preflight.yml index 19fb7ec..b0d9b30 100644 --- a/ansible/roles/grafana_alloy/tasks/preflight.yml +++ b/ansible/roles/grafana_alloy/tasks/preflight.yml @@ -54,9 +54,11 @@ - grafana_alloy_host_include_exporter_metrics is boolean - grafana_alloy_node_enabled is boolean - grafana_alloy_sponsord_enabled is boolean + - grafana_alloy_sponsord_onramp_enabled is boolean fail_msg: >- grafana_alloy_host_metrics_enabled, _self_metrics_enabled, _logs_enabled, - _host_include_exporter_metrics, _node_enabled and _sponsord_enabled must be + _host_include_exporter_metrics, _node_enabled, _sponsord_enabled and + _sponsord_onramp_enabled must be booleans (true/false), not quoted strings — each gates whole component groups in config.alloy. @@ -157,11 +159,13 @@ resource attributes on most series/spans/log lines, so they are restricted to a low-cardinality charset (letters, digits, dot, underscore, slash, ≤ 63 chars): {{ item }} does not fit. Offending variables: - grafana_alloy_service_name / sponsord_service_name / service_namespace / + grafana_alloy_service_name / sponsord_service_name / + sponsord_onramp_service_name / service_namespace / instance_id / deployment_environment / region. loop: - "{{ grafana_alloy_service_name }}" - "{{ grafana_alloy_sponsord_service_name }}" + - "{{ grafana_alloy_sponsord_onramp_service_name }}" - "{{ grafana_alloy_service_namespace }}" # _ga_instance_id is REQUIRED (never "") — it has a derived non-empty default. - "{{ _ga_instance_id }}" diff --git a/ansible/roles/grafana_alloy/templates/config.alloy.j2 b/ansible/roles/grafana_alloy/templates/config.alloy.j2 index fc28c52..2a72ecc 100644 --- a/ansible/roles/grafana_alloy/templates/config.alloy.j2 +++ b/ansible/roles/grafana_alloy/templates/config.alloy.j2 @@ -301,15 +301,19 @@ loki.relabel "journal_rules" { source_labels = ["__journal__boot_id"] target_label = "boot_id" } +{# The units that log tracing-subscriber's plain text (the sponsord daemon and its + onramp), parsed for a level by loki.process "daemon_level" below. #} +{% set ga_text_units = (['sponsord'] if grafana_alloy_sponsord_enabled | bool else []) + + (['sponsord-onramp'] if grafana_alloy_sponsord_onramp_enabled | bool else []) %} {% if grafana_alloy_logs_drop_priority_regex | length > 0 %} // Cost guardrail: the regex is anchored to the WHOLE value by Prometheus // relabel semantics, so "debug" drops debug and nothing else. // - // decdn-node{% if grafana_alloy_sponsord_enabled | bool %} and sponsord are{% else %} is{% endif %} exempt: journald reports every line it writes as info, so + // decdn-node{% if ga_text_units %} and {{ ga_text_units | join(' and ') }} are{% else %} is{% endif %} exempt: journald reports every line it writes as info, so // judging it by priority would be judging it by a level it never set (a // regex containing "info" would drop its errors). loki.process - // "daemon_level" below applies the same regex to its real{% if not grafana_alloy_sponsord_enabled | bool %}, JSON{% endif %} level. The + // "daemon_level" below applies the same regex to its real{% if not ga_text_units %}, JSON{% endif %} level. The // exemption blanks a scratch copy of the keyword for that one unit; like // every __-prefixed label it never leaves this ruleset. rule { @@ -319,7 +323,7 @@ loki.relabel "journal_rules" { rule { source_labels = ["__journal__systemd_unit"] - regex = {{ ('(?:decdn-node|sponsord)\\.service' if grafana_alloy_sponsord_enabled | bool else 'decdn-node\\.service') | tojson }} + regex = {{ ('(?:' ~ (['decdn-node'] + ga_text_units) | join('|') ~ ')\\.service') | tojson }} target_label = "__priority_guardrail" replacement = "" } @@ -457,13 +461,14 @@ loki.process "daemon_level" { } {{ ga_daemon_level_stages() }} } -{% if grafana_alloy_sponsord_enabled | bool %} +{% if ga_text_units %} +{% set ga_text_selector = '{unit=~"(' ~ ga_text_units | join('|') ~ ')\\\\.service"}' %} - // sponsord logs tracing-subscriber's plain-text format with colour off - // (NO_COLOR=1 in its unit): " WARN : ". The + // {{ ga_text_units | join(' and ') }} log{% if ga_text_units | length == 1 %}s{% endif %} tracing-subscriber's plain-text format with colour off + // (NO_COLOR=1 in the units): " WARN : ". The // level is the second whitespace-separated field. stage.match { - selector = `{unit="sponsord.service"}` + selector = `{{ ga_text_selector }}` stage.regex { expression = `^\S+\s+(?PTRACE|DEBUG|INFO|WARN|ERROR)\s` @@ -471,12 +476,12 @@ loki.process "daemon_level" { {{ ga_daemon_level_stages() }} } - // What sponsord writes raw when it exits on an error carries no level field: + // What these write raw when they exit on an error carries no level field: // anyhow's "Error: ..." and "Caused by:" lines, and a panic message. They are // the lines that say WHY it stopped, so they are errors. Any other unlevelled // line keeps its journald level. stage.match { - selector = `{unit="sponsord.service"} |~ "^(Error: |Caused by:|thread '.+' panicked)"` + selector = `{{ ga_text_selector }} |~ "^(Error: |Caused by:|thread '.+' panicked)"` stage.template { source = "decdn_level" @@ -542,6 +547,17 @@ loki.relabel "journal_identity" { replacement = {{ grafana_alloy_sponsord_service_name | tojson }} } {% endif %} +{% if grafana_alloy_sponsord_onramp_enabled | bool and grafana_alloy_sponsord_onramp_service_name | length > 0 %} +{# The onramp's stream (the unit roles/sponsord_onramp installs). It has no + /metrics, so this and the daemon_level parsing above are all it gets. #} + + rule { + source_labels = ["unit"] + regex = "sponsord-onramp\\.service" + target_label = "service_name" + replacement = {{ grafana_alloy_sponsord_onramp_service_name | tojson }} + } +{% endif %} } loki.write "cloud" { diff --git a/ansible/roles/sponsord/README.md b/ansible/roles/sponsord/README.md index d7d544d..696d492 100644 --- a/ansible/roles/sponsord/README.md +++ b/ansible/roles/sponsord/README.md @@ -11,8 +11,9 @@ It is independent of the node. It needs an RPC endpoint and the PaymentPool cont never a local `decdn-node`, so it can run on its own host or beside a node. The role covers the daemon only. Its public companion, `sponsord-onramp` (Turnstile -gate, installers, behind a TLS reverse proxy), is not deployed here yet. The onramp -expects the daemon on the same host and reads the API token from +gate, installers, behind a TLS reverse proxy), is the +[`sponsord_onramp`](../sponsord_onramp/README.md) role: list the host in +`sponsord_onramp_hosts` too. It runs on the same host and reads the API token from `/etc/sponsord/api-token`. ## What it does @@ -29,7 +30,7 @@ expects the daemon on the same host and reads the API token from | File | Who provides it | |------|-----------------| - | `api-token` | The role generates it on the host when absent and never replaces it, because the onramp holds a copy. | + | `api-token` | The role generates it on the host when absent and never replaces it, because the onramp (or your own gate) uses it. | | `treasury-keystore.json`, `treasury-password` | **You** do. The role never creates the treasury wallet. | | `secret.env` (`SPONSORD_RPC_URL`) | `sponsord_rpc_url` from inventory, or a file you write on the host. The two-way rules match `decdn_rpc_url`: an empty `sponsord_rpc_url` with a file the role wrote earlier fails the deploy, because that means `secret.yml` went missing. A host-written file may hold `SPONSORD_RPC_URL` **only**: as an `EnvironmentFile` it would override every other setting, so the role refuses any other key in it. | @@ -112,8 +113,9 @@ yourself. ## Day 2 -- **Rotate the API token:** write the new token to `/etc/sponsord/api-token` and to - the onramp, then re-run the role, which restarts sponsord. +- **Rotate the API token:** write the new token to `/etc/sponsord/api-token`, then + re-run the role, which restarts sponsord. The onramp reads the same file and + restarts with the daemon (`PartOf=`). - **Rotate the treasury:** replace the keystore and password files, then re-run. The new wallet must own `sponsord_pool_id`, or sponsord refuses to start and the deploy fails at `/healthz`. diff --git a/ansible/roles/sponsord_onramp/README.md b/ansible/roles/sponsord_onramp/README.md new file mode 100644 index 0000000..5d90401 --- /dev/null +++ b/ansible/roles/sponsord_onramp/README.md @@ -0,0 +1,192 @@ +# Role: `sponsord_onramp` + +Deploys `sponsord-onramp` from [decdn/sponsord](https://github.com/decdn/sponsord), +the public side of the deCDN onboarding sponsor. It serves: + +- the gate page people pass (Cloudflare Turnstile) before the sponsor pays for + their download; +- the `decdn.sh` / `decdn.ps1` installers, which install `decdn` and + `decdn-sponsored` from pinned releases; +- the API the `decdn-sponsored` CLI polls for its capability. + +It holds no wallet. For each person who passes the gate it asks the +[`sponsord`](../sponsord/README.md) daemon on the same host for a capability, using +the daemon's API token. So the daemon must run on the same host: in this repo, list +the host in `sponsord_hosts` as well as `sponsord_onramp_hosts` (`playbooks/sponsord.yml` +refuses it otherwise). The role itself checks no group name: it fails early when +the daemon's token file is missing, and its `/healthz` gate fails when the daemon +is unreachable. + +## What it does + +- **Installs the binary**, exactly as the `sponsord` role does: + - `manual` (the default) copies a binary built in a `decdn/sponsord` checkout. + `sponsord_onramp_release_target_dir` defaults to `sponsord_release_target_dir`, + so one `cargo build --release` covers both. + - `release` downloads `sponsord-onramp-v` and verifies it against the + GPG-signed `SHA256SUMS`, with the KEYS file vendored in the `sponsord` role. + - Upstream has cut no release yet, so `manual` is the only working method today. +- **Reads two secrets, writes none.** Both stay root `0600` under `/etc/sponsord` + and reach the onramp as `LoadCredential=` credentials, never as environment: + + | File | Who provides it | + |------|-----------------| + | `api-token` | The `sponsord` role (generated on the host). The onramp uses the daemon's own file. | + | `turnstile-secret` | **You** do. The role only checks it is a non-empty regular file and locks it to `0600`. | + +- **Writes non-secret settings** to `/etc/sponsord/sponsord-onramp.env` (0644): the + public URL, the public RPC URL, CapacityBond and SlashJudge, the sitekey, the + release pins and any limits you set. +- **Runs a hardened unit.** `sponsord-onramp.service` runs with `DynamicUser=`, no + capabilities and a read-only filesystem. + - `PartOf=sponsord.service`: a daemon restart restarts the onramp, because the + onramp reads the daemon's limits only at start. + - It exits at start while the daemon is unreachable, so it retries every 5 s + with no start limit. +- **Gates the deploy on `/healthz`.** The onramp binds only after it has reached the + daemon and validated its settings (the release pins, its cap and TTL against the + daemon's maximum). If `/healthz` does not answer 200 within the readiness window, + the deploy fails. +- **Restarts on out-of-band changes.** As in the `sponsord` role, a hash record of + every restart input (the token, the Turnstile secret, the env file, the gate page, + the unit, the binary) restarts the onramp when one changes outside the role. +- **Puts Caddy in front** (`sponsord_onramp_proxy: caddy`, the default): + - Installs the distribution's `caddy` package (Debian main; on Ubuntu it is in + **universe**, which must be enabled). No third-party apt repository is added. + The package is not allowed to start Caddy on its stock config: Caddy first + starts once the role's Caddyfile has validated. + - Writes `/etc/caddy/Caddyfile`: TLS for `sponsord_onramp_domain`, then a reverse + proxy to the onramp on loopback. The file is validated with `caddy validate` + before it lands. The admin API is off, so nothing on the host can reconfigure + the running proxy over its admin endpoint (:2019), and HTTP/3 is off, so udp/443 + stays closed. + - Needs tcp/80 (ACME challenge, https redirect) and tcp/443 open. This repo's + playbooks open both (`playbooks/group_vars/all.yml`); from your own playbook, add + them to `baseline_extra_inbound` yourself. + - Probes the domain through Caddy on the host. Plain http must answer with Caddy's + https redirect, and `https:///healthz` must answer through TLS; both + failures are fatal. One exception: with ACME, a certificate that is not issued + yet (the TLS handshake fails, or the certificate cannot be verified) only warns, + since it can lag DNS. + - Refuses to replace a `/etc/caddy/Caddyfile` that it did not write and that is + not the caddy package's untouched default, unless you set + `sponsord_onramp_caddy_overwrite_config: true`. + +## Before the first deploy + +1. **Deploy `sponsord`** on the host (see [its README](../sponsord/README.md)). The + same `make deploy-sponsord` run can do both. +2. **Create a Turnstile widget** for your domain in the Cloudflare dashboard. Note + its sitekey (public) and secret. +3. **Copy the secret onto the host**, as root: + + ```bash + umask 077 + printf '%s' '' | sudo tee /etc/sponsord/turnstile-secret >/dev/null + sudo chmod 600 /etc/sponsord/turnstile-secret + ``` + +4. **Point DNS** for the domain (A/AAAA) at the host, so Caddy can get a + certificate. +5. **Pick the releases the installers install.** For `decdn`, a `vX.Y.Z` tag and the + SHA-256 of that release's `SHA256SUMS`. For `decdn-sponsored`, a + `decdn-sponsored-vX.Y.Z` tag and the same digest; both values are printed in its + release notes. +6. **Set inventory** (`group_vars/sponsord_onramp_hosts.yml`): the domain, the + public RPC URL, the sitekey and the four release-pin values. The contracts come + from `sponsord_network`. + +Then run `make deploy-sponsord` from `ansible/`. + +## Variables + +See [`defaults/main.yml`](defaults/main.yml) for the full list with comments. + +| Variable | Default | Notes | +|----------|---------|-------| +| `sponsord_onramp_install_method` | `manual` | `manual` or `release` | +| `sponsord_onramp_manual_bin_src` / `_release_target_dir` | `""` / `sponsord_release_target_dir` | `manual` only. | +| `sponsord_onramp_version` | `""` | Required in `release` mode, e.g. `0.1.0` → tag `sponsord-onramp-v0.1.0`. | +| `sponsord_onramp_network` | `sponsord_network` | Supplies CapacityBond and SlashJudge. Must match the daemon's network. | +| `sponsord_onramp_capacity_bond_address`, `_slash_judge_address` | from profile | Explicit values win. SlashJudge may be `""`. | +| `sponsord_onramp_rpc_url` | `""` | **Required. Public**: served to every user. No credentials, query or API key. | +| `sponsord_onramp_proxy` | `caddy` | `caddy` or `none`. Set it in inventory: the firewall holes read it. | +| `sponsord_onramp_domain` | `""` | Required with Caddy. | +| `sponsord_onramp_public_url` | `https://` | Required with `none`: the https URL your proxy serves. | +| `sponsord_onramp_caddy_tls` | `acme` | `internal` uses Caddy's local CA (staging, tests). | +| `sponsord_onramp_acme_email` | `""` | Optional ACME contact. | +| `sponsord_onramp_caddy_overwrite_config` | `false` | Let the role take over a Caddyfile it did not write. | +| `sponsord_onramp_client_ip_header` | `X-Forwarded-For` with Caddy, else `""` | With your own proxy, set it only if that proxy is the only way in. | +| `sponsord_onramp_turnstile_sitekey` | `""` | **Required.** | +| `sponsord_onramp_decdn_release`, `_decdn_sums_sha256` | `""` | **Required.** `vX.Y.Z` and 64 lowercase hex. | +| `sponsord_onramp_cli_release`, `_cli_sums_sha256` | `""` | **Required.** `decdn-sponsored-vX.Y.Z` and 64 lowercase hex. | +| `sponsord_onramp_min_cli_version` | `""` | Older `decdn-sponsored` CLIs are told to re-run the installer. | +| `sponsord_onramp_brand_name`, `_gate_template_src` | `""` | The gate page's name, or your own HTML page (a control-machine file). | +| `sponsord_onramp_spending_cap_micro_usdc`, `_ttl_secs` | `""` | Requested per capability. `""` takes the daemon's maximum; more than it fails the deploy. | +| `sponsord_onramp_fund_rate_per_min`, `_poll_rate_per_min` | `""` | Per-address rate limits (upstream 10 and 120; `0` turns one off). | +| `sponsord_onramp_bind_address` / `_port` | `127.0.0.1` / `8080` | IPv4 loopback only (asserted). | +| `sponsord_onramp_readiness_retries` / `_delay` | `30` / `2` | `/healthz` window (about 60 s). | + +## Bringing your own proxy + +With `sponsord_onramp_proxy: none` the role installs no Caddy and opens no port. +Your proxy must terminate TLS for `sponsord_onramp_public_url` and forward to +`127.0.0.1:8080`. Set `sponsord_onramp_client_ip_header` only if the proxy is the +only way to reach the onramp: `CF-Connecting-IP` behind Cloudflare, or +`X-Forwarded-For` if your proxy appends the client address. Otherwise clients can +choose their own address and get around the rate limits. Switching from `caddy` +to `none` closes the firewall holes but leaves the Caddy package installed and +running on 80/443; the role warns about it on every run until you remove it. + +## Observability + +With `decdn_grafana_cloud_enabled: true`, the `grafana_alloy` role ships the +onramp's journal with `service_name="sponsord-onramp"` and a `level` parsed from each +line (it has no `/metrics`). This repo's playbooks turn that on for hosts in +`sponsord_onramp_hosts`. From your own playbook, set +`grafana_alloy_sponsord_onramp_enabled`. Caddy's unit state is in the systemd +collector; its access logs are not shipped. + +## Day 2 + +- **Rotate the Turnstile secret:** write the new one to + `/etc/sponsord/turnstile-secret` and re-run the role. It restarts the onramp. +- **Ship a new CLI:** point `sponsord_onramp_cli_release` / `_cli_sums_sha256` (and + the `decdn` pair) at the new release and re-run. New installs get it; set + `sponsord_onramp_min_cli_version` to make existing users re-run the installer. +- **Certificate trouble:** `sudo journalctl -u caddy -e`. Check that the domain + resolves to the host and that tcp/80 and tcp/443 reach it. +- **Logs:** `journalctl -u sponsord-onramp`. +- **Removal:** + + ```bash + sudo systemctl disable --now sponsord-onramp caddy + sudo rm /etc/systemd/system/sponsord-onramp.service + sudo systemctl daemon-reload + sudo rm -f /usr/local/bin/sponsord-onramp /etc/sponsord/sponsord-onramp.env \ + /etc/sponsord/onramp-gate.html /etc/sponsord/.onramp-inputs.sha256 \ + /etc/sponsord/turnstile-secret + sudo rm -rf /usr/local/lib/sponsord-onramp + sudo apt-get remove caddy + ``` + + Then take the host out of `sponsord_onramp_hosts`, so the next deploy closes + tcp/80 and tcp/443. + +## Testing + +- `molecule/sponsord-onramp` runs `playbooks/sponsord.yml` (daemon, then onramp) on + Debian 12 and Ubuntu 24.04, against stub binaries and the real distro Caddy with + `tls internal`. It checks: + - idempotence; + - the `https://` chain through Caddy, and that a client-sent `X-Forwarded-For` + cannot choose the address the onramp sees; + - that a Turnstile rotation restarts the onramp and not the daemon; + - that the `/healthz` gate fails the deploy when the onramp will not start; + - the refusal of a foreign Caddyfile, and the confirmed takeover. +- `molecule/validation` holds the negative cases. +- `tests/firewall-holes` (`make test-scripts`) pins the tcp/80 and tcp/443 holes. +- `tests/alloy-config` (`make lint-alloy`) runs the onramp's log lines through the + real Alloy. +- Release mode is not exercised here. Its tasks mirror the `sponsord` role's, which + `molecule/sponsord` tests against a signed mirror. diff --git a/ansible/roles/sponsord_onramp/defaults/main.yml b/ansible/roles/sponsord_onramp/defaults/main.yml new file mode 100644 index 0000000..363338c --- /dev/null +++ b/ansible/roles/sponsord_onramp/defaults/main.yml @@ -0,0 +1,139 @@ +--- +# sponsord_onramp role defaults. sponsord-onramp is the public side of the deCDN +# onboarding sponsor: the gate page people pass (Cloudflare Turnstile), the +# decdn.sh / decdn.ps1 installers, and the API the decdn-sponsored CLI polls. It +# holds no wallet. It calls the sponsord daemon on the same host with the daemon's +# API token, so a host in sponsord_onramp_hosts must also be in sponsord_hosts. +# +# The onramp listens on loopback. Caddy, installed by this role unless you turn it +# off, terminates TLS for sponsord_onramp_domain and is the only public listener +# (AGENTS.md hard rule 2). Upstream: github.com/decdn/sponsord, docs/operator.md. + +# --- Install method ----------------------------------------------------------- +# "release": download the pinned sponsord-onramp-v GitHub Release tarball +# and verify it against the GPG-signed SHA256SUMS. "manual": copy a locally-built +# binary from the control machine. Upstream has cut NO sponsord-onramp-v* tag yet, +# so "manual" is the default. +sponsord_onramp_install_method: manual # "release" | "manual" +sponsord_onramp_manual_bin_src: "" # manual: control-machine path to the binary +# Alternative to sponsord_onramp_manual_bin_src: the control-machine `target/release` +# dir of a decdn/sponsord checkout (falls back to target//release). The role +# checks the binary is an ELF for the host's architecture before shipping it. +sponsord_onramp_release_target_dir: "{{ sponsord_release_target_dir | default('') }}" + +# --- Release to install (REQUIRED in "release" mode) --------------------------- +sponsord_onramp_version: "" # e.g. "0.1.0" -> tag sponsord-onramp-v0.1.0 +# Derived from the host's gathered architecture (vars/main/arch.yml). +sponsord_onramp_target: "{{ _sponsord_onramp_arch_targets.get(ansible_facts.get('architecture', ''), '') }}" +sponsord_onramp_release_base: "https://github.com/decdn/sponsord/releases/download" +sponsord_onramp_verify_release_signature: true +# Upstream signs every decdn/sponsord release with the keys in its one KEYS file, +# vendored once, in the sponsord role. Both roles ship side by side (this repo and +# the decdn.node collection), so the path holds in either. +sponsord_onramp_release_keyring: "{{ role_path }}/../sponsord/files/sponsord-release-KEYS.asc" + +# --- Chain ---------------------------------------------------------------------- +# The onramp takes the chain id and the PaymentPool from the daemon (/v1/info). It +# hands users' decdn the CapacityBond (node discovery) and, if set, the SlashJudge +# contract. With a network profile both come from upstream's deployment manifest, +# mirrored into vars/main/networks.yml by scripts/sync-network-profiles.py; an +# explicit inventory value still wins. Defaults to the daemon's sponsord_network. +sponsord_onramp_network: "{{ sponsord_network | default('') }}" +# Internal: set sponsord_onramp_network, not this. +_sponsord_onramp_profile: "{{ _sponsord_onramp_network_profiles.get(sponsord_onramp_network, {}) }}" +sponsord_onramp_capacity_bond_address: "{{ _sponsord_onramp_profile.get('capacity_bond_address', '') }}" +# "" omits it. +sponsord_onramp_slash_judge_address: "{{ _sponsord_onramp_profile.get('slash_judge_address', '') }}" + +# REQUIRED. The JSON-RPC endpoint users' decdn reads the chain through. NOT a +# secret, and it must not hold one: the onramp publishes it to every user +# (GET /v1/profile) and bakes it into the installers. Use a public endpoint, never +# the keyed URL the daemon uses (sponsord_rpc_url). +sponsord_onramp_rpc_url: "" + +# --- Listener ------------------------------------------------------------------- +# IPv4 loopback only (asserted): Caddy, or your own proxy, is the public side. +sponsord_onramp_bind_address: 127.0.0.1 +sponsord_onramp_port: 8080 +# The daemon, on this host. Follows the sponsord role's listener when inventory +# sets it; the fallbacks match roles/sponsord/defaults/main.yml. +sponsord_onramp_daemon_url: >- + http://{{ sponsord_bind_address | default('127.0.0.1') }}:{{ sponsord_port | default(8090) }} + +# --- Public address + TLS front ------------------------------------------------- +# "caddy": install Caddy from the distro and let it terminate TLS for +# sponsord_onramp_domain (ACME by default), opening tcp/80 and tcp/443. +# "none": bring your own proxy (nginx, a Cloudflare Tunnel, ...); no package, no +# firewall hole, and sponsord_onramp_public_url is required. +# playbooks/group_vars/all.yml reads this to open the ports, so set it in +# inventory, not in a play's vars. +sponsord_onramp_proxy: caddy # "caddy" | "none" +sponsord_onramp_domain: "" # caddy: REQUIRED, e.g. downloads.example.org +# The base URL users reach, baked into the installers and the gate links. https +# only. Derived from the domain in caddy mode. +sponsord_onramp_public_url: >- + {{ ('https://' ~ sponsord_onramp_domain) if sponsord_onramp_proxy == 'caddy' else '' }} +# "acme": a publicly trusted certificate (Let's Encrypt / ZeroSSL). The domain's +# A/AAAA records must point at this host and tcp/80 + tcp/443 must reach it. +# "internal": Caddy's own local CA, for staging and tests; browsers will not trust it. +sponsord_onramp_caddy_tls: acme # "acme" | "internal" +sponsord_onramp_acme_email: "" # optional ACME account contact +# The role owns /etc/caddy/Caddyfile. If that file exists without the role's marker +# and is not the package's untouched default, it belongs to someone else, and the +# role refuses to replace it unless this is true. +sponsord_onramp_caddy_overwrite_config: false +# The header Caddy (or your proxy) puts the client address in; the onramp rate +# limits and gates by it. Caddy (without trusted_proxies) sets X-Forwarded-For to +# the client it saw, discarding any client-sent value, and the onramp uses the +# right-most entry, so clients cannot choose their address. With your own +# proxy, set it only if that proxy is the ONLY way in (CF-Connecting-IP behind +# Cloudflare); otherwise clients pick their own address. "" = the TCP peer. +sponsord_onramp_client_ip_header: "{{ 'X-Forwarded-For' if sponsord_onramp_proxy == 'caddy' else '' }}" + +# --- Gate ----------------------------------------------------------------------- +# Cloudflare Turnstile: create a widget for sponsord_onramp_domain in the +# Cloudflare dashboard. The sitekey is public; the secret is provisioned on the +# host (0600 root) and reaches the onramp as a systemd credential. +sponsord_onramp_turnstile_sitekey: "" # REQUIRED +sponsord_onramp_brand_name: "" # [deCDN] name on the gate page +# A control-machine HTML file served as the gate page instead of the built-in one +# (placeholders: upstream docs/operator.md). "" = the built-in page. +sponsord_onramp_gate_template_src: "" + +# --- Releases the installers install (REQUIRED) ------------------------------- +# A tag and the SHA-256 of that release's SHA256SUMS file. Both values for +# decdn-sponsored are printed in its release notes (upstream RELEASING.md). +sponsord_onramp_decdn_release: "" # decdn/decdn tag, e.g. v0.1.0 +sponsord_onramp_decdn_sums_sha256: "" # 64 lowercase hex +sponsord_onramp_cli_release: "" # e.g. decdn-sponsored-v0.1.0 +sponsord_onramp_cli_sums_sha256: "" # 64 lowercase hex +# Oldest decdn-sponsored accepted; older CLIs are told to re-run the installer. +sponsord_onramp_min_cli_version: "" +sponsord_onramp_releases_base: "" # [https://github.com/decdn] + +# --- Limits ("" = the onramp's own default, shown in brackets) ----------------- +sponsord_onramp_spending_cap_micro_usdc: "" # [the daemon's maximum] cap per capability +sponsord_onramp_ttl_secs: "" # [the daemon's maximum] TTL per capability +sponsord_onramp_fund_rate_per_min: "" # [10] POST /v1/fund per address; 0 = off +sponsord_onramp_poll_rate_per_min: "" # [120] GET /v1/capability per address; 0 = off + +# --- Files on the host ---------------------------------------------------------- +# Beside the daemon's files: the directory and the API token follow the sponsord +# role's sponsord_etc / sponsord_api_token_file when inventory sets them (the +# fallbacks match roles/sponsord/defaults/main.yml). +sponsord_onramp_etc: "{{ sponsord_etc | default('/etc/sponsord') }}" +sponsord_onramp_api_token_file: "{{ sponsord_api_token_file | default(sponsord_onramp_etc ~ '/api-token') }}" +sponsord_onramp_turnstile_secret_file: "{{ sponsord_onramp_etc }}/turnstile-secret" # operator-provisioned (0600) +sponsord_onramp_env_file: "{{ sponsord_onramp_etc }}/sponsord-onramp.env" # non-secret (0644, role-owned) +sponsord_onramp_gate_template_file: "{{ sponsord_onramp_etc }}/onramp-gate.html" # when _src is set (0644) +# sha256 record of every restart input (token, Turnstile secret, env file, gate +# page, unit, binary) as of the last converge that saw the onramp healthy. +sponsord_onramp_inputs_checksum_file: "{{ sponsord_onramp_etc }}/.onramp-inputs.sha256" + +sponsord_onramp_bin: /usr/local/bin/sponsord-onramp + +# --- Readiness -------------------------------------------------------------------- +# The onramp binds only after reading the daemon's /v1/info, so /healthz answering +# 200 is the config gate. A timeout FAILS the deploy. Window ~60s. +sponsord_onramp_readiness_retries: 30 +sponsord_onramp_readiness_delay: 2 diff --git a/ansible/roles/sponsord_onramp/handlers/main.yml b/ansible/roles/sponsord_onramp/handlers/main.yml new file mode 100644 index 0000000..604b04c --- /dev/null +++ b/ansible/roles/sponsord_onramp/handlers/main.yml @@ -0,0 +1,19 @@ +--- +# Each handler reloads systemd itself rather than relying on a separate "Reload +# systemd" handler running first (see roles/sponsord/handlers/main.yml). +- name: Restart sponsord-onramp + ansible.builtin.systemd: + name: sponsord-onramp + state: restarted + daemon_reload: true + +# Not a reload: the role's Caddyfile turns the admin API off, which `caddy reload` +# (the packaged unit's ExecReload) needs. +# Under --check on a host without Caddy its install is only simulated, so there +# is no unit to restart (tasks/caddy.yml guards its start the same way). +- name: Restart caddy + ansible.builtin.systemd: + name: caddy + state: restarted + daemon_reload: true + when: not ansible_check_mode or 'caddy' in ansible_facts.packages diff --git a/ansible/roles/sponsord_onramp/meta/main.yml b/ansible/roles/sponsord_onramp/meta/main.yml new file mode 100644 index 0000000..bbd70a3 --- /dev/null +++ b/ansible/roles/sponsord_onramp/meta/main.yml @@ -0,0 +1,17 @@ +--- +galaxy_info: + role_name: sponsord_onramp + author: deCDN Contributors + description: >- + deCDN sponsord-onramp, the public gate in front of sponsord — release-tarball or + manual local-binary install, hardened DynamicUser systemd unit with credentials, + loopback listener behind an optional Caddy TLS front. + license: MIT + min_ansible_version: "2.15" + galaxy_tags: [decdn, sponsord, caddy, systemd, web3, debian] + platforms: + - name: Debian + versions: [bookworm, trixie] + - name: Ubuntu + versions: [noble, resolute] +dependencies: [] diff --git a/ansible/roles/sponsord_onramp/tasks/caddy.yml b/ansible/roles/sponsord_onramp/tasks/caddy.yml new file mode 100644 index 0000000..e09f310 --- /dev/null +++ b/ansible/roles/sponsord_onramp/tasks/caddy.yml @@ -0,0 +1,109 @@ +--- +# Caddy in front of the onramp: the host's only public HTTP listener (tcp/80 for +# the ACME challenge and the https redirect, tcp/443). This repo's +# playbooks/group_vars/all.yml opens both in the baseline firewall for hosts in +# sponsord_onramp_hosts while sponsord_onramp_proxy is "caddy". The distro package +# brings its own unit (user caddy, CAP_NET_BIND_SERVICE as its only ambient +# capability, PrivateTmp, ProtectSystem=full); main.yml has already refused a +# Caddyfile this role does not own. + +# curl: the probe at the end of this file. policy_rc_d 101 stops the package's +# maintainer scripts from starting Caddy on its stock config: the firewall already +# has 80/443 open, so a run that failed after the install would leave that default +# server public. Caddy first starts from the Restart caddy handler, once the +# managed Caddyfile below has validated. +- name: Install Caddy from the distribution + ansible.builtin.apt: + name: [caddy, curl] + state: present + update_cache: true + cache_valid_time: 3600 + policy_rc_d: 101 + +# `caddy validate` loads the config the way `caddy run` does (it also provisions +# the TLS automation), so a Caddyfile that does not parse or provision never lands. +- name: Template the Caddyfile + ansible.builtin.template: + src: Caddyfile.j2 + dest: "{{ _sponsord_onramp_caddyfile }}" + owner: root + group: root + mode: "0644" + validate: caddy validate --adapter caddyfile --config %s + notify: Restart caddy + +- name: Apply pending Caddy changes + ansible.builtin.meta: flush_handlers + +# Under --check on a host without Caddy the install above is only simulated, so +# there is no unit to start yet. +- name: Enable and start Caddy + ansible.builtin.systemd: + name: caddy + enabled: true + state: started + when: not ansible_check_mode or 'caddy' in ansible_facts.packages + +# Caddy serves the site at all: plain http answers with its https redirect. No +# certificate is involved, so this holds from the first run, ACME or not. +- name: Probe Caddy's http redirect for the domain + ansible.builtin.uri: + url: http://127.0.0.1/healthz + headers: + Host: "{{ sponsord_onramp_domain }}" + follow_redirects: none + status_code: 308 + timeout: 5 + register: sponsord_onramp_caddy_redirect + retries: "{{ sponsord_onramp_readiness_retries }}" + delay: "{{ sponsord_onramp_readiness_delay }}" + until: sponsord_onramp_caddy_redirect.status | default(-1) == 308 + changed_when: false + when: not ansible_check_mode + +# The whole chain, as a user would reach it but pinned to this host: TLS for the +# domain, the proxy, the onramp. Fatal, except that with ACME a certificate that is +# not issued yet (DNS not pointed here, a rate limit) only warns: Caddy then aborts +# the handshake (curl 35) or serves one curl cannot verify (60). Any other failure +# (a 502 from a dead upstream, nothing on 443, a hang) is a fault either way. +- name: Probe the onramp through Caddy + ansible.builtin.command: + argv: >- + {{ ['curl', '--silent', '--show-error', '--fail', '--max-time', '5', + '--retry', sponsord_onramp_readiness_retries | string, + '--retry-delay', sponsord_onramp_readiness_delay | string, + '--retry-max-time', '60', '--retry-all-errors', + '--resolve', sponsord_onramp_domain ~ ':443:127.0.0.1'] + + (['--insecure'] if sponsord_onramp_caddy_tls == 'internal' else []) + + ['https://' ~ sponsord_onramp_domain ~ '/healthz'] }} + register: sponsord_onramp_caddy_probe + changed_when: false + failed_when: false + when: not ansible_check_mode + +- name: Report an onramp that is not reachable through Caddy + ansible.builtin.fail: + msg: |- + https://{{ sponsord_onramp_domain }}/healthz did not answer through Caddy on this + host (curl exit {{ sponsord_onramp_caddy_probe.rc }}: {{ sponsord_onramp_caddy_probe.stderr | trim }}). + The onramp itself is healthy on loopback, so the fault is in Caddy: + sudo journalctl -u caddy -e + when: + - not ansible_check_mode + - sponsord_onramp_caddy_probe.rc != 0 + - sponsord_onramp_caddy_tls == 'internal' or sponsord_onramp_caddy_probe.rc not in [35, 60] + +- name: Warn that the public certificate is not in place yet + ansible.builtin.debug: + msg: |- + WARNING: https://{{ sponsord_onramp_domain }} has no trusted certificate yet + (curl exit {{ sponsord_onramp_caddy_probe.rc }}: {{ sponsord_onramp_caddy_probe.stderr | trim }}). + Caddy serves the site (its http redirect answered), so ACME has not finished: + check that the domain's A/AAAA records point at this host and that tcp/80 and + tcp/443 reach it, then read + sudo journalctl -u caddy -e + Users cannot reach the onramp until this answers. + when: + - not ansible_check_mode + - sponsord_onramp_caddy_tls == 'acme' + - sponsord_onramp_caddy_probe.rc in [35, 60] diff --git a/ansible/roles/sponsord_onramp/tasks/install.yml b/ansible/roles/sponsord_onramp/tasks/install.yml new file mode 100644 index 0000000..ffb8ddc --- /dev/null +++ b/ansible/roles/sponsord_onramp/tasks/install.yml @@ -0,0 +1,257 @@ +--- +# Install the sponsord-onramp binary. Two methods, selected by sponsord_onramp_install_method: +# "release": download the pinned sponsord-onramp-v GitHub Release tarball, +# verify it against the GPG-signed SHA256SUMS manifest, then extract. +# Idempotent and upgrade-aware via a version stamp. +# "manual" (default today): copy a locally-built binary from the control machine. +# The verification steps mirror roles/decdn_node/tasks/install.yml, which carries +# the full rationale for each (status-parsed gpg, exact manifest entry, private +# staging dir kept on failure). + +- name: Install from the pinned GitHub Release tarball + when: sponsord_onramp_install_method == "release" + block: + - name: Name the release archive + ansible.builtin.set_fact: + # The archive holds ONE flat binary at its root (upstream release.yml). + sponsord_onramp_release_archive: >- + sponsord-onramp-{{ sponsord_onramp_version }}-{{ sponsord_onramp_target }}.tar.gz + sponsord_onramp_release_url: "{{ sponsord_onramp_release_base }}/sponsord-onramp-v{{ sponsord_onramp_version }}" + + - name: Read the installed-version stamp + ansible.builtin.slurp: + src: /usr/local/lib/sponsord-onramp/installed-version + register: sponsord_onramp_stamp + failed_when: false + changed_when: false + + - name: Decide whether (re)install is needed + ansible.builtin.set_fact: + sponsord_onramp_need_install: >- + {{ (sponsord_onramp_stamp.content is not defined) + or ((sponsord_onramp_stamp.content | b64decode | trim) != sponsord_onramp_version) }} + + - name: Report that the release download is not simulated in check mode + ansible.builtin.debug: + msg: >- + check mode: skipping the download + signature verification of sponsord-onramp + {{ sponsord_onramp_version }}. The config and unit below still diff normally. + when: + - sponsord_onramp_need_install + - ansible_check_mode + + - name: Install sponsord-onramp {{ sponsord_onramp_version }} + when: + - sponsord_onramp_need_install + - not ansible_check_mode + block: + - name: Create a private download staging directory + ansible.builtin.tempfile: + state: directory + prefix: sponsord-onramp-install- + register: sponsord_onramp_stage + changed_when: false + + - name: Restrict the staging directory to root + ansible.builtin.file: + path: "{{ sponsord_onramp_stage.path }}" + state: directory + owner: root + group: root + mode: "0700" + + - name: WARN that release signature verification is DISABLED + ansible.builtin.debug: + msg: |- + sponsord_onramp_verify_release_signature is false. SHA256SUMS is fetched + UNAUTHENTICATED from {{ sponsord_onramp_release_url }}, so the checksum check + proves only that the download was not corrupted — NOT that the binary + came from deCDN. This binary holds the daemon API token. + when: not sponsord_onramp_verify_release_signature + + - name: Ensure gnupg is available for signature verification + ansible.builtin.package: + name: gnupg + state: present + when: sponsord_onramp_verify_release_signature + + - name: Download the release tarball + ansible.builtin.get_url: + url: "{{ sponsord_onramp_release_url }}/{{ sponsord_onramp_release_archive }}" + dest: "{{ sponsord_onramp_stage.path }}/{{ sponsord_onramp_release_archive }}" + owner: root + group: root + mode: "0600" + + - name: Download the SHA256SUMS manifest and signature + ansible.builtin.get_url: + url: "{{ sponsord_onramp_release_url }}/{{ item }}" + dest: "{{ sponsord_onramp_stage.path }}/{{ item }}" + owner: root + group: root + mode: "0600" + loop: "{{ ['SHA256SUMS', 'SHA256SUMS.asc'] if sponsord_onramp_verify_release_signature else ['SHA256SUMS'] }}" + + - name: Create the keyring staging directory + ansible.builtin.file: + path: "{{ sponsord_onramp_stage.path }}/keyring" + state: directory + owner: root + group: root + mode: "0700" + when: sponsord_onramp_verify_release_signature + + - name: Stage the release signing keyring + ansible.builtin.copy: + src: "{{ sponsord_onramp_release_keyring }}" + dest: "{{ sponsord_onramp_stage.path }}/keyring/KEYS.asc" + owner: root + group: root + mode: "0600" + when: sponsord_onramp_verify_release_signature + + # Isolated GNUPGHOME; VALIDSIG required, an expired/revoked key refused, + # --no-autostart so no gpg-agent outlives the task (see decdn_node). + - name: Verify the SHA256SUMS signature against the release keyring + ansible.builtin.shell: + chdir: "{{ sponsord_onramp_stage.path }}" + executable: /bin/bash + cmd: | + set -euo pipefail + export GNUPGHOME="{{ sponsord_onramp_stage.path }}/gnupg" + mkdir -p "$GNUPGHOME" + chmod 700 "$GNUPGHOME" + gpg --batch --quiet --no-autostart --import keyring/KEYS.asc + gpg --batch --no-autostart --status-fd 3 --verify SHA256SUMS.asc SHA256SUMS \ + 3>gpg-status.txt + if grep -qE '^\[GNUPG:\] (EXPKEYSIG|REVKEYSIG|EXPSIG)' gpg-status.txt; then + echo "SHA256SUMS carries a good signature from an EXPIRED or REVOKED" >&2 + echo "release key. Refusing to install. Re-sync" >&2 + echo "roles/sponsord/files/sponsord-release-KEYS.asc from upstream KEYS," >&2 + echo "or verify this release out of band." >&2 + exit 1 + fi + grep -q '^\[GNUPG:\] VALIDSIG ' gpg-status.txt + register: sponsord_onramp_sig_check + changed_when: false + failed_when: sponsord_onramp_sig_check.rc != 0 + when: sponsord_onramp_verify_release_signature + + - name: Verify the tarball against SHA256SUMS + ansible.builtin.shell: + chdir: "{{ sponsord_onramp_stage.path }}" + executable: /bin/bash + cmd: | + set -euo pipefail + f={{ sponsord_onramp_release_archive | quote }} + entry=$(awk -v f="$f" 'substr($0, index($0, " ") + 2) == f' SHA256SUMS) + n=$(printf '%s\n' "$entry" | grep -c . || true) + if [ "$n" -ne 1 ]; then + echo "SHA256SUMS has $n entries for $f (expected exactly 1)." >&2 + exit 1 + fi + printf '%s\n' "$entry" | sha256sum --check --strict - + register: sponsord_onramp_sum_check + changed_when: false + failed_when: sponsord_onramp_sum_check.rc != 0 + + - name: Extract the binary into /usr/local/bin + ansible.builtin.unarchive: + src: "{{ sponsord_onramp_stage.path }}/{{ sponsord_onramp_release_archive }}" + dest: "{{ sponsord_onramp_bin | dirname }}" + remote_src: true + owner: root + group: root + mode: "0755" + notify: Restart sponsord-onramp + + always: + - name: Report the retained staging directory after a failure + ansible.builtin.debug: + msg: >- + Verification failed; leaving {{ sponsord_onramp_stage.path }} in place for + inspection (gpg-status.txt, SHA256SUMS, SHA256SUMS.asc, the archive). + Remove it by hand once done. + when: + - sponsord_onramp_stage.path is defined + - sponsord_onramp_sig_check.failed | default(false) or sponsord_onramp_sum_check.failed | default(false) + + - name: Remove the download staging directory + ansible.builtin.file: + path: "{{ sponsord_onramp_stage.path }}" + state: absent + when: + - sponsord_onramp_stage.path is defined + - not (sponsord_onramp_sig_check.failed | default(false) + or sponsord_onramp_sum_check.failed | default(false)) + +- name: Install a locally-built binary from the control machine + when: sponsord_onramp_install_method == "manual" + block: + - name: Copy the sponsord-onramp binary from the control machine + ansible.builtin.copy: + src: "{{ _sponsord_onramp_bin_src }}" + dest: "{{ sponsord_onramp_bin }}" + owner: root + group: root + mode: "0755" + notify: Restart sponsord-onramp + + # A leftover stamp would make a later release deploy of the SAME version skip + # the download, keeping this manual binary. + - name: Clear the release version stamp in manual mode + ansible.builtin.file: + path: /usr/local/lib/sponsord-onramp/installed-version + state: absent + +# Runs every converge, so a binary that drifted from the pin is caught. With +# sponsord_onramp_version empty (the usual manual case) it degrades to a liveness check. +# An exact match on clap's `sponsord-onramp ` line, so a 0.1.0 pin does not +# accept 10.1.0 or 0.1.0-rc.1. +- name: Run the installed sponsord-onramp --version (version backstop) + ansible.builtin.command: "{{ sponsord_onramp_bin }} --version" + register: sponsord_onramp_installed_version + changed_when: false + failed_when: false + # The binary only exists once the copy/extract above really ran. + when: not ansible_check_mode + +- name: Require the installed sponsord-onramp to run and match the pinned version + ansible.builtin.assert: + that: + - sponsord_onramp_installed_version.rc == 0 + - >- + sponsord_onramp_version | length == 0 + or (sponsord_onramp_installed_version.stdout | trim) == 'sponsord-onramp ' ~ sponsord_onramp_version + fail_msg: >- + {{ sponsord_onramp_bin }} --version exited {{ sponsord_onramp_installed_version.rc }} with + "{{ sponsord_onramp_installed_version.stdout | trim }}"{{ + (', expected "sponsord-onramp ' ~ sponsord_onramp_version ~ '"') + if sponsord_onramp_version | length > 0 else '' }}. + The binary on the host is broken or is not the pinned release. In release mode, + remove /usr/local/lib/sponsord-onramp/installed-version to force a fresh download. + when: not ansible_check_mode + +# Written only once the version check passed, so a tarball that installed a wrong +# or broken binary is fetched again next run instead of being trusted via the stamp. +- name: Record the installed release version + when: + - sponsord_onramp_install_method == "release" + - sponsord_onramp_need_install | default(false) + - not ansible_check_mode + block: + - name: Ensure the stamp directory exists + ansible.builtin.file: + path: /usr/local/lib/sponsord-onramp + state: directory + owner: root + group: root + mode: "0755" + + - name: Record the installed version + ansible.builtin.copy: + dest: /usr/local/lib/sponsord-onramp/installed-version + content: "{{ sponsord_onramp_version }}\n" + owner: root + group: root + mode: "0644" diff --git a/ansible/roles/sponsord_onramp/tasks/main.yml b/ansible/roles/sponsord_onramp/tasks/main.yml new file mode 100644 index 0000000..d14841f --- /dev/null +++ b/ansible/roles/sponsord_onramp/tasks/main.yml @@ -0,0 +1,695 @@ +--- +# Provision sponsord-onramp: validate, gate on the host's secrets, install, +# configure, start, gate on /healthz, then put Caddy in front. Every pure-input +# assert runs before the first host mutation, and every gate (host files included) +# before the binary is installed. The Turnstile widget and the release pins are +# operator inputs — see roles/sponsord_onramp/README.md. + +# The onramp needs the sponsord daemon on the same host: it reads the daemon's API +# token file and calls it on loopback. Nothing here names an inventory group; the +# token gate below and the /healthz gate (the onramp binds only after reaching the +# daemon) are what prove the daemon is there. + +# --- Install method + target ---------------------------------------------------- +- name: Require a valid install method + ansible.builtin.assert: + that: + - sponsord_onramp_install_method in ["release", "manual"] + fail_msg: >- + sponsord_onramp_install_method must be "release" (pinned GitHub Release + tarball) or "manual" (copy a locally-built binary from the control machine). + +- name: Require a pinned release version + ansible.builtin.assert: + that: + - sponsord_onramp_version | length > 0 + fail_msg: >- + sponsord_onramp_install_method "release" requires sponsord_onramp_version (e.g. + "0.1.0") and a matching sponsord-onramp-v GitHub Release. Until upstream + cuts one, build the binary from a decdn/sponsord checkout and use + sponsord_onramp_install_method "manual". + when: sponsord_onramp_install_method == "release" + +- name: Validate sponsord_onramp_verify_release_signature is a real boolean + ansible.builtin.assert: + that: + - sponsord_onramp_verify_release_signature is boolean + fail_msg: >- + sponsord_onramp_verify_release_signature must be a boolean (true/false). Leave it + true unless you install from a mirror that strips SHA256SUMS.asc. + when: sponsord_onramp_install_method == "release" + +- name: Resolve the sponsord-onramp target triple + when: >- + sponsord_onramp_install_method == "release" + or (sponsord_onramp_release_target_dir | length > 0 and sponsord_onramp_manual_bin_src | length == 0) + block: + - name: Require a supported target triple + ansible.builtin.assert: + that: + - sponsord_onramp_target in _sponsord_onramp_arch_targets.values() + fail_msg: >- + sponsord_onramp_target resolved to "{{ sponsord_onramp_target }}", which is not + one of the Linux targets upstream releases + ({{ _sponsord_onramp_arch_targets.values() | join(', ') }}). It is derived from + the host's gathered architecture + ("{{ ansible_facts.get('architecture', 'not gathered') }}"): gather facts or set + sponsord_onramp_target explicitly. + + - name: Require the target triple to match the host architecture + ansible.builtin.assert: + that: + - _sponsord_onramp_arch_targets[ansible_facts['architecture']] == sponsord_onramp_target + fail_msg: >- + sponsord_onramp_target is {{ sponsord_onramp_target }}, but this host is + {{ ansible_facts['architecture'] }} and needs + {{ _sponsord_onramp_arch_targets[ansible_facts['architecture']] }}. Drop the + explicit sponsord_onramp_target (it is derived from the host) or fix it. + when: ansible_facts.get('architecture', '') in _sponsord_onramp_arch_targets + +# The same resolution as roles/sponsord: an explicit sponsord_onramp_manual_bin_src +# is copied verbatim (a test stub is legitimately not an ELF); a target dir is +# searched and the result must be an ELF for the host. +- name: Resolve and validate the manual binary source + when: sponsord_onramp_install_method == "manual" + block: + - name: Require a manual binary source + ansible.builtin.assert: + that: + - sponsord_onramp_manual_bin_src | length > 0 or sponsord_onramp_release_target_dir | length > 0 + fail_msg: >- + sponsord_onramp_install_method "manual" needs sponsord_onramp_manual_bin_src (a + control-machine path to the sponsord-onramp binary) or + sponsord_onramp_release_target_dir (the control-machine target/release dir of + a decdn/sponsord checkout; it defaults to sponsord_release_target_dir). + + - name: Decide whether target-dir resolution applies + ansible.builtin.set_fact: + _sponsord_onramp_use_dir: "{{ sponsord_onramp_manual_bin_src | length == 0 }}" + _sponsord_onramp_target_dir: "{{ sponsord_onramp_release_target_dir | regex_replace('/+$', '') }}" + _sponsord_onramp_bin_src: "{{ sponsord_onramp_manual_bin_src }}" + + - name: Stat the binary in each candidate directory + ansible.builtin.stat: + path: "{{ item }}/sponsord-onramp" + register: _sponsord_onramp_bin_stat + loop: + - "{{ _sponsord_onramp_target_dir }}" + - "{{ (_sponsord_onramp_target_dir | dirname) ~ '/' ~ sponsord_onramp_target ~ '/release' }}" + loop_control: + label: "{{ item }}" + delegate_to: localhost + become: false + when: _sponsord_onramp_use_dir | bool + + - name: Select the first candidate directory that holds the binary + ansible.builtin.set_fact: + _sponsord_onramp_resolved_dir: >- + {{ (_sponsord_onramp_bin_stat.results | selectattr('stat.exists') | map(attribute='item') + | first) | default('', true) }} + when: _sponsord_onramp_use_dir | bool + + - name: Fail if no candidate directory held the binary + ansible.builtin.assert: + that: + - _sponsord_onramp_resolved_dir | length > 0 + fail_msg: >- + No sponsord-onramp binary under sponsord_onramp_release_target_dir + ({{ sponsord_onramp_release_target_dir }}) or its cross output dir + ({{ (_sponsord_onramp_target_dir | dirname) ~ '/' ~ sponsord_onramp_target ~ '/release' }}). + Build it in a decdn/sponsord checkout — `cargo build --release -p + sponsord-onramp`, or cross-compiling for the host, `cross build --release -p + sponsord-onramp --target {{ sponsord_onramp_target }}`. + when: _sponsord_onramp_use_dir | bool + + - name: Derive the effective binary source path + ansible.builtin.set_fact: + _sponsord_onramp_bin_src: "{{ _sponsord_onramp_resolved_dir ~ '/sponsord-onramp' }}" + when: _sponsord_onramp_use_dir | bool + + - name: Identify the on-disk format of the resolved binary + ansible.builtin.command: + argv: [file, -b, --, "{{ _sponsord_onramp_bin_src }}"] + register: _sponsord_onramp_bin_file + changed_when: false + check_mode: false + delegate_to: localhost + become: false + when: _sponsord_onramp_use_dir | bool + + - name: Require the binary to be an ELF for the host's architecture + ansible.builtin.assert: + that: + - "'ELF' in _sponsord_onramp_bin_file.stdout" + - _sponsord_onramp_arch_file_tokens[sponsord_onramp_target] in _sponsord_onramp_bin_file.stdout + fail_msg: >- + {{ _sponsord_onramp_bin_src }} is "{{ _sponsord_onramp_bin_file.stdout }}" — not + an ELF {{ _sponsord_onramp_arch_file_tokens[sponsord_onramp_target] }} binary for + {{ sponsord_onramp_target }}. Cross-compile for the host: `cross build --release + -p sponsord-onramp --target {{ sponsord_onramp_target }}`. + when: _sponsord_onramp_use_dir | bool + +# --- Chain ------------------------------------------------------------------------ +- name: Require a known sponsord_onramp_network + ansible.builtin.assert: + that: + - sponsord_onramp_network == "" or sponsord_onramp_network in _sponsord_onramp_network_profiles + fail_msg: >- + sponsord_onramp_network "{{ sponsord_onramp_network }}" is not a known network. + Known: {{ _sponsord_onramp_network_profiles.keys() | join(', ') }}. Leave it "" + to set sponsord_onramp_capacity_bond_address yourself. + +# The onramp takes its chain from the daemon, so contract addresses from another +# network's profile would point users' decdn at the wrong chain's contracts. +- name: Require the onramp and the daemon to name the same network + ansible.builtin.assert: + that: + - sponsord_onramp_network == sponsord_network + fail_msg: >- + sponsord_onramp_network is "{{ sponsord_onramp_network }}" but the daemon's + sponsord_network is "{{ sponsord_network }}". The onramp serves the daemon's + chain, so both must name the same network. Drop sponsord_onramp_network (it + follows sponsord_network) or fix one of them. + when: + - sponsord_onramp_network | length > 0 + - sponsord_network | default('') | length > 0 + +# Also when the daemon has no profile but an explicit chain id: the onramp's +# profile must be that chain's. +- name: Require the onramp's network to be the daemon's chain + ansible.builtin.assert: + that: + - _sponsord_onramp_profile.chain_id | string == sponsord_chain_id | string + fail_msg: >- + sponsord_onramp_network "{{ sponsord_onramp_network }}" is chain + {{ _sponsord_onramp_profile.chain_id }}, but the daemon's sponsord_chain_id is + {{ sponsord_chain_id }}. Its CapacityBond and SlashJudge would belong to another + chain. Set the onramp's contract addresses for chain {{ sponsord_chain_id }} + explicitly and clear sponsord_onramp_network. + when: + - sponsord_onramp_network | length > 0 + - sponsord_chain_id | default('') | string | length > 0 + +- name: Report a contract address overridden in inventory + ansible.builtin.debug: + msg: >- + sponsord_onramp_network "{{ sponsord_onramp_network }}" is in use, but inventory + overrides {{ item }}. The inventory value wins over the profile; make sure that + is deliberate (a stale hand-copied address is the usual cause). + loop: [capacity_bond_address, slash_judge_address] + when: + - sponsord_onramp_network | length > 0 + - (lookup('ansible.builtin.vars', 'sponsord_onramp_' ~ item) | lower) != (_sponsord_onramp_profile[item] | lower) + +- name: Require the CapacityBond (and a well-formed SlashJudge) address + ansible.builtin.assert: + that: + - sponsord_onramp_capacity_bond_address is match('^0x[0-9a-fA-F]{40}$') + - sponsord_onramp_capacity_bond_address != '0x' ~ '0' * 40 + - >- + sponsord_onramp_slash_judge_address == '' + or (sponsord_onramp_slash_judge_address is match('^0x[0-9a-fA-F]{40}$') + and sponsord_onramp_slash_judge_address != '0x' ~ '0' * 40) + fail_msg: >- + sponsord_onramp_capacity_bond_address must be 0x + 40 hex, not zero, and + sponsord_onramp_slash_judge_address "" or the same shape. Set + sponsord_onramp_network (or the daemon's sponsord_network) to take both from the + role's mirror of upstream's deployment manifest — never hand-copy them. + +# --- Listener ----------------------------------------------------------------------- +# AGENTS.md hard rule 2: the onramp's own listener stays on loopback; the public +# side is Caddy (or your proxy) terminating TLS in front of it. +- name: Require the listener to stay on loopback + ansible.builtin.assert: + that: + - sponsord_onramp_bind_address is match('^127(\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])){3}$') + - sponsord_onramp_port | string is match('^[0-9]+$') + - sponsord_onramp_port | int >= 1 + - sponsord_onramp_port | int <= 65535 + - sponsord_onramp_daemon_url is match('^http://127(\.[0-9]{1,3}){3}:[0-9]+$') + fail_msg: >- + sponsord_onramp_bind_address must be an IPv4 loopback address (got + "{{ sponsord_onramp_bind_address }}"), sponsord_onramp_port a TCP port (got + "{{ sponsord_onramp_port }}") and sponsord_onramp_daemon_url the daemon's loopback + listener as http://127.x.y.z:port (got "{{ sponsord_onramp_daemon_url }}"). The + public side is the TLS proxy in front (AGENTS.md hard rule 2), never the onramp + itself. + +# --- Public address + TLS front ---------------------------------------------------- +- name: Validate the proxy settings + ansible.builtin.assert: + that: + - sponsord_onramp_proxy in ['caddy', 'none'] + - sponsord_onramp_caddy_tls in ['acme', 'internal'] + - sponsord_onramp_caddy_overwrite_config is boolean + fail_msg: >- + sponsord_onramp_proxy must be "caddy" or "none", sponsord_onramp_caddy_tls "acme" + or "internal", and sponsord_onramp_caddy_overwrite_config a boolean. + +- name: Require a domain for Caddy + ansible.builtin.assert: + that: + # A DNS name: labels of letters, digits and inner hyphens, at least one dot. + - sponsord_onramp_domain is match('^([A-Za-z0-9]([A-Za-z0-9-]{0,61}[A-Za-z0-9])?\.)+[A-Za-z]{2,63}$') + - >- + sponsord_onramp_acme_email == '' + or sponsord_onramp_acme_email is match('^[A-Za-z0-9._%+-]+@([A-Za-z0-9-]+\.)+[A-Za-z]{2,63}$') + fail_msg: >- + sponsord_onramp_proxy "caddy" needs sponsord_onramp_domain, the DNS name users + reach (got "{{ sponsord_onramp_domain }}"), and sponsord_onramp_acme_email must be + "" or an email address. For acme its A/AAAA records must point at this host. + when: sponsord_onramp_proxy == 'caddy' + +# Mirrors upstream's base_url + script_safe_url: the value is baked into the POSIX +# and PowerShell installers inside quoted strings. In the character class below, +# \x27 is ' and \x5c\x5c a backslash: a '' or \\ there would not survive Jinja's +# string-literal decoding. +- name: Require an https public base URL the installers can hold + ansible.builtin.assert: + that: + - sponsord_onramp_public_url is match('^https://[^/?#@]+(/[^?#]*)?$') + - sponsord_onramp_public_url is not search('[\s\x27"`$\x5c\x5c{}]') + fail_msg: >- + sponsord_onramp_public_url must be an https base URL with no credentials, query + or fragment, and none of whitespace ' " ` $ \ { } (got + "{{ sponsord_onramp_public_url }}"). With sponsord_onramp_proxy "none" it is + required: the address your proxy serves the onramp at. + +# Public by design: GET /v1/profile hands it to every user. A key in it is a leaked +# key, so the role refuses anything that looks like a credential. The message never +# echoes the value, in case it does hold one. +- name: Require a public RPC URL with no credentials in it + ansible.builtin.assert: + that: + - sponsord_onramp_rpc_url is match('^https?://[^/?#@]+(/[^?#]*)?$') + - sponsord_onramp_rpc_url is not search('[\s\x27"`$\x5c\x5c{}]') + fail_msg: >- + sponsord_onramp_rpc_url must be an http(s) URL with no userinfo, query or + fragment, and none of whitespace ' " ` $ \ { }. The onramp PUBLISHES it to every + user and bakes it into the installers, so it must be a public endpoint (e.g. + https://sepolia-rollup.arbitrum.io/rpc), never the keyed URL the daemon uses. + Check its path holds no API key either. + +# --- Gate + release pins -------------------------------------------------------------- +- name: Validate the Turnstile sitekey and gate settings + ansible.builtin.assert: + that: + - sponsord_onramp_turnstile_sitekey is match('^[A-Za-z0-9_-]+$') + - sponsord_onramp_brand_name | string is match('^[^\x00-\x1f\x7f]*\Z') + fail_msg: >- + sponsord_onramp_turnstile_sitekey is required: letters, digits, '_' and '-' (got + "{{ sponsord_onramp_turnstile_sitekey }}"), from the Turnstile widget for + {{ sponsord_onramp_domain | default('your domain', true) }}. sponsord_onramp_brand_name + may hold no control characters. + +- name: Validate the release pins the installers use + ansible.builtin.assert: + that: + - sponsord_onramp_decdn_release is match('^v' ~ _semver ~ '$') + - sponsord_onramp_cli_release is match('^decdn-sponsored-v' ~ _semver ~ '$') + - sponsord_onramp_decdn_sums_sha256 is match('^[0-9a-f]{64}$') + - sponsord_onramp_cli_sums_sha256 is match('^[0-9a-f]{64}$') + - sponsord_onramp_min_cli_version == '' or sponsord_onramp_min_cli_version is match('^' ~ _semver ~ '$') + - >- + sponsord_onramp_releases_base == '' + or (sponsord_onramp_releases_base is match('^https://[^/?#@]+(/[^?#]*)?$') + and sponsord_onramp_releases_base is not search('[\s\x27"`$\x5c\x5c{}]')) + fail_msg: >- + The installers need sponsord_onramp_decdn_release (vMAJOR.MINOR.PATCH), + sponsord_onramp_cli_release (decdn-sponsored-vMAJOR.MINOR.PATCH), and the SHA-256 + of each release's SHA256SUMS in sponsord_onramp_decdn_sums_sha256 / + sponsord_onramp_cli_sums_sha256 (64 lowercase hex; decdn-sponsored's release notes + print both values). sponsord_onramp_min_cli_version must be "" or MAJOR.MINOR.PATCH, + sponsord_onramp_releases_base "" or an https base URL. + vars: + _semver: '(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-[0-9A-Za-z.-]+)?' + +- name: Validate the onramp's limits + ansible.builtin.assert: + that: + - item.value | string is match('^[0-9]+$') + - item.key is search('_rate_per_min$') or item.value | int > 0 + fail_msg: >- + {{ item.key }} must be "" (the onramp's default) or a non-negative integer + ({{ 'zero turns the limit off' if item.key is search('_rate_per_min$') else 'greater than zero' }}), + got "{{ item.value }}". + loop: "{{ _sponsord_onramp_limits | dict2items | rejectattr('value', 'equalto', '') | list }}" + loop_control: + label: "{{ item.key }}" + vars: + _sponsord_onramp_limits: + sponsord_onramp_spending_cap_micro_usdc: "{{ sponsord_onramp_spending_cap_micro_usdc }}" + sponsord_onramp_ttl_secs: "{{ sponsord_onramp_ttl_secs }}" + sponsord_onramp_fund_rate_per_min: "{{ sponsord_onramp_fund_rate_per_min }}" + sponsord_onramp_poll_rate_per_min: "{{ sponsord_onramp_poll_rate_per_min }}" + +- name: Validate the client IP header name + ansible.builtin.assert: + that: + - sponsord_onramp_client_ip_header == '' or sponsord_onramp_client_ip_header is match('^[A-Za-z0-9-]+$') + fail_msg: >- + sponsord_onramp_client_ip_header must be "" (the TCP peer) or an HTTP header name + such as X-Forwarded-For or CF-Connecting-IP (got + "{{ sponsord_onramp_client_ip_header }}"). + +# --- Host secrets (the daemon's token, the Turnstile secret) ------------------------ +- name: Check the daemon API token and the Turnstile secret + ansible.builtin.stat: + path: "{{ item }}" + follow: false + get_checksum: false + register: sponsord_onramp_secret_stat + loop: + - "{{ sponsord_onramp_api_token_file }}" + - "{{ sponsord_onramp_turnstile_secret_file }}" + +# The sponsord role, which runs first on this host, generated (or gated on) the +# token; under --check its generation is only simulated, so a fresh host has none. +- name: Require the daemon API token as a regular file + ansible.builtin.assert: + that: + - sponsord_onramp_secret_stat.results[0].stat.isreg | default(false) + fail_msg: >- + {{ sponsord_onramp_api_token_file }} is + {{ 'missing' if not sponsord_onramp_secret_stat.results[0].stat.exists else 'not a regular file' }}. + The onramp runs beside the sponsord daemon and uses its token, which the sponsord + role creates: deploy sponsord on this host first (in this repo, list the host in + sponsord_hosts too; `make deploy-sponsord` runs both). + when: not (ansible_check_mode and not sponsord_onramp_secret_stat.results[0].stat.exists) + +- name: Require the Turnstile secret as a regular file + ansible.builtin.assert: + that: + - sponsord_onramp_secret_stat.results[1].stat.isreg | default(false) + fail_msg: |- + {{ sponsord_onramp_turnstile_secret_file }} is {{ _what }}. + The role never handles the Turnstile secret: copy it from the Cloudflare + dashboard (the widget for {{ sponsord_onramp_domain | default('your domain', true) }}) + onto this host, as root: + umask 077 + printf '%s' '' | sudo tee {{ sponsord_onramp_turnstile_secret_file }} >/dev/null + sudo chmod 600 {{ sponsord_onramp_turnstile_secret_file }} + vars: + _what: >- + {{ 'missing' if not sponsord_onramp_secret_stat.results[1].stat.exists + else 'not a regular file (symlink/directory?)' }} + +# Ownership + mode only; the content is the operator's. +- name: Lock down the Turnstile secret + ansible.builtin.file: + path: "{{ sponsord_onramp_turnstile_secret_file }}" + owner: root + group: root + mode: "0600" + follow: false + +# grep -q: the secret never reaches stdout. Upstream strips one trailing newline; +# an empty value would only fail at the first person through the gate. +- name: Confirm the Turnstile secret is not empty + ansible.builtin.command: + cmd: grep -q '[^[:space:]]' {{ sponsord_onramp_turnstile_secret_file | quote }} + register: sponsord_onramp_turnstile_grep + changed_when: false + check_mode: false + # 1 = no non-blank character (reported below); 2 = grep could not read the file. + failed_when: sponsord_onramp_turnstile_grep.rc not in [0, 1] + +- name: Report an empty Turnstile secret + ansible.builtin.fail: + msg: >- + {{ sponsord_onramp_turnstile_secret_file }} is empty. Write the widget's secret + key into it, as root: umask 077; printf '%s' '' | sudo tee + {{ sponsord_onramp_turnstile_secret_file }} >/dev/null + when: sponsord_onramp_turnstile_grep.rc == 1 + +- name: Read whether Caddy is already installed + ansible.builtin.package_facts: + when: sponsord_onramp_proxy == 'caddy' + +# Before any mutation: a Caddyfile someone else wrote is never replaced silently. +# It is judged by the file, not by whether the package is installed: a Caddy run +# from a static binary leaves no dpkg trace, and apt keeps an existing conffile. +- name: Check the Caddyfile + ansible.builtin.stat: + path: "{{ _sponsord_onramp_caddyfile }}" + follow: false + get_checksum: true + checksum_algorithm: md5 + register: sponsord_onramp_caddyfile_stat + when: sponsord_onramp_proxy == 'caddy' + +- name: Check the Caddyfile for this role's marker + ansible.builtin.command: + argv: [grep, -qF, "MANAGED BY the sponsord_onramp role", "{{ _sponsord_onramp_caddyfile }}"] + register: sponsord_onramp_caddyfile_marker + changed_when: false + check_mode: false + # 1 = no marker; 2 = unreadable, which must not pass for "no marker". + failed_when: sponsord_onramp_caddyfile_marker.rc not in [0, 1] + when: + - sponsord_onramp_proxy == 'caddy' + - sponsord_onramp_caddyfile_stat.stat.isreg | default(false) + +# The package's own default Caddyfile (a first run that installed Caddy and then +# stopped before templating leaves exactly that) is safe to replace. dpkg records +# its md5 among the package's conffiles; empty when the package is not installed. +- name: Read the packaged Caddyfile's checksum + ansible.builtin.shell: + executable: /bin/bash + cmd: | + set -o pipefail + { dpkg-query -W -f='${Conffiles}\n' caddy 2>/dev/null || true; } \ + | awk -v f={{ _sponsord_onramp_caddyfile | quote }} '$1 == f { print $2 }' + register: sponsord_onramp_caddyfile_pristine + changed_when: false + check_mode: false + when: + - sponsord_onramp_proxy == 'caddy' + - sponsord_onramp_caddyfile_marker.rc | default(0) == 1 + +- name: Refuse to replace a Caddyfile this role did not write + ansible.builtin.assert: + that: + - sponsord_onramp_caddyfile_stat.stat.isreg + - >- + sponsord_onramp_caddy_overwrite_config + or sponsord_onramp_caddyfile_marker.rc | default(0) == 0 + or (sponsord_onramp_caddyfile_pristine.stdout | default('') | trim | length > 0 + and sponsord_onramp_caddyfile_pristine.stdout | trim == sponsord_onramp_caddyfile_stat.stat.checksum) + fail_msg: >- + {{ _sponsord_onramp_caddyfile }} exists and is not the role's (no "MANAGED BY the + sponsord_onramp role" line, and not the caddy package's untouched default), or + is not a regular file. The role owns that whole file, so it would discard + whatever else this Caddy serves. Move the other sites elsewhere (or onto your + own proxy, with sponsord_onramp_proxy "none"), or set + sponsord_onramp_caddy_overwrite_config: true to let the role replace it. + when: + - sponsord_onramp_proxy == 'caddy' + - sponsord_onramp_caddyfile_stat.stat.exists + +# Switching to "none" closes the firewall holes but leaves the role's Caddy running +# on 80/443 (your own proxy could not bind them). The role removes nothing. +- name: Check for a Caddyfile this role wrote while proxy was "caddy" + ansible.builtin.command: + argv: [grep, -qF, "MANAGED BY the sponsord_onramp role", "{{ _sponsord_onramp_caddyfile }}"] + register: sponsord_onramp_stale_caddy + changed_when: false + check_mode: false + failed_when: false + when: sponsord_onramp_proxy == 'none' + +- name: Warn that the role's Caddy is still installed + ansible.builtin.debug: + msg: >- + WARNING: sponsord_onramp_proxy is "none", but {{ _sponsord_onramp_caddyfile }} is + this role's, so its Caddy may still be running and holding tcp/80 and tcp/443 + (the firewall no longer opens them). Remove it: sudo systemctl disable --now + caddy && sudo apt-get remove caddy. + when: + - sponsord_onramp_proxy == 'none' + - sponsord_onramp_stale_caddy.rc | default(1) == 0 + +# --- Install, configure, start ------------------------------------------------------ +- name: Install the sponsord-onramp binary + ansible.builtin.import_tasks: install.yml + +- name: Install the custom gate page + ansible.builtin.copy: + src: "{{ sponsord_onramp_gate_template_src }}" + dest: "{{ sponsord_onramp_gate_template_file }}" + owner: root + group: root + mode: "0644" + notify: Restart sponsord-onramp + when: sponsord_onramp_gate_template_src | length > 0 + +- name: Remove a custom gate page that is no longer configured + ansible.builtin.file: + path: "{{ sponsord_onramp_gate_template_file }}" + state: absent + notify: Restart sponsord-onramp + when: sponsord_onramp_gate_template_src | length == 0 + +- name: Template the non-secret sponsord-onramp.env + ansible.builtin.template: + src: sponsord-onramp.env.j2 + dest: "{{ sponsord_onramp_env_file }}" + owner: root + group: root + mode: "0644" + notify: Restart sponsord-onramp + +- name: Install the sponsord-onramp systemd unit + ansible.builtin.template: + src: sponsord-onramp.service.j2 + dest: /etc/systemd/system/sponsord-onramp.service + owner: root + group: root + mode: "0644" + notify: Restart sponsord-onramp + +# --- Restart inputs ----------------------------------------------------------------- +# As roles/sponsord: hashes of everything the onramp starts from, recorded once it +# is healthy, so a rotated daemon token or Turnstile secret (both edited on the +# host, out of band) or a restart lost to a failed run is caught next converge. +- name: Hash the restart inputs + ansible.builtin.stat: + path: "{{ item }}" + get_checksum: true + checksum_algorithm: sha256 + register: sponsord_onramp_inputs_stat + loop: + - "{{ sponsord_onramp_api_token_file }}" + - "{{ sponsord_onramp_turnstile_secret_file }}" + - "{{ sponsord_onramp_env_file }}" + - "{{ sponsord_onramp_gate_template_file }}" + - /etc/systemd/system/sponsord-onramp.service + - "{{ sponsord_onramp_bin }}" + +- name: Read the restart-input checksums this role last recorded + ansible.builtin.slurp: + src: "{{ sponsord_onramp_inputs_checksum_file }}" + register: sponsord_onramp_inputs_recorded + failed_when: >- + sponsord_onramp_inputs_recorded.failed + and 'not found' not in (sponsord_onramp_inputs_recorded.msg | default('')) + changed_when: false + +- name: Read whether sponsord-onramp is already running + ansible.builtin.systemd_service: + name: sponsord-onramp + register: sponsord_onramp_is_active + +- name: Summarise the restart-input checksums + ansible.builtin.set_fact: + sponsord_onramp_inputs_sums: >- + {{ sponsord_onramp_inputs_stat.results | map(attribute='stat') | map('default', {}, true) + | map(attribute='checksum', default='') | join(' ') }} + sponsord_onramp_inputs_recorded_sums: >- + {{ sponsord_onramp_inputs_recorded.content | default('') | b64decode | trim }} + +- name: Note restart inputs that changed since the onramp last started healthy + ansible.builtin.debug: + msg: >- + {{ 'The daemon token, the Turnstile secret, the env file, the gate page, the unit + or the binary differs from what sponsord-onramp was last confirmed healthy on' + if sponsord_onramp_inputs_recorded_sums | length > 0 + else 'sponsord-onramp is running but was never confirmed healthy on its current inputs' }} + — restarting sponsord-onramp. + changed_when: true + notify: Restart sponsord-onramp + when: >- + (sponsord_onramp_inputs_recorded_sums | length > 0 + and sponsord_onramp_inputs_sums != sponsord_onramp_inputs_recorded_sums) + or (sponsord_onramp_inputs_recorded_sums | length == 0 + and sponsord_onramp_is_active.status.ActiveState | default('') == 'active') + +- name: Apply pending sponsord-onramp changes + ansible.builtin.meta: flush_handlers + +- name: Enable and start sponsord-onramp + ansible.builtin.systemd: + name: sponsord-onramp + enabled: true + state: started + daemon_reload: true + +# --- Readiness (fatal) ------------------------------------------------------------ +# The onramp binds only after it has read the daemon's /v1/info and checked its own +# cap and TTL against the daemon's maximum, so this is the deploy's config gate. +# Retry while nothing answers (-1); any other answer than 200 stops at once. +# KICS flags plain HTTP here; the target is the loopback listener asserted above. +- name: Wait for sponsord-onramp /healthz + ansible.builtin.uri: + url: "http://{{ sponsord_onramp_bind_address }}:{{ sponsord_onramp_port }}/healthz" + status_code: 200 + return_content: true + timeout: 5 + register: sponsord_onramp_health + retries: "{{ sponsord_onramp_readiness_retries }}" + delay: "{{ sponsord_onramp_readiness_delay }}" + until: sponsord_onramp_health.status | default(-1) != -1 + changed_when: false + failed_when: false + when: not ansible_check_mode + +- name: Report a sponsord-onramp that did not become healthy + ansible.builtin.fail: + msg: |- + sponsord-onramp did not answer 200 on + http://{{ sponsord_onramp_bind_address }}:{{ sponsord_onramp_port }}/healthz within + ~{{ (sponsord_onramp_readiness_retries | int) * (sponsord_onramp_readiness_delay | int) }}s + (last status {{ sponsord_onramp_health.status | default('none') }}). It binds only + after reaching the daemon at {{ sponsord_onramp_daemon_url }} with its API token + and validating its settings, so the usual causes are a daemon that is down, a + cap or TTL above the daemon's maximum, or a malformed release pin. Read the + reason on the host: + sudo journalctl -u sponsord-onramp -e + when: + - not ansible_check_mode + - >- + sponsord_onramp_health.status | default(-1) != 200 + or sponsord_onramp_health.json.ok | default(false) is not true + +# As roles/sponsord: the 200 must come from THIS unit, not another holder of the port. +- name: Read the sponsord-onramp unit state + ansible.builtin.systemd_service: + name: sponsord-onramp + register: sponsord_onramp_unit_state + when: not ansible_check_mode + +- name: Read the owner of the sponsord-onramp listener + ansible.builtin.command: + argv: [ss, -Hltnp, "sport = :{{ sponsord_onramp_port }}"] + register: sponsord_onramp_listener + changed_when: false + when: not ansible_check_mode + +- name: Require the listener to belong to the running sponsord-onramp unit + ansible.builtin.assert: + that: + - sponsord_onramp_unit_state.status.ActiveState == 'active' + - sponsord_onramp_unit_state.status.SubState == 'running' + - ('pid=' ~ sponsord_onramp_unit_state.status.MainPID ~ ',') in sponsord_onramp_listener.stdout + fail_msg: >- + Something answered /healthz on {{ sponsord_onramp_bind_address }}:{{ sponsord_onramp_port }}, + but it is not the sponsord-onramp unit (state + {{ sponsord_onramp_unit_state.status.ActiveState }}/{{ sponsord_onramp_unit_state.status.SubState }}, + MainPID {{ sponsord_onramp_unit_state.status.MainPID }}; listeners: + {{ sponsord_onramp_listener.stdout | default('none', true) }}). Another service holds + the port: free it or set sponsord_onramp_port. + when: not ansible_check_mode + +- name: Record the restart-input checksums + ansible.builtin.copy: + dest: "{{ sponsord_onramp_inputs_checksum_file }}" + owner: root + group: root + mode: "0600" + content: "{{ sponsord_onramp_inputs_sums }}\n" + when: not ansible_check_mode + +# --- TLS front ---------------------------------------------------------------------- +- name: Put Caddy in front of the onramp + ansible.builtin.import_tasks: caddy.yml + when: sponsord_onramp_proxy == 'caddy' diff --git a/ansible/roles/sponsord_onramp/templates/Caddyfile.j2 b/ansible/roles/sponsord_onramp/templates/Caddyfile.j2 new file mode 100644 index 0000000..6365236 --- /dev/null +++ b/ansible/roles/sponsord_onramp/templates/Caddyfile.j2 @@ -0,0 +1,28 @@ +# MANAGED BY the sponsord_onramp role — do not edit by hand. +# +# The public side of sponsord-onramp: TLS for {{ sponsord_onramp_domain }}, then a +# reverse proxy to the onramp on loopback. Caddy redirects http to https and, +# with no trusted_proxies, replaces any client-sent X-Forwarded-For with the real +# client address, which the onramp rate limits by. +{ + # No admin API: nothing on this host can reconfigure the running proxy over + # the admin endpoint (:2019). + # Config changes restart Caddy instead of reloading it. + admin off + # HTTP/1.1 and HTTP/2 only: HTTP/3 would need udp/443, which the firewall + # does not open. + servers { + protocols h1 h2 + } +{% if sponsord_onramp_acme_email | length > 0 %} + email {{ sponsord_onramp_acme_email }} +{% endif %} +} + +{{ sponsord_onramp_domain }} { +{% if sponsord_onramp_caddy_tls == 'internal' %} + # Caddy's local CA: for staging and tests only, browsers do not trust it. + tls internal +{% endif %} + reverse_proxy {{ sponsord_onramp_bind_address }}:{{ sponsord_onramp_port }} +} diff --git a/ansible/roles/sponsord_onramp/templates/sponsord-onramp.env.j2 b/ansible/roles/sponsord_onramp/templates/sponsord-onramp.env.j2 new file mode 100644 index 0000000..5904d9b --- /dev/null +++ b/ansible/roles/sponsord_onramp/templates/sponsord-onramp.env.j2 @@ -0,0 +1,36 @@ +# MANAGED BY the sponsord_onramp role — do not edit by hand. NO secrets: the +# daemon's API token and the Turnstile secret reach the onramp as systemd +# credentials (see sponsord-onramp.service). Every value here is non-secret; the +# RPC URL and the contract addresses are even served to every user. +ONRAMP_BIND={{ sponsord_onramp_bind_address }}:{{ sponsord_onramp_port }} +ONRAMP_DAEMON_URL={{ sponsord_onramp_daemon_url }} +ONRAMP_PUBLIC_URL={{ sponsord_onramp_public_url }} +ONRAMP_RPC_URL={{ sponsord_onramp_rpc_url }} +ONRAMP_CAPACITY_BOND_ADDR={{ sponsord_onramp_capacity_bond_address }} +ONRAMP_GATE=turnstile +ONRAMP_TURNSTILE_SITEKEY={{ sponsord_onramp_turnstile_sitekey }} +ONRAMP_DECDN_RELEASE={{ sponsord_onramp_decdn_release }} +ONRAMP_DECDN_SUMS_SHA256={{ sponsord_onramp_decdn_sums_sha256 }} +ONRAMP_CLI_RELEASE={{ sponsord_onramp_cli_release }} +ONRAMP_CLI_SUMS_SHA256={{ sponsord_onramp_cli_sums_sha256 }} +{% if sponsord_onramp_brand_name | length > 0 %} +{# to_json: double-quoted with \" and \\ escaped, which systemd's EnvironmentFile + parser unescapes (preflight refuses control characters, so no other escape is + emitted). #} +ONRAMP_BRAND_NAME={{ sponsord_onramp_brand_name | string | to_json(ensure_ascii=false) }} +{% endif %} +{% if sponsord_onramp_gate_template_src | length > 0 %} +ONRAMP_GATE_TEMPLATE={{ sponsord_onramp_gate_template_file }} +{% endif %} +{% for name, value in [ + ('ONRAMP_SLASH_JUDGE_ADDR', sponsord_onramp_slash_judge_address), + ('ONRAMP_CLIENT_IP_HEADER', sponsord_onramp_client_ip_header), + ('ONRAMP_MIN_CLI_VERSION', sponsord_onramp_min_cli_version), + ('ONRAMP_RELEASES_BASE', sponsord_onramp_releases_base), + ('ONRAMP_SPENDING_CAP_MICRO_USDC', sponsord_onramp_spending_cap_micro_usdc), + ('ONRAMP_TTL_SECS', sponsord_onramp_ttl_secs), + ('ONRAMP_FUND_RATE_PER_MIN', sponsord_onramp_fund_rate_per_min), + ('ONRAMP_POLL_RATE_PER_MIN', sponsord_onramp_poll_rate_per_min), + ] if value | string | length > 0 %} +{{ name }}={{ value }} +{% endfor %} diff --git a/ansible/roles/sponsord_onramp/templates/sponsord-onramp.service.j2 b/ansible/roles/sponsord_onramp/templates/sponsord-onramp.service.j2 new file mode 100644 index 0000000..4b37832 --- /dev/null +++ b/ansible/roles/sponsord_onramp/templates/sponsord-onramp.service.j2 @@ -0,0 +1,66 @@ +# MANAGED BY the sponsord_onramp role — do not edit by hand. +# +# sponsord-onramp: the gate page, the installers and the CLI API, in front of the +# sponsord daemon on this host. It listens on loopback only +# ({{ sponsord_onramp_bind_address }}:{{ sponsord_onramp_port }}); {{ ('Caddy terminates TLS for ' ~ sponsord_onramp_domain) if sponsord_onramp_proxy == 'caddy' else "the operator's own proxy terminates TLS" }}. +# It keeps only in-flight hand-offs in memory. The daemon's API token and the +# Turnstile secret never sit in the environment: systemd reads the root-only files +# and hands them to this unit as credentials. +[Unit] +Description=sponsord-onramp: the public onramp in front of sponsord +Documentation=https://github.com/decdn/sponsord +After=network-online.target sponsord.service +Wants=network-online.target sponsord.service +# The onramp reads the daemon's limits once, at start; restart it with the daemon. +PartOf=sponsord.service +# It exits at start while the daemon is unreachable; keep retrying until it is up. +StartLimitIntervalSec=0 + +[Service] +Type=simple +ExecStart={{ sponsord_onramp_bin }} +EnvironmentFile={{ sponsord_onramp_env_file }} +# Plain-text logs into journald, without ANSI colour codes. +Environment=NO_COLOR=1 + +# Credentials land at $CREDENTIALS_DIRECTORY (%d), readable only by this unit. +LoadCredential=api-token:{{ sponsord_onramp_api_token_file }} +LoadCredential=turnstile-secret:{{ sponsord_onramp_turnstile_secret_file }} +Environment=ONRAMP_DAEMON_TOKEN_FILE=%d/api-token +Environment=ONRAMP_TURNSTILE_SECRET_FILE=%d/turnstile-secret + +KillSignal=SIGTERM +TimeoutStopSec=30 +Restart=on-failure +RestartSec=5 + +# --- Hardening ------------------------------------------------------------- +# Stateless: no StateDirectory; nothing on disk is writable beyond its private /tmp. +DynamicUser=true +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +PrivateDevices=true +ProtectKernelTunables=true +ProtectKernelModules=true +ProtectKernelLogs=true +ProtectControlGroups=true +ProtectClock=true +ProtectHostname=true +CapabilityBoundingSet= +# AF_INET/AF_INET6 for the listener, the daemon and Turnstile's siteverify; +# AF_UNIX + AF_NETLINK for the libc resolver. +RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK +RestrictNamespaces=true +RestrictRealtime=true +RestrictSUIDSGID=true +LockPersonality=true +MemoryDenyWriteExecute=true +SystemCallArchitectures=native +SystemCallFilter=@system-service +SystemCallErrorNumber=EPERM +UMask=0077 + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/sponsord_onramp/vars/main/arch.yml b/ansible/roles/sponsord_onramp/vars/main/arch.yml new file mode 100644 index 0000000..0b4387b --- /dev/null +++ b/ansible/roles/sponsord_onramp/vars/main/arch.yml @@ -0,0 +1,16 @@ +--- +# Host architecture (ansible_facts['architecture']) -> the upstream release target +# triple. These are the two Linux targets decdn/sponsord's release builds for +# sponsord-onramp (LINUX in .github/scripts/release_plan.py); the defaults derive +# sponsord_onramp_target from this map and tasks/main.yml asserts the result is one +# of its values. Role vars, not defaults: the supported set is a fact about +# upstream's release matrix, not an operator knob. +_sponsord_onramp_arch_targets: + x86_64: x86_64-unknown-linux-gnu + aarch64: aarch64-unknown-linux-gnu + +# sponsord_onramp_target -> the architecture token `file -b` prints for a matching +# ELF, used by the manual dir-mode binary check. +_sponsord_onramp_arch_file_tokens: + x86_64-unknown-linux-gnu: x86-64 + aarch64-unknown-linux-gnu: aarch64 diff --git a/ansible/roles/sponsord_onramp/vars/main/caddy.yml b/ansible/roles/sponsord_onramp/vars/main/caddy.yml new file mode 100644 index 0000000..ed3e16e --- /dev/null +++ b/ansible/roles/sponsord_onramp/vars/main/caddy.yml @@ -0,0 +1,4 @@ +--- +# The Caddyfile the distribution's caddy.service loads (its ExecStart names this +# path), so it is a fact of the package, not a knob. +_sponsord_onramp_caddyfile: /etc/caddy/Caddyfile diff --git a/ansible/roles/sponsord_onramp/vars/main/networks.yml b/ansible/roles/sponsord_onramp/vars/main/networks.yml new file mode 100644 index 0000000..38341b4 --- /dev/null +++ b/ansible/roles/sponsord_onramp/vars/main/networks.yml @@ -0,0 +1,15 @@ +--- +# GENERATED by scripts/sync-network-profiles.py -- do not hand-edit. +# Source: decdn/decdn @ 5ca3e7f1c5ac84890fb5b666b29b6f384e3797ec +# crates/cli/src/known_chains.rs (KNOWN_CHAINS, KnownChain::addresses) +# + contracts/deployments/.json +# +# The subset of roles/decdn_node/vars/main/networks.yml sponsord_onramp needs. The role's +# chain defaults read this map lazily when sponsord_onramp_network is set, so an explicit +# inventory value still wins. Protocol facts: upstream's deployment manifests are +# the source, this file only mirrors them. +_sponsord_onramp_network_profiles: + arbitrum-sepolia: + chain_id: 421614 + capacity_bond_address: "0x65a9e38F41B9bE2fcB97950a0E3c33e506E6E353" + slash_judge_address: "0x8e4d5945615bDF00106ff429da07bd08bCdEdB94" diff --git a/ansible/tests/alloy-config/render.yml b/ansible/tests/alloy-config/render.yml index 0ed9fbb..f8f0bb8 100644 --- a/ansible/tests/alloy-config/render.yml +++ b/ansible/tests/alloy-config/render.yml @@ -316,6 +316,26 @@ grafana_alloy_sponsord_enabled: true grafana_alloy_logs_drop_priority_regex: 'info|debug|trace' + # The same host also running the onramp (sponsord_onramp_hosts): its journal + # stream gets a service_name and the plain-text level parsing, no scrape. + - name: Render the co-located node + sponsord + onramp configuration + ansible.builtin.template: + src: "{{ playbook_dir }}/../../roles/grafana_alloy/templates/{{ item.src }}" + dest: "{{ _ga_render_dir }}/colocatedonramp.{{ item.ext }}" + mode: "0644" + loop: *ga_render_pair + loop_control: + label: "colocatedonramp.{{ item.ext }}" + vars: + _ga_instance_id: decdn-node-1 + _ga_journal_groups: [systemd-journal, adm] + grafana_alloy_bin_effective: /usr/bin/alloy + grafana_alloy_deployment_environment: production + grafana_alloy_region: US + grafana_alloy_sponsord_enabled: true + grafana_alloy_sponsord_onramp_enabled: true + grafana_alloy_logs_drop_priority_regex: 'info|debug|trace' + - name: Export the pinned release for the validation script ansible.builtin.copy: dest: "{{ _ga_render_dir }}/pin.env" diff --git a/ansible/tests/alloy-config/validate.sh b/ansible/tests/alloy-config/validate.sh index af8dee9..1ae823b 100755 --- a/ansible/tests/alloy-config/validate.sh +++ b/ansible/tests/alloy-config/validate.sh @@ -269,10 +269,17 @@ assert_has sponsordonly.alloy 'forward_to = [prometheus.relabel.sponsord_id assert_has sponsordonly.alloy 'replacement = "sponsord"' "the sponsord metric service_name" assert_has sponsordonly.alloy 'regex = "sponsord\\.service"' "the sponsord log-stream unit match" assert_has sponsordonly.alloy 'replacement = "sponsord"' "the sponsord log-stream service_name" -assert_has sponsordonly.alloy '(decdn-node|sponsord|alloy|ssh|sshd)' "sponsord in the systemd collector's unit list" +assert_has sponsordonly.alloy '(decdn-node|sponsord|sponsord-onramp|caddy|alloy|ssh|sshd)' "sponsord, the onramp and Caddy in the systemd collector's unit list" assert_has colocated.alloy 'prometheus.scrape "decdn_node"' "the node scrape on a co-located host" assert_has colocated.alloy 'prometheus.scrape "sponsord"' "the sponsord scrape on a co-located host" assert_has colocated.alloy '"__address__" = "127.0.0.1:8090"' "the default sponsord listener" +assert_lacks defaults.alloy 'sponsord-onramp\\.service' "an onramp log rule while it is disabled" +assert_lacks colocated.alloy 'sponsord-onramp\\.service' "an onramp log rule while it is disabled" +assert_lacks colocatedonramp.alloy 'prometheus.scrape "sponsord_onramp"' "an onramp scrape (it has no /metrics)" +assert_has colocatedonramp.alloy 'regex = "sponsord-onramp\\.service"' "the onramp log-stream unit match" +assert_has colocatedonramp.alloy 'replacement = "sponsord-onramp"' "the onramp log-stream service_name" +assert_has colocatedonramp.alloy '"(?:decdn-node|sponsord|sponsord-onramp)\\.service"' "the onramp in the priority exemption" +assert_has colocatedonramp.alloy '{unit=~"(sponsord|sponsord-onramp)\\.service"}' "the onramp in the plain-text level selector" echo "ok: per-daemon scrape toggles" # --- Gate 1c: URLs are redacted, and the daemon's level comes from its line ---- @@ -308,6 +315,12 @@ samples_json() { # nanosecond timestamp ["$1","Caused by: s-cause connection refused"], ["$1","s-plain an unlevelled line"], ["$1","Error: s-rpcurl read PaymentPool.usdc(): error sending request for url (https://arb-sepolia.example-provider.io/v2/SECRET-KEY?x=SECRET-Q)"]]}, + {"stream":{"journal__systemd_unit":"sponsord-onramp.service","journal_priority_keyword":"info"},"values":[ + ["$1","2026-10-05T07:59:11.000001Z DEBUG sponsord_onramp::http: ro-debug"], + ["$1","2026-10-05T07:59:11.000002Z INFO sponsord_onramp: ro-info"], + ["$1","2026-10-05T07:59:11.000003Z WARN sponsord_onramp::gate::turnstile: ro-warn"], + ["$1","Error: ro-raw the daemon at http://127.0.0.1:8090 is unreachable"], + ["$1","ro-plain an unlevelled line"]]}, {"stream":{"journal__systemd_unit":"other.service","journal_priority_keyword":"info"},"values":[ ["$1","{\"level\":\"ERROR\",\"fields\":{\"message\":\"m-other-info\"}}"]]}, {"stream":{"journal__systemd_unit":"other.service","journal_priority_keyword":"error"},"values":[ @@ -438,7 +451,7 @@ no_secrets() { # Defaults (guardrail "debug|trace"), sponsord not enabled: its plain-text lines # are not re-levelled and stand at journald's info. -run_level_harness defaults.alloy 34 +run_level_harness defaults.alloy 39 kept s-trace info sponsord.service kept s-error info sponsord.service kept m-warn-uc warning decdn-node.service @@ -450,6 +463,8 @@ kept m-weird info decdn-node.service # a non-string level is not a label valu kept m-text info decdn-node.service # not JSON: journald's stands kept m-other-info info other.service # another unit's JSON is never re-levelled kept m-other-error error other.service +kept ro-warn info sponsord-onramp.service # onramp not enabled: journald's level +kept ro-debug info sponsord-onramp.service dropped m-trace # TRACE is not a journald keyword, so listed dropped m-debug # padded + lower-cased before the drop dropped m-other-debug # the journald-priority drop, other units @@ -508,6 +523,7 @@ kept m-nolevel info decdn-node.service kept m-weird info decdn-node.service kept m-text info decdn-node.service kept m-other-error error other.service +dropped ro-warn # onramp not enabled: judged at journald's info dropped m-info dropped m-trace dropped m-debug @@ -516,7 +532,7 @@ dropped m-other-debug echo "ok: daemon log level follows the JSON body (info in the guardrail)" # sponsord enabled: its level is parsed from the plain-text line. -run_level_harness sponsordonly.alloy 32 +run_level_harness sponsordonly.alloy 37 no_secrets kept s-rpcurl error sponsord.service # the "Error: " match still sees the redacted line kept s-info info sponsord.service @@ -528,6 +544,7 @@ kept s-plain info sponsord.service # any other unlevelled line: journald's dropped s-trace dropped s-debug kept m-error error decdn-node.service # the node's JSON stage is unaffected +kept ro-warn info sponsord-onramp.service # the onramp is not enabled here echo "ok: sponsord log level follows its text line (defaults)" # Co-located, guardrail "info|debug|trace": both daemons exempt from the @@ -546,8 +563,22 @@ dropped s-info dropped s-trace dropped m-info dropped m-other-info +dropped ro-warn # the onramp is not enabled: not exempt +dropped ro-raw echo "ok: both daemons judged by their own level (info in the guardrail)" +# The onramp too: exempt, and levelled from its plain-text line like sponsord. +run_level_harness colocatedonramp.alloy 32 +no_secrets +kept ro-warn warning sponsord-onramp.service +kept ro-raw error sponsord-onramp.service # its exit error: an error +kept ro-plain info sponsord-onramp.service # exempt: no level of its own to judge by +kept s-warn warning sponsord.service # sponsord's stage is unaffected +kept m-error error decdn-node.service +dropped ro-info +dropped ro-debug +echo "ok: the onramp judged by its own level (info in the guardrail)" + # --- Gate 2: every ExecStart flag exists ------------------------------------- # `alloy run` ignores nothing: an unknown flag exits non-zero, i.e. a systemd # crash-loop on the target host the moment the unit starts. diff --git a/ansible/tests/firewall-holes/check.sh b/ansible/tests/firewall-holes/check.sh new file mode 100755 index 0000000..2928421 --- /dev/null +++ b/ansible/tests/firewall-holes/check.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# The baseline firewall's public holes are derived from a host's groups in +# playbooks/group_vars/ (all.yml builds the list; decdn_nodes.yml and +# sponsord_hosts.yml both point baseline_extra_inbound at it). Resolve them for +# every host shape in inventory.yml, outside any role, which is how the sponsord +# play sees a co-located node, and compare with expected.json. +# Needs ansible-core and jq. tests/scripts-test.sh runs it. +set -euo pipefail + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +playbooks="$here/../../playbooks" + +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT + +# Each host writes its resolved list to .json with the core copy module (the +# json stdout callback lives in ansible.posix, which a core-only install lacks). +# From this directory, so ansible/ansible.cfg (inventory, become) is not read. +( + cd "$here" + ANSIBLE_DEPRECATION_WARNINGS=0 ansible all -i "$here/inventory.yml" --playbook-dir "$playbooks" \ + -m ansible.builtin.copy \ + -a "dest=$work/{{ inventory_hostname }}.json content={{ baseline_extra_inbound | to_json }} mode=0600" \ + "$work/ansible.log" 2>&1 +) || { cat "$work/ansible.log" >&2; echo "ansible could not resolve the inventory" >&2; exit 1; } +got="$(cd "$work" && jq -S -n '[inputs | {key: (input_filename | rtrimstr(".json")), value: .}] | from_entries' ./*.json \ + | jq -S 'with_entries(.key |= ltrimstr("./"))')" +want="$(jq -S . "$here/expected.json")" + +if [[ "$got" != "$want" ]]; then + echo "baseline_extra_inbound differs from expected.json:" >&2 + diff <(echo "$want") <(echo "$got") >&2 || true + exit 1 +fi +# all.yml falls back to two role defaults outside the roles' plays; they must agree. +roles="$here/../../roles" +grep -qE '^decdn_bind_port: 4433([[:space:]]|$)' "$roles/decdn_node/defaults/main.yml" \ + || { echo "decdn_bind_port's default is not 4433: update the fallback in playbooks/group_vars/all.yml" >&2; exit 1; } +grep -qE '^sponsord_onramp_proxy: caddy([[:space:]]|$)' "$roles/sponsord_onramp/defaults/main.yml" \ + || { echo "sponsord_onramp_proxy's default is not caddy: update the fallback in playbooks/group_vars/all.yml" >&2; exit 1; } + +echo "baseline_extra_inbound matches for $(jq -r 'keys | join(", ")' <<<"$want")" diff --git a/ansible/tests/firewall-holes/expected.json b/ansible/tests/firewall-holes/expected.json new file mode 100644 index 0000000..d317da9 --- /dev/null +++ b/ansible/tests/firewall-holes/expected.json @@ -0,0 +1,16 @@ +{ + "node": [{"proto": "udp", "port": 4433, "comment": "decdn QUIC"}], + "node-port": [{"proto": "udp", "port": 5555, "comment": "decdn QUIC"}], + "sponsord": [], + "co-located": [{"proto": "udp", "port": 4433, "comment": "decdn QUIC"}], + "onramp": [ + {"proto": "tcp", "port": 80, "comment": "sponsord-onramp (Caddy, http)"}, + {"proto": "tcp", "port": 443, "comment": "sponsord-onramp (Caddy, https)"} + ], + "onramp-own-proxy": [], + "co-located-onramp": [ + {"proto": "udp", "port": 4433, "comment": "decdn QUIC"}, + {"proto": "tcp", "port": 80, "comment": "sponsord-onramp (Caddy, http)"}, + {"proto": "tcp", "port": 443, "comment": "sponsord-onramp (Caddy, https)"} + ] +} diff --git a/ansible/tests/firewall-holes/inventory.yml b/ansible/tests/firewall-holes/inventory.yml new file mode 100644 index 0000000..5d896b9 --- /dev/null +++ b/ansible/tests/firewall-holes/inventory.yml @@ -0,0 +1,27 @@ +--- +# Fixture for check.sh: one host per group shape. Nothing here is ever contacted; +# check.sh only asks Ansible to resolve each host's variables. +all: + vars: + ansible_connection: local + children: + decdn_nodes: + hosts: + node: {} + node-port: + decdn_bind_port: 5555 + co-located: {} + co-located-onramp: {} + sponsord_hosts: + hosts: + sponsord: {} + co-located: {} + onramp: {} + onramp-own-proxy: {} + co-located-onramp: {} + sponsord_onramp_hosts: + hosts: + onramp: {} + onramp-own-proxy: + sponsord_onramp_proxy: none + co-located-onramp: {} diff --git a/ansible/tests/playbook-guards/inventory.yml b/ansible/tests/playbook-guards/inventory.yml new file mode 100644 index 0000000..fbec66c --- /dev/null +++ b/ansible/tests/playbook-guards/inventory.yml @@ -0,0 +1,14 @@ +--- +# Fixture for tests/scripts-test.sh's playbook-guard cases. Nothing here is ever +# contacted: the cases run only the guard play, which gathers no facts. +all: + vars: + ansible_connection: local + children: + sponsord_hosts: + hosts: + both: {} + sponsord_onramp_hosts: + hosts: + both: {} + onramp-only: {} diff --git a/docs/requirements.md b/docs/requirements.md index 47682b9..d9ec9e2 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -27,7 +27,7 @@ and they run the same roles. | | Supported | How it is tested | |---|---|---| -| OS | Debian 12 (bookworm), Debian 13 (trixie), Ubuntu 24.04 (noble), Ubuntu 26.04 (resolute) | molecule converges `decdn_node` on all four in systemd containers; `grafana_alloy`'s install path on Debian 12 only (its disabled path on all four); `sponsord` on Debian 12 and Ubuntu 24.04; `baseline` on real hosts | +| OS | Debian 12 (bookworm), Debian 13 (trixie), Ubuntu 24.04 (noble), Ubuntu 26.04 (resolute) | molecule converges `decdn_node` on all four in systemd containers; `grafana_alloy`'s install path on Debian 12 only (its disabled path on all four); `sponsord` and `sponsord_onramp` on Debian 12 and Ubuntu 24.04; `baseline` on real hosts | | Architecture | x86_64, aarch64 | upstream builds both; the Ansible role derives the target from the host | | Ansible (control machine) | ansible-core ≥ 2.15 | CI runs the current release | | cloud-init | the provider image's own; the bootstrap installs its pinned ansible-core on the host | the user-data is booted with the distro's cloud-init in Debian 12 and Ubuntu 26.04 containers, to a running node | @@ -42,6 +42,12 @@ On Ubuntu 25.10 and later, `sudo` is sudo-rs; see the `ansible_become_exe` note - **Inbound: udp/4433 (QUIC)** from anywhere. This is the node's only public port. Open it in the host firewall (the Ansible `baseline` does this) **and** in your cloud provider's security group or firewall, which the host cannot see. +- **A sponsord-onramp host (Ansible only) also needs inbound tcp/80 and tcp/443** for + Caddy, from anywhere: 443 serves the onramp, 80 answers the ACME challenge and + redirects to https. `baseline` opens both for hosts in `sponsord_onramp_hosts` while + `sponsord_onramp_proxy` is `caddy` (the default; `none` opens neither); open them in + the provider's firewall too, and point the domain's A/AAAA records at the host. + Caddy comes from the distribution: on Ubuntu, enable the **universe** component. - **Outbound:** HTTPS to your RPC provider and to any cache origin; QUIC/UDP to peers; HTTPS to the iroh relays (upstream's defaults unless you set your own). - **NAT:** a node without direct inbound reachability still works through iroh relays, diff --git a/scripts/sync-network-profiles.py b/scripts/sync-network-profiles.py index a9a8138..3664775 100755 --- a/scripts/sync-network-profiles.py +++ b/scripts/sync-network-profiles.py @@ -8,9 +8,11 @@ would bake into node.toml. Operators then set `decdn_network: ` instead of hand-copying eight addresses that change wholesale on every upstream redeploy. -Also writes ansible/roles/sponsord/vars/main/networks.yml, the subset sponsord needs -(chain_id + the PaymentPool address), from the same parsed data, so the two roles can -never disagree about a chain. --check covers both files. +Also writes the subsets two other roles need, from the same parsed data, so the roles +can never disagree about a chain: ansible/roles/sponsord/vars/main/networks.yml +(chain_id + the PaymentPool address) and ansible/roles/sponsord_onramp/vars/main/ +networks.yml (chain_id + CapacityBond and SlashJudge, which the onramp hands to users' +decdn). --check covers all three files. Everything is read from git at --ref (default origin/main), never from the working tree, so a checkout sitting on a feature branch still yields main's deployment. Two @@ -40,6 +42,7 @@ REPO = Path(__file__).resolve().parent.parent OUT = REPO / "ansible/roles/decdn_node/vars/main/networks.yml" SPONSORD_OUT = REPO / "ansible/roles/sponsord/vars/main/networks.yml" +ONRAMP_OUT = REPO / "ansible/roles/sponsord_onramp/vars/main/networks.yml" # Upstream ChainAddresses field -> decdn_node variable suffix (decdn_). # Order is the output order. @@ -57,6 +60,8 @@ # The FIELDS values sponsord's profile keeps (its role variables are # sponsord_). Order is the output order. SPONSORD_FIELDS = ["payment_pool_address"] +# Likewise for sponsord_onramp (variables sponsord_onramp_). +ONRAMP_FIELDS = ["capacity_bond_address", "slash_judge_address"] ADDRESS = re.compile(r"^0x[0-9a-fA-F]{40}$") REGISTRY = re.compile(r"KNOWN_CHAINS\s*:\s*&\[\s*KnownChain\s*\]\s*=\s*&\[(?P.*?)\];", re.S) @@ -166,7 +171,7 @@ def render(commit: str, profiles: list[dict]) -> str: return "\n".join(lines) + "\n" -def render_sponsord(commit: str, profiles: list[dict]) -> str: +def render_subset(commit: str, profiles: list[dict], role: str, fields: list[str]) -> str: lines = [ "---", "# GENERATED by scripts/sync-network-profiles.py -- do not hand-edit.", @@ -174,15 +179,15 @@ def render_sponsord(commit: str, profiles: list[dict]) -> str: "# crates/cli/src/known_chains.rs (KNOWN_CHAINS, KnownChain::addresses)", "# + contracts/deployments/.json", "#", - "# The subset of roles/decdn_node/vars/main/networks.yml sponsord needs. The role's", - "# chain defaults read this map lazily when sponsord_network is set, so an explicit", + f"# The subset of roles/decdn_node/vars/main/networks.yml {role} needs. The role's", + f"# chain defaults read this map lazily when {role}_network is set, so an explicit", "# inventory value still wins. Protocol facts: upstream's deployment manifests are", "# the source, this file only mirrors them.", - "_sponsord_network_profiles:", + f"_{role}_network_profiles:", ] for p in profiles: lines += [f" {p['name']}:", f" chain_id: {p['chain_id']}"] - lines += [f' {var}: "{p[var]}"' for var in SPONSORD_FIELDS] + lines += [f' {var}: "{p[var]}"' for var in fields] return "\n".join(lines) + "\n" @@ -198,7 +203,11 @@ def main() -> None: args = ap.parse_args() commit, profiles = load(args.upstream, args.ref) - outputs = [(OUT, render(commit, profiles)), (SPONSORD_OUT, render_sponsord(commit, profiles))] + outputs = [ + (OUT, render(commit, profiles)), + (SPONSORD_OUT, render_subset(commit, profiles, "sponsord", SPONSORD_FIELDS)), + (ONRAMP_OUT, render_subset(commit, profiles, "sponsord_onramp", ONRAMP_FIELDS)), + ] if args.check: stale = False for out, text in outputs: @@ -212,6 +221,7 @@ def main() -> None: sys.exit(1) return for out, text in outputs: + out.parent.mkdir(parents=True, exist_ok=True) out.write_text(text) print(f"wrote {out.relative_to(REPO)}") diff --git a/tests/scripts-test.sh b/tests/scripts-test.sh index 47c0533..f5ec4c0 100755 --- a/tests/scripts-test.sh +++ b/tests/scripts-test.sh @@ -1,10 +1,12 @@ #!/usr/bin/env bash # Tests for the repo's own guard rails that no molecule scenario or chart render # exercises: the ansible/ Makefile's scoping guards, the release gate, the -# lint-compose and lint-cloud-init invariants (negative cases), and — with +# lint-compose and lint-cloud-init invariants (negative cases), the firewall holes +# playbooks/group_vars/ derives per host, and — with # UPSTREAM= — the upstream-mirror generators' exit codes. # `make test-scripts` runs it; CI's `scripts` job does too. Needs make, docker -# (compose v2), jq, flock; the cloud-init cases also need cloud-init, shellcheck and yq. +# (compose v2), jq, flock; the cloud-init cases also need cloud-init, shellcheck and yq, +# the firewall-holes case ansible-core. set -euo pipefail repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" @@ -161,6 +163,38 @@ else skipped+=("lint-cloud-init negatives (needs cloud-init on PATH; CI installs it)") fi +# --- baseline firewall holes per host shape, and playbook guards --------------------- +if command -v ansible >/dev/null; then + expect 0 "firewall holes resolve per host shape, co-located included" \ + "$repo/ansible/tests/firewall-holes/check.sh" + # playbooks/sponsord.yml refuses an onramp host outside sponsord_hosts. Its guard + # play is lifted out with yq and run alone (running the whole playbook would need + # the roles' collections just to parse), from a directory without ansible.cfg, + # against a fixture nothing connects to. + if command -v yq >/dev/null; then + yq '[.[] | select(.name == "Require every sponsord-onramp host to be a sponsord host")]' \ + "$repo/ansible/playbooks/sponsord.yml" > "$work/guard.yml" + [[ "$(yq 'length' "$work/guard.yml")" == 1 ]] \ + || fail "playbooks/sponsord.yml has no single sponsord-onramp placement play" + guard() { # + (cd "$repo/ansible/tests/playbook-guards" && ANSIBLE_DEPRECATION_WARNINGS=0 \ + ansible-playbook -i inventory.yml "$work/guard.yml" --limit "$1" &2; fail "the onramp-only refusal did not come from the placement check"; } + elif [[ -n ${CI:-} ]]; then + fail "yq is not on PATH in CI; the playbook-guard cases would be skipped" + else + skipped+=("playbook guards (needs yq)") + fi +elif [[ -n ${CI:-} ]]; then + fail "ansible is not on PATH in CI; the firewall-holes check would be skipped" +else + skipped+=("firewall holes and playbook guards (need ansible-core on PATH)") +fi + # --- upstream-mirror generators (optional: needs a decdn/decdn checkout) -------------- if [[ -n "${UPSTREAM:-}" ]]; then expect 0 "network profiles current" "$repo/scripts/sync-network-profiles.py" "$UPSTREAM" --check