diff --git a/.github/workflows/molecule.yml b/.github/workflows/molecule.yml index 9e5fe48..dcd8b89 100644 --- a/.github/workflows/molecule.yml +++ b/.github/workflows/molecule.yml @@ -27,6 +27,7 @@ jobs: runs-on: ubuntu-latest # Docker is preinstalled # ~4m41s at JOBS=3 with 8 scenarios. os-matrix (3 containers) and lifecycle took the # suite to ~10m at JOBS=3 on a local 16-thread box; re-measure from CI's first run. + # sponsord added a 12th scenario (2 containers, Debian 12 + Ubuntu 24.04). # A bound, not a target: without one a wedged privileged systemd container burns # the 360-minute default, and with cancel-in-progress above, some branch-protection # setups read the resulting cancelled check as "not failed" rather than as a failure. diff --git a/.github/workflows/upstream-drift.yml b/.github/workflows/upstream-drift.yml index 866de31..aa249bb 100644 --- a/.github/workflows/upstream-drift.yml +++ b/.github/workflows/upstream-drift.yml @@ -3,7 +3,8 @@ name: Upstream drift # Three things in this repo mirror decdn/decdn and go stale when upstream moves: # - roles/decdn_node/vars/main/networks.yml (contract addresses; upstream -# redeploys change every one of them at once) +# redeploys change every one of them at once) and its subset +# roles/sponsord/vars/main/networks.yml (the same script and check) # - charts/decdn-node/files/monitoring/ (dashboards + alert rules) # - ansible/molecule/schema/files/schema-keys.txt (the node.toml key inventory) # This job regenerates each from upstream main and fails when a committed copy diff --git a/AGENTS.md b/AGENTS.md index e923c27..7d1ccb4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -54,8 +54,8 @@ mirror (`vars/main/networks.yml`), never hand-copied into inventory or docs. ``` ansible/ # the deployment project (DevSec-hardened, lean roles) - playbooks/ # site.yml (decdn node), backup.yml, decommission.yml - roles/ # baseline, decdn_node, grafana_alloy + playbooks/ # site.yml (decdn node + sponsord), sponsord.yml, backup.yml, decommission.yml + roles/ # baseline, decdn_node, grafana_alloy, sponsord inventory/ galaxy/ molecule/ # see ansible/README.md cloud-init/ # user-data.yaml + on-host bootstrap.sh; pinned ansible-core/collections (see its README.md) compose/ # Docker Compose deploy path for a single host (see its README.md) @@ -69,7 +69,8 @@ scripts/ # upstream-mirror generators + the release gate ``` **Generated mirrors of upstream — regenerate, never hand-edit:** -`ansible/roles/decdn_node/vars/main/networks.yml` (`scripts/sync-network-profiles.py`), +`ansible/roles/decdn_node/vars/main/networks.yml` and its subset +`ansible/roles/sponsord/vars/main/networks.yml` (both `scripts/sync-network-profiles.py`), `charts/decdn-node/files/monitoring/` (`scripts/sync-monitoring.sh`) and `ansible/molecule/schema/files/schema-keys.txt` (`gen-schema-keys.py`). The weekly `upstream-drift` workflow flags staleness. These are the only protocol facts (contract @@ -125,6 +126,36 @@ addresses) the repo carries, and they carry their upstream commit. everything, so only the real pinned binary proves the rendered config loads. See `ansible/roles/grafana_alloy/README.md`. +- **`ansible/roles/sponsord`** — the deCDN onboarding sponsor (`decdn/sponsord`): the + treasury wallet's signer for capped capabilities plus the PaymentPool keeper. It is + independent of the node, so `playbooks/sponsord.yml` (`make deploy-sponsord`, also + imported by `site.yml`) targets its own `sponsord_hosts` group, standalone or + co-located with a node. + - **Install:** manual binary by default, or a GPG-verified `sponsord-v*` release + (none cut yet). + - **Unit:** `DynamicUser`; the API token (generated on the host, never replaced), + treasury keystore and password (operator-provisioned) are `LoadCredential=` + credentials, never env. The keystore is re-copied at 0600 into the unit's + `RuntimeDirectory` by `ExecStartPre`, because newer systemd (255) writes credentials 0440 + and upstream rejects a group-readable keystore. Debian 12's systemd 252 hides + this (0400); the `molecule/sponsord` Ubuntu 24.04 platform catches it, because + the stub enforces upstream's keystore mode check. + - **RPC URL:** in `0600 /etc/sponsord/secret.env`, with the same dual-home and + provenance guards as `decdn_rpc_url`. A host-provisioned file may carry + `SPONSORD_RPC_URL` only (an EnvironmentFile overrides every other setting). + - **Listener:** loopback only (asserted), no firewall hole. + - **Config gate:** sponsord has no `config validate` and only env/flag config, so + `/healthz` after start is the gate, and it is **fatal**. The daemon binds only + after the keystore decrypts and the on-chain pool-owner check passes. + - **Alloy toggles:** `playbooks/group_vars/all.yml` derives + `grafana_alloy_node_enabled` / `grafana_alloy_sponsord_enabled` from group + membership. They are host-scoped so a co-located host's two plays render one + config. + - **Molecule:** under docker, `/` must be `--make-rshared` (see + `molecule/sponsord/prepare.yml`) or every credential directory is empty. + - **Not in cloud-init:** its lint only allows `decdn_nodes`. + - The public `sponsord-onramp` is not deployed yet. + - **`cloud-init/`** — the Ansible path with no control machine. `user-data.yaml` carries only public material (the lint refuses secret-looking keys, credentials in URLs and unknown `bootstrap.env` keys) and a stage-1 `decdn-bootstrap`. That script clones this @@ -190,18 +221,20 @@ make security # KICS over ansible/, the rendered chart and compose/ # Ansible — run from ansible/ make deps # vendor pinned Galaxy collections make check / deploy # site.yml; fleet-wide unless LIMIT=; INVENTORY= +make check-sponsord / deploy-sponsord # sponsord.yml only (sponsord_hosts) make backup / decommission LIMIT=… # lifecycle playbooks (decommission requires LIMIT) make build / galaxy-check # the decdn.node collection ``` **Inventory is private; the firewall hole is not.** This repo is public, so `ansible/inventory/hosts.yml` is git-ignored and a real fleet lives in a private overlay. -Inventory-adjacent group_vars do not load for an overlay, so anything every node needs regardless of inventory (today only the udp/4433 -`baseline_extra_inbound` hole) lives in `ansible/playbooks/group_vars/decdn_nodes.yml`. +Inventory-adjacent group_vars do not load for an overlay, so anything every node needs regardless of inventory (today the udp/4433 +`baseline_extra_inbound` hole) lives in `ansible/playbooks/group_vars/decdn_nodes.yml`, and +the Alloy per-daemon toggles every host needs in `ansible/playbooks/group_vars/all.yml`. Don't move it back under `inventory/`. -**Galaxy collection (`decdn.node`).** The three roles (`baseline` + `decdn_node` + -`grafana_alloy`) ship as a +**Galaxy collection (`decdn.node`).** The four roles (`baseline` + `decdn_node` + +`grafana_alloy` + `sponsord`) ship as a distributable collection. The overlay lives in `ansible/galaxy/` and is staged into a clean collection tree by `galaxy/build.sh` — there is **no** `galaxy.yml` at the `ansible/` root (that would make ansible-lint treat the deploy project as a collection). Build/validate with diff --git a/README.md b/README.md index d07c5d5..bf28628 100644 --- a/README.md +++ b/README.md @@ -97,7 +97,7 @@ This is the canonical statement; the per-path READMEs add only what is specific | Path | What it is | |------|------------| -| [`ansible/`](ansible/README.md) | The Ansible project: `inventory/`, `playbooks/` (`site.yml`, `backup.yml`, `decommission.yml`), `roles/` (`baseline`, `decdn_node`, `grafana_alloy`), `galaxy/` (the `decdn.node` collection), `molecule/`. | +| [`ansible/`](ansible/README.md) | The Ansible project: `inventory/`, `playbooks/` (`site.yml`, `sponsord.yml`, `backup.yml`, `decommission.yml`), `roles/` (`baseline`, `decdn_node`, `grafana_alloy`, `sponsord`), `galaxy/` (the `decdn.node` collection), `molecule/`. | | [`cloud-init/`](cloud-init/README.md) | The cloud-init deploy path: `user-data.yaml`, the on-host `bootstrap.sh`, and the pinned ansible-core and collections it installs. | | [`compose/`](compose/README.md) | The Docker Compose deploy path. | | [`charts/decdn-node/`](charts/decdn-node/README.md) | The Helm chart, with vendored dashboards and alert rules in `files/monitoring/`. | diff --git a/ansible/Makefile b/ansible/Makefile index f8b1ee6..c2ef7ee 100644 --- a/ansible/Makefile +++ b/ansible/Makefile @@ -1,6 +1,6 @@ # Convenience targets for the deCDN Ansible project. # Always run from the ansible/ directory. -.PHONY: deps lint check deploy backup decommission molecule molecule-serial molecule-fanout molecule-all build galaxy-check +.PHONY: deps lint check deploy check-sponsord deploy-sponsord backup decommission molecule molecule-serial molecule-fanout molecule-all build galaxy-check SHELL := /bin/bash # Install the required Galaxy collections (>= constraints in requirements.yml) @@ -43,20 +43,20 @@ INVENTORY ?= inventory/hosts.yml # A missing or unparseable inventory fails the run. Ansible's default only warns, # skips the `hosts: decdn_nodes` play and exits 0, so a mistyped INVENTORY= would # report a successful deploy that did nothing. This applies to the playbook targets -# (check/deploy/backup/decommission) only: +# (check/deploy/check-sponsord/deploy-sponsord/backup/decommission) only: # ansible.cfg can't set it globally, because ansible-lint's own syntax checks run # with no inventory present. strict_inventory = ANSIBLE_INVENTORY_UNPARSED_FAILED=True # Fail loud on the two ways a run can look scoped but silently be fleet-wide. Scoped to -# the playbook targets (check/deploy/backup/decommission) via MAKECMDGOALS so an +# the playbook targets (check/deploy[-sponsord]/backup/decommission) via MAKECMDGOALS so an # unrelated `make lint` is never blocked. NOTE: this # does not catch an argument that is well-formed but selects nothing — `--tags` with a # typo (decdn-node vs decdn_node) runs zero tasks and still exits 0. See ansible/README.md. -ifneq ($(filter check deploy backup decommission,$(MAKECMDGOALS)),) +ifneq ($(filter check deploy check-sponsord deploy-sponsord backup decommission,$(MAKECMDGOALS)),) # Echo back the goal the operator actually typed, so the hint never nudges a # `make check` dry-run toward the mutating `make deploy`. - scoped_goal := $(firstword $(filter check deploy backup decommission,$(MAKECMDGOALS))) + scoped_goal := $(firstword $(filter check deploy check-sponsord deploy-sponsord backup decommission,$(MAKECMDGOALS))) ifeq ($(origin LIMIT),environment) $(error LIMIT is set in your environment ($(LIMIT)) and would silently scope this run. Unset it or pass it on the command line: make $(scoped_goal) LIMIT=) endif @@ -92,6 +92,14 @@ check: deploy: $(strict_inventory) ansible-playbook -i '$(INVENTORY)' playbooks/site.yml $(limit_arg) $(ANSIBLE_ARGS) +# sponsord alone (playbooks/sponsord.yml, hosts in sponsord_hosts). check/deploy +# above already include it via site.yml; these leave every decdn-node untouched. +check-sponsord: + $(strict_inventory) ansible-playbook -i '$(INVENTORY)' playbooks/sponsord.yml --check --diff $(limit_arg) $(ANSIBLE_ARGS) + +deploy-sponsord: + $(strict_inventory) ansible-playbook -i '$(INVENTORY)' playbooks/sponsord.yml $(limit_arg) $(ANSIBLE_ARGS) + # Encrypted backup of each node (identity by default; -e decdn_backup_scope=full for # the whole data dir, which stops the node for the copy). One host at a time. # Needs decdn_backup_age_recipients. See docs/lifecycle.md. @@ -104,16 +112,17 @@ decommission: $(strict_inventory) ansible-playbook -i '$(INVENTORY)' playbooks/decommission.yml $(limit_arg) $(ANSIBLE_ARGS) # --- Tests ------------------------------------------------------------------- -# Containerised converge + idempotence + verify of the decdn_node role against a -# stub daemon (needs Docker; a privileged systemd container). See molecule/. +# Containerised converge + idempotence + verify of the roles against stub daemons +# (needs Docker; privileged systemd containers). See molecule/. # Scenarios: default (rendered values + idempotence), schema (config key-set drift -# against the upstream field list), validation (bad knobs, both roles, must be +# against the upstream field list), validation (bad knobs, every role, must be # rejected by their own asserts), generate-keystore (opt-in host-side wallet), # host-env (host-provisioned /etc/decdn/decdn.env), slow-readiness (advisory # /metrics probe timeout), grafana-cloud and grafana-cloud-token (opt-in Grafana # Cloud observability, host-provisioned vs inventory token), os-matrix (default's -# plays on Debian 13, Ubuntu 24.04 and 26.04) and lifecycle (network profile, -# backup + decommission). The glob below is the authority; this list is a guide. +# plays on Debian 13, Ubuntu 24.04 and 26.04), lifecycle (network profile, +# backup + decommission) and sponsord (the sponsord role on a host with no node, +# through playbooks/sponsord.yml, on Debian 12 and Ubuntu 24.04). The glob below is the authority; this list is a guide. # # The scenarios run concurrently: distinct container names, no published host ports, # per-scenario ephemeral dirs, and what they share on the control machine (the @@ -200,8 +209,8 @@ molecule-all: deps molecule test --all # --- Galaxy collection (decdn.node) ------------------------------------------ -# Stage baseline + decdn_node + grafana_alloy into a clean collection tree and build the -# artifact under build/. Only the three roles ship; see galaxy/README.md. Publishing +# Stage baseline + decdn_node + grafana_alloy + sponsord into a clean collection tree and +# build the artifact under build/. Only the four roles ship; see galaxy/README.md. Publishing # runs from a vX.Y.Z tag in .github/workflows/release.yml (see RELEASING.md). build: ./galaxy/build.sh diff --git a/ansible/README.md b/ansible/README.md index d65697a..9c4057b 100644 --- a/ansible/README.md +++ b/ansible/README.md @@ -8,13 +8,16 @@ hardened host baseline. The roles also ship as the `decdn.node` Galaxy collectio | **`site.yml`** | Harden the host and deploy the public **deCDN node** (`decdn-node`). | `make check` / `make deploy` | | `backup.yml` | Encrypted backup of a node's identity or full state. | `make backup` | | `decommission.yml` | Stop a node and remove its service (keeps the identity; no on-chain steps). | `make decommission` | +| `sponsord.yml` | Harden the host and deploy **sponsord**, the onboarding sponsor, on hosts in `sponsord_hosts`, with or without a node. Also run by `site.yml`. | `make check-sponsord` / `make deploy-sponsord` | ``` baseline host hardening: DevSec os/ssh, nftables default-deny inbound, │ fail2ban, unattended-upgrades, chrony, an admin sudo user ├─ grafana_alloy opt-in Grafana Cloud agent (metrics, journald, traces), loopback-only - └─ decdn_node public QUIC udp/4433; metrics + admin loopback; signed-tarball or - local-build install; hardened systemd unit; backup/decommission + ├─ decdn_node public QUIC udp/4433; metrics + admin loopback; signed-tarball or + │ local-build install; hardened systemd unit; backup/decommission + └─ sponsord treasury signer + PaymentPool keeper; loopback API; DynamicUser + unit with systemd credentials; standalone or beside a node ``` ## Security model @@ -245,6 +248,25 @@ guardrails and rollback: [`roles/grafana_alloy/README.md`](roles/grafana_alloy/R To import upstream's decdn dashboards and alert rules into that stack, see [`charts/decdn-node/files/monitoring/`](../charts/decdn-node/files/monitoring/README.md). +### sponsord (optional) + +The fourth role, `sponsord`, deploys the deCDN onboarding sponsor. It holds the +treasury wallet that owns a PaymentPool pool, signs capabilities for its onramp and +keeps the pool topped up. It needs no node. Put its host in the `sponsord_hosts` group: +a dedicated host or a node host listed in both groups. Then run +`make deploy-sponsord`; `make deploy` covers it too. + +- **You provision** the treasury keystore and password on the host. The role never + creates that wallet. +- **The API token** is generated on the host. +- **The RPC URL** is dual-homed like `decdn_rpc_url`. +- **Observability:** with Grafana Cloud on, Alloy scrapes sponsord on hosts in + `sponsord_hosts` and the node on hosts in `decdn_nodes` + (`playbooks/group_vars/all.yml`). + +Setup and variables: [`roles/sponsord/README.md`](roles/sponsord/README.md). The public +`sponsord-onramp` is not deployed by this repo yet. + ### Backup and decommission ```bash @@ -273,15 +295,18 @@ make lint-alloy # render grafana_alloy's templates, then `alloy validate` th `make molecule` runs every scenario under `molecule/`: **`default`** (described below), `schema` (config key-set drift against the upstream field list), `validation` (bad knobs, -for both roles, must be rejected by their own asserts), `generate-keystore` (opt-in +for every role, must be rejected by their own asserts), `generate-keystore` (opt-in host-side wallet), `host-env` (host-provisioned `/etc/decdn/decdn.env`), `slow-readiness` (advisory `/metrics` probe timeout), `grafana-cloud` (the opt-in observability wiring — see [Grafana Cloud observability](#grafana-cloud-observability-opt-in)) `grafana-cloud-token` (the same wiring with the API token carried through git-ignored inventory instead: role-authored env file + provenance record), `os-matrix` (default's plays on Debian 13, Ubuntu 24.04 and Ubuntu 26.04; `default` -itself is Debian 12) and `lifecycle` (a `decdn_network` profile with an override, both -backup scopes decrypted and checked, a rejected and a real decommission). +itself is Debian 12), `lifecycle` (a `decdn_network` profile with an override, both +backup scopes decrypted and checked, a rejected and a real decommission) and +`sponsord` (`playbooks/sponsord.yml` on a host with no node: generated token, credential +rotation restart, the fatal `/healthz` gate, Alloy scraping sponsord and not the node; +`grafana-cloud` covers sponsord co-located with a node). They are independent, so they run concurrently, and each line of output is prefixed with its scenario name because the runs interleave. `make molecule-serial` is the escape hatch when that interleaving gets in the way of reading a failure. @@ -289,7 +314,7 @@ interleaving gets in the way of reading a failure. `grafana-cloud` runs against a *stub* Alloy that exits 0 for every subcommand, so it proves the role's plumbing but cannot prove the rendered `config.alloy` is loadable. That gap is closed by `make lint-alloy` (repo root; CI job `alloy-config`), which renders -the templates in eight variable combinations — including one with every observability +the templates in twelve variable combinations — including one with every observability sub-knob off, which must reproduce the pre-machine-monitoring pipeline exactly — and runs the **real**, digest-pinned Alloy binary's `alloy validate` over them plus a check that every `ExecStart` flag actually @@ -335,12 +360,12 @@ the RPC URL when it is not provisioned on the host instead). Highlights: ## Packaging as a Galaxy collection (`decdn.node`) -The three roles (`baseline`, `decdn_node`, `grafana_alloy`) are also packaged as the +The four roles (`baseline`, `decdn_node`, `grafana_alloy`, `sponsord`) are also packaged as the distributable **`decdn.node`** collection, for operators who bring their own playbooks. The collection overlay lives in [`galaxy/`](galaxy/) (`galaxy.yml`, the collection `README.md`/`CHANGELOG.md`, `meta/runtime.yml`, `build.sh`). It is deliberately **not** -a `galaxy.yml` at the project root: `galaxy/build.sh` stages only the three roles into a +a `galaxy.yml` at the project root: `galaxy/build.sh` stages only the four roles into a clean `ansible_collections/decdn/node/` tree and builds the artifact, so this project stays a plain Ansible project (the `make deploy`/`lint` flow is unchanged). diff --git a/ansible/galaxy/CHANGELOG.md b/ansible/galaxy/CHANGELOG.md index 66f5a45..7112cbf 100644 --- a/ansible/galaxy/CHANGELOG.md +++ b/ansible/galaxy/CHANGELOG.md @@ -8,6 +8,41 @@ collection adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) ### Added +- `sponsord` role: deploys the deCDN onboarding sponsor (decdn/sponsord) on its own + host or beside a node. It installs a local binary (`sponsord_install_method: manual`, + the default until upstream tags a release) or a `sponsord-v` release + verified against its GPG-signed `SHA256SUMS`. The unit uses `DynamicUser`, and the + API token, treasury keystore and password reach it as `LoadCredential=` + credentials. Newer systemd (255 on Ubuntu 24.04) writes credentials `0440` and sponsord refuses a + group-readable keystore, so the keystore is re-copied `0600` into the unit's + `RuntimeDirectory` at start. The API token is generated on the host and never + replaced. The RPC + URL comes from `sponsord_rpc_url` or a host-provisioned `0600 + /etc/sponsord/secret.env`, with the same provenance guards as `decdn_rpc_url`. A + host-provisioned file may carry `SPONSORD_RPC_URL` only, because an + `EnvironmentFile` would override every validated setting. +- The listener is loopback only. The deploy fails unless `/healthz` answers + `{"ok":true}` from a listener owned by the running unit. +- A hash record of every restart input (credentials, env files, unit, binary) + restarts the daemon after a rotation, an out-of-band edit, or an earlier failed + run. +- In release mode, the version stamp is written only after an exact `--version` + match. + `sponsord_network` takes the chain and PaymentPool from + `roles/sponsord/vars/main/networks.yml`, generated by + `scripts/sync-network-profiles.py`. +- `grafana_alloy_sponsord_enabled` (default `false`): scrape sponsord's `/metrics` + as `job="sponsord"` with its own `service_name`. Its journald stream gets the + same `service_name`, and a `level` label parsed from each line, because journald + files all of a daemon's stdout as info. Like decdn-node, it is exempt from the + journald-priority guardrail and judged by that parsed level instead. +- `grafana_alloy_node_enabled` (default `true`) turns the decdn-node scrape off on + a host without a node. The repo's playbooks derive both settings from + `sponsord_hosts` / `decdn_nodes`. +- The systemd collector's default unit list now includes `sponsord.service`. That + changes `config.alloy` once on every Grafana-enabled host, so expect one Alloy + restart on the next deploy. + - `decdn_get_logs_max_block_span` (default `""`, daemon default 10000, `>= 1`): the ceiling on one chain-watcher `eth_getLogs` block span, rendered as `blockchain.get_logs_max_block_span` (decdn/decdn @ 3ebf5f17). The poller halves diff --git a/ansible/galaxy/README.md b/ansible/galaxy/README.md index 06c1f86..7e35a2d 100644 --- a/ansible/galaxy/README.md +++ b/ansible/galaxy/README.md @@ -2,13 +2,14 @@ Deploy and harden a **public [deCDN](https://decdn.org) node**. This collection is the public, reusable slice of the [`decdn/devops`](https://github.com/decdn/devops) -repository — three roles and nothing else: +repository — four roles and nothing else: | Role | Purpose | |------|---------| | `decdn.node.baseline` | Debian/Ubuntu host baseline — nftables default-deny inbound, fail2ban, unattended-upgrades, chrony, an admin sudo user, then DevSec OS + SSH hardening (applied last). | | `decdn.node.decdn_node` | The `decdn-node` daemon under a hardened systemd unit — from locally built binaries (the default until upstream tags a release) or a GPG-verified release tarball; public QUIC udp/4433, loopback metrics + admin RPC. `decdn_network` sets the chain from upstream's manifest; `tasks_from: backup` / `decommission` for day 2. | -| `decdn.node.grafana_alloy` | Opt-in Grafana Cloud observability agent — loopback-only Alloy receiver and hardened telemetry export. | +| `decdn.node.grafana_alloy` | Opt-in Grafana Cloud observability agent — loopback-only Alloy receiver and hardened telemetry export; scrapes the node, `sponsord`, or both (`grafana_alloy_node_enabled` / `grafana_alloy_sponsord_enabled`). | +| `decdn.node.sponsord` | The `sponsord` onboarding sponsor (treasury signer + PaymentPool keeper), standalone or beside a node — local binary or GPG-verified `sponsord-v*` release, `DynamicUser` unit with the API token and treasury wallet as systemd credentials, loopback-only API, `/healthz` deploy gate. `sponsord_network` sets the chain and PaymentPool from upstream's manifest. | ## Requirements @@ -79,6 +80,7 @@ full variable list, the eth-keystore prerequisite, and day-2 ops: - [`roles/baseline`](https://github.com/decdn/devops/tree/main/ansible/roles/baseline) - [`roles/decdn_node`](https://github.com/decdn/devops/tree/main/ansible/roles/decdn_node) - [`roles/grafana_alloy`](https://github.com/decdn/devops/tree/main/ansible/roles/grafana_alloy) +- [`roles/sponsord`](https://github.com/decdn/devops/tree/main/ansible/roles/sponsord) ## Security model diff --git a/ansible/galaxy/build.sh b/ansible/galaxy/build.sh index 029b23a..f81f88b 100755 --- a/ansible/galaxy/build.sh +++ b/ansible/galaxy/build.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # Stage and build the public `decdn.node` Galaxy collection. # -# Only the roles/baseline, roles/decdn_node, and roles/grafana_alloy sources ship. All deploy machinery +# Only the roles/baseline, roles/decdn_node, roles/grafana_alloy and roles/sponsord sources ship. All deploy machinery # (inventory, Makefile, ansible.cfg) is excluded BY CONSTRUCTION — it is simply # never copied into the staging tree. This keeps the artifact clean and leaves the # internal project untouched (no galaxy.yml at the project root, so ansible-lint @@ -16,7 +16,7 @@ repo_root="$(cd "$ansible_dir/.." && pwd)" # repo root build_dir="$ansible_dir/build" stage="$build_dir/ansible_collections/decdn/node" -roles=(baseline decdn_node grafana_alloy) +roles=(baseline decdn_node grafana_alloy sponsord) echo "staging decdn.node -> $stage" rm -rf "$stage" diff --git a/ansible/galaxy/galaxy.yml b/ansible/galaxy/galaxy.yml index 9d9f4b1..c54cfcf 100644 --- a/ansible/galaxy/galaxy.yml +++ b/ansible/galaxy/galaxy.yml @@ -3,7 +3,7 @@ # slice of this DevOps repo: deploy + harden a deCDN node. # # This file is an OVERLAY, not the project root manifest. It is assembled into a -# clean collection tree (with the baseline, decdn_node and grafana_alloy roles) by galaxy/build.sh; +# clean collection tree (with the baseline, decdn_node, grafana_alloy and sponsord roles) by galaxy/build.sh; # it is deliberately NOT placed at ansible/ root, so the deploy project stays a # plain Ansible project (bare role names, roles_path, production-profile # ansible-lint) rather than being reinterpreted as a collection. See galaxy/README.md. @@ -15,7 +15,8 @@ authors: - deCDN Contributors description: >- Deploy and harden a public deCDN node: DevSec host baseline, the decdn_node - daemon (with backup and decommission), and opt-in Grafana Cloud observability. + daemon (with backup and decommission), the sponsord onboarding sponsor, and + opt-in Grafana Cloud observability. license: - MIT # Galaxy tags must match ^[a-z0-9]+$ (no hyphens/underscores). @@ -34,6 +35,7 @@ tags: # (authorized_key) # decdn_node -> ansible.builtin only # grafana_alloy -> ansible.builtin only +# sponsord -> ansible.builtin only # community.general is NOT used by any role, so it is intentionally absent here. dependencies: devsec.hardening: ">=10.0.0" diff --git a/ansible/inventory/group_vars/sponsord_hosts.yml b/ansible/inventory/group_vars/sponsord_hosts.yml new file mode 100644 index 0000000..ab348c1 --- /dev/null +++ b/ansible/inventory/group_vars/sponsord_hosts.yml @@ -0,0 +1,37 @@ +--- +# Shared config for sponsord hosts (playbooks/sponsord.yml, also run by site.yml). +# Uncomment and fill in; per-host values can move to host_vars//main.yml. +# Full variable list: roles/sponsord/defaults/main.yml and roles/sponsord/README.md. +# +# Before the first deploy, on the host as root (the role never creates the wallet): +# /etc/sponsord/treasury-keystore.json the treasury keystore (decdn key-gen), 0600 +# /etc/sponsord/treasury-password its password, 0600 +# /etc/sponsord/secret.env SPONSORD_RPC_URL=… (or sponsord_rpc_url in +# a git-ignored host_vars//secret.yml) +# The API token is generated on the host at /etc/sponsord/api-token. + +# --- Binary ------------------------------------------------------------------------ +# Until upstream tags a sponsord-v* release: a decdn/sponsord checkout's target dir +# (`cargo build --release -p sponsord`; the role checks the ELF matches the host). +# sponsord_release_target_dir: /path/to/sponsord/target/release +# Once a release exists: +# sponsord_install_method: release +# sponsord_version: "0.1.0" + +# --- Chain + pool ------------------------------------------------------------------ +# chain_id and the PaymentPool address from upstream's deployment manifest. +# sponsord_network: arbitrum-sepolia +# The pool `decdn pool open` printed when the treasury wallet opened it (REQUIRED). +# sponsord_pool_id: "0x…" + +# --- Limits ("" = the daemon's default) --------------------------------------------- +# sponsord_max_spending_cap_micro_usdc: "" # [5000000 = $5] +# sponsord_max_ttl_secs: "" # [172800 = 48h] +# sponsord_pool_low_water_micro_usdc: "" # [20000000 = $20] +# sponsord_pool_refill_micro_usdc: "" # [100000000 = $100] + +# --- Grafana Cloud (opt-in) -------------------------------------------------------- +# Same agent as the node's. On a sponsord-only host the region label has no +# decdn_region to fall back on, so set it here. +# decdn_grafana_cloud_enabled: true +# grafana_alloy_region: US diff --git a/ansible/inventory/host_vars/decdn-node-1/secret.yml.example b/ansible/inventory/host_vars/decdn-node-1/secret.yml.example index 5593255..402a097 100644 --- a/ansible/inventory/host_vars/decdn-node-1/secret.yml.example +++ b/ansible/inventory/host_vars/decdn-node-1/secret.yml.example @@ -40,3 +40,11 @@ decdn_rpc_url: "https://sepolia-rollup.arbitrum.io/rpc" # commented out to keep the operator-provisioned host path instead. # # grafana_alloy_api_token: "glc_example_token_replace_me" + +# --- sponsord (only on a host in sponsord_hosts) -------------------------------- +# +# Same choice for sponsord's RPC endpoint: set it here and the role authors +# /etc/sponsord/secret.env (root 0600), or leave it commented out and provision that +# file on the host (roles/sponsord/files/secret.env.example). SENSITIVE. +# +# sponsord_rpc_url: "https://sepolia-rollup.arbitrum.io/rpc" diff --git a/ansible/inventory/hosts.yml.example b/ansible/inventory/hosts.yml.example index b39a7a1..e4e97d5 100644 --- a/ansible/inventory/hosts.yml.example +++ b/ansible/inventory/hosts.yml.example @@ -50,3 +50,13 @@ decdn_nodes: # big_disk: # hosts: # decdn-node-2: + +# OPTIONAL: sponsord, the onboarding sponsor (treasury signer + PaymentPool keeper) — +# provisioned by playbooks/sponsord.yml (`make deploy-sponsord`) and by site.yml. It +# needs no node: give it its own host, or list a node host here as well to co-locate +# them. Its knobs are in group_vars/sponsord_hosts.yml; the treasury keystore and +# password are provisioned on the host (roles/sponsord/README.md). +# sponsord_hosts: +# hosts: +# sponsord-1: +# ansible_host: REPLACE_WITH_SPONSORD_VPS_IP_OR_DNS diff --git a/ansible/molecule/grafana-cloud/converge.yml b/ansible/molecule/grafana-cloud/converge.yml index 6b8d0a6..8016d83 100644 --- a/ansible/molecule/grafana-cloud/converge.yml +++ b/ansible/molecule/grafana-cloud/converge.yml @@ -1,7 +1,10 @@ --- # Positive path only — see molecule.yml for scope and ../validation for the -# negative (block/rescue) matrix. Both roles run in sequence: Alloy first so its -# receivers exist before the daemon is started and exports spans at them. +# negative (block/rescue) matrix. The roles run in sequence: Alloy first so its +# receivers exist before the daemon is started and exports spans at them, then +# sponsord on the same host — the CO-LOCATED shape (a host in both decdn_nodes and +# sponsord_hosts), so Alloy must scrape both daemons. ../sponsord covers a +# sponsord-only host. # # The stub does not serve /metrics, so decdn_node's readiness probe would sit out # its full advisory window (~60s of retries); decdn_readiness_* below bounds that @@ -53,8 +56,22 @@ # so every gate and the template must coerce it instead of taking its length # or emitting a bare number into a string field. grafana_alloy_prom_username: 1234567 + + # --- Co-located sponsord (what playbooks/group_vars/all.yml derives for a host + # in both groups; set by hand here because this converge uses roles, not the + # playbook) ------------------------------------------------------------------- + grafana_alloy_sponsord_enabled: true + sponsord_install_method: manual + sponsord_manual_bin_src: "{{ _proj_dir }}/molecule/sponsord/files/sponsord-stub" + sponsord_network: arbitrum-sepolia + sponsord_pool_id: "0x00000000000000000000000000000000000000000000000000000000000000aa" + sponsord_rpc_url: "https://rpc.example.invalid/" + sponsord_readiness_retries: 10 + sponsord_readiness_delay: 1 roles: - role: grafana_alloy tags: [observability] - role: decdn_node tags: [decdn_node, node] + - role: sponsord + tags: [sponsord] diff --git a/ansible/molecule/grafana-cloud/prepare.yml b/ansible/molecule/grafana-cloud/prepare.yml index 0447cb4..357627d 100644 --- a/ansible/molecule/grafana-cloud/prepare.yml +++ b/ansible/molecule/grafana-cloud/prepare.yml @@ -1,14 +1,23 @@ --- -# Stage the host-provisioned fixtures both roles' gates demand: +# Stage the host-provisioned fixtures all three roles' gates demand: # 1. decdn_node's operator keystore + node identity + password (same trick as # ../default/prepare.yml — placeholders at 0644 that the role locks to 0600). # 2. The Grafana Cloud credential env file for grafana_alloy, which unlike those # is asserted ALREADY-SECURED by preflight (the role never rewrites it), so # stage it directly at root:root 0600 with all four required keys. +# 3. sponsord's operator-provisioned treasury keystore + password, plus shared +# propagation on / so its LoadCredential= directories are not empty (see +# ../sponsord/prepare.yml: a container-only step). - name: Prepare hosts: all become: true tasks: + # ansible.posix.mount cannot change propagation, hence the command. + - name: Give / shared mount propagation, as systemd does on a real host # noqa: command-instead-of-module + ansible.builtin.command: + cmd: mount --make-rshared / + changed_when: true + - name: Ensure data + config directories exist ansible.builtin.file: path: "{{ item }}" @@ -17,6 +26,18 @@ loop: - /var/lib/decdn - /etc/decdn + - /etc/sponsord + + - name: Stage placeholder sponsord treasury keystore + password + ansible.builtin.copy: + dest: "{{ item.dest }}" + content: "{{ item.content }}" + mode: "0600" + loop: + - {dest: /etc/sponsord/treasury-keystore.json, content: "{{ '{}' }}\n"} + - {dest: /etc/sponsord/treasury-password, content: "molecule-placeholder\n"} + loop_control: + label: "{{ item.dest }}" # Staged looser than target so verify.yml can attribute the 0600 lock-down # to the role rather than to what prepare pre-set. diff --git a/ansible/molecule/grafana-cloud/verify.yml b/ansible/molecule/grafana-cloud/verify.yml index d361e3a..9171e8a 100644 --- a/ansible/molecule/grafana-cloud/verify.yml +++ b/ansible/molecule/grafana-cloud/verify.yml @@ -46,6 +46,13 @@ '127.0.0.1:4318' in ga_config - >- '127.0.0.1:9090' in ga_config + # Co-located sponsord: its own scrape beside the node's. + - >- + 'prometheus.scrape "decdn_node"' in ga_config + - >- + 'prometheus.scrape "sponsord"' in ga_config + - >- + '"__address__" = "127.0.0.1:8090"' in ga_config - >- '0.0.0.0' not in ga_config - >- @@ -317,7 +324,7 @@ The alloy account is not a member of systemd-journal and adm, so loki.source.journal would start without error and collect nothing. - - name: Assert both services are running + - name: Assert every service is running ansible.builtin.assert: that: - >- @@ -326,7 +333,12 @@ - >- 'decdn-node.service' in ansible_facts.services and ansible_facts.services['decdn-node.service'].state == 'running' - fail_msg: "alloy or decdn-node is not running — journalctl -u alloy -e / -u decdn-node -e" + - >- + 'sponsord.service' in ansible_facts.services + and ansible_facts.services['sponsord.service'].state == 'running' + fail_msg: >- + alloy, decdn-node or sponsord is not running — journalctl -u alloy -e / + -u decdn-node -e / -u sponsord -e # A host-managed env file is never rendered by the role, so only the provenance # checksum can notice an out-of-band rotation. Prove that signal restarts the @@ -379,6 +391,7 @@ grafana_alloy_loki_url: https://logs-prod-xx.molecule.invalid/loki/api/v1/push grafana_alloy_prom_username: 1234567 grafana_alloy_batch_send_size: 512 + grafana_alloy_sponsord_enabled: true - name: Read Alloy's PID after the host-side credential edit ansible.builtin.command: diff --git a/ansible/molecule/sponsord/converge.yml b/ansible/molecule/sponsord/converge.yml new file mode 100644 index 0000000..c2e0d63 --- /dev/null +++ b/ansible/molecule/sponsord/converge.yml @@ -0,0 +1,7 @@ +--- +# The real standalone entry point, not include_role: this is what proves +# playbooks/group_vars/all.yml turns the node scrape off on a sponsord-only host. +# Variables come from molecule.yml's inventory group_vars (sponsord_hosts); baseline +# is skipped there. +- name: Converge + ansible.builtin.import_playbook: ../../playbooks/sponsord.yml diff --git a/ansible/molecule/sponsord/files/sponsord-stub b/ansible/molecule/sponsord/files/sponsord-stub new file mode 100755 index 0000000..5f41da7 --- /dev/null +++ b/ansible/molecule/sponsord/files/sponsord-stub @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +"""Stub sponsord binary for the Molecule scenarios. + +Stands in for the real `sponsord` so the role can be exercised end to end without +a release, a chain or a funded treasury: + + * `--version` -> print a version line and exit 0 (the role's version backstop). + * no argument -> enforce the startup contract the real daemon enforces, then + serve /healthz and /metrics on SPONSORD_BIND until killed: + - SPONSORD_RPC_URL, _CHAIN_ID, _PAYMENT_POOL_ADDR, _POOL_ID set + (_CHAIN_ID is stricter than upstream, which defaults it; + the role always renders it); + - SPONSORD_API_TOKEN_FILE readable and >= 32 bytes; + - SPONSORD_TREASURY_PASSWORD_FILE readable and non-empty; + - SPONSORD_TREASURY_KEYSTORE a regular file, not a symlink, + with no group/other permission bits (upstream's check), + holding JSON (catches a copy of the wrong credential). + Any violation exits 1 with the reason on stderr (the journal), + so the role's /healthz gate sees a daemon that never binds. + With SPONSORD_MOLECULE_RPC_LITERAL=1 (a unit drop-in the + sponsord scenario's prepare.yml installs), SPONSORD_RPC_URL must + equal EXPECTED_RPC_URL byte for byte, which proves the role's + EnvironmentFile quoting. + The marker file UNHEALTHY_MARKER makes it exit 1 on start, + which is how a scenario proves the readiness gate is fatal. + * anything else -> exit 2. + +No protocol behaviour: the scenarios verify host prep, secrets, unit hardening +and loopback binding, not capability signing. +""" +import json +import os +import stat +import sys +from http.server import BaseHTTPRequestHandler, HTTPServer + +VERSION = "sponsord 0.0.0-molecule-stub" +UNHEALTHY_MARKER = "/etc/sponsord/stub-unhealthy" +# Must match sponsord_rpc_url in the scenario's inventory, which carries shell +# metacharacters and quotes on purpose. +EXPECTED_RPC_URL = "https://rpc.example.invalid/?key=$HOME$(touch /tmp/sponsord-rpc-expanded)&q=o'brien \"x\"" + + +def die(msg): + print(f"sponsord-stub: {msg}", file=sys.stderr) + sys.exit(1) + + +def read_secret(var): + path = os.environ.get(var) + if not path: + die(f"{var} is not set") + try: + with open(path, "rb") as f: + value = f.read() + except OSError as e: + die(f"{var}: cannot read {path}: {e}") + value = value.removesuffix(b"\n").removesuffix(b"\r") + if not value: + die(f"{var}: {path} is empty") + return value + + +def check_startup(): + if os.path.exists(UNHEALTHY_MARKER): + die(f"{UNHEALTHY_MARKER} exists: refusing to start") + for var in ("SPONSORD_RPC_URL", "SPONSORD_CHAIN_ID", "SPONSORD_PAYMENT_POOL_ADDR", "SPONSORD_POOL_ID"): + if not os.environ.get(var): + die(f"{var} is not set") + if os.environ.get("SPONSORD_MOLECULE_RPC_LITERAL") == "1" and os.environ["SPONSORD_RPC_URL"] != EXPECTED_RPC_URL: + die("SPONSORD_RPC_URL does not match the inventory literal (EnvironmentFile quoting broke it)") + if len(read_secret("SPONSORD_API_TOKEN_FILE")) < 32: + die("API token is shorter than 32 bytes") + read_secret("SPONSORD_TREASURY_PASSWORD_FILE") + keystore = os.environ.get("SPONSORD_TREASURY_KEYSTORE") + if not keystore: + die("SPONSORD_TREASURY_KEYSTORE is not set") + try: + st = os.lstat(keystore) + except OSError as e: + die(f"keystore {keystore}: {e}") + if not stat.S_ISREG(st.st_mode): + die(f"keystore {keystore} is not a regular file") + if st.st_mode & 0o077: + die(f"keystore {keystore} is mode {oct(st.st_mode & 0o777)}: group/other bits set") + # A keystore is JSON; a copy of the wrong credential (the password) is not. + try: + with open(keystore, "rb") as f: + json.load(f) + except (OSError, ValueError) as e: + die(f"keystore {keystore} is not JSON: {e}") + + +class Handler(BaseHTTPRequestHandler): + def do_GET(self): + if self.path == "/healthz": + body, ctype = b'{"ok":true}', "application/json" + elif self.path == "/metrics": + body, ctype = b"sponsord_pool_topups_total 0\n", "text/plain; version=0.0.4" + else: + self.send_error(404) + return + self.send_response(200) + self.send_header("Content-Type", ctype) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args): + pass + + +def main(): + if sys.argv[1:] == ["--version"]: + print(VERSION) + return + if sys.argv[1:]: + print(f"sponsord-stub: unexpected arguments {sys.argv[1:]}", file=sys.stderr) + sys.exit(2) + check_startup() + host, _, port = os.environ.get("SPONSORD_BIND", "127.0.0.1:8090").rpartition(":") + HTTPServer((host, int(port)), Handler).serve_forever() + + +if __name__ == "__main__": + main() diff --git a/ansible/molecule/sponsord/molecule.yml b/ansible/molecule/sponsord/molecule.yml new file mode 100644 index 0000000..0b085c9 --- /dev/null +++ b/ansible/molecule/sponsord/molecule.yml @@ -0,0 +1,91 @@ +--- +# sponsord STANDALONE: a host in sponsord_hosts only, no decdn-node anywhere, +# converged through the real playbooks/sponsord.yml (so playbooks/group_vars/all.yml +# derives the Alloy toggles exactly as a deploy would) against a stub daemon +# (files/sponsord-stub) and the grafana-cloud scenario's Alloy stub. +# +# converge the role end to end: host-generated API token, treasury files +# locked to 0600, inventory-authored secret.env (an RPC URL full of +# shell metacharacters the stub checks byte for byte), loopback +# listener, /healthz gate; Alloy scraping sponsord and NOT the node. +# idempotence an unchanged host must not restart anything. +# side_effect rotate the treasury password out of band -> the next converge +# must restart; make the daemon refuse to start -> the /healthz +# gate must FAIL the converge; then put it back. +# verify files, modes, unit hardening, credentials, listener, Alloy config. +# +# `baseline` is skipped (real-host-only — see ../default/molecule.yml). The +# broken-input matrix for this role lives in ../validation. +driver: + name: docker +platforms: + - name: sponsord-molecule + # Same digest-pinned image as the other scenarios (repo convention). Re-resolve + # all of them together to bump. + image: geerlingguy/docker-debian12-ansible@sha256:a131e3b7a9a38dda40e98b49b39d6200ffab0ad853c714a513f5eea41bab9a25 + pre_build_image: true + command: /usr/lib/systemd/systemd + privileged: true + cgroupns_mode: host + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + groups: + - sponsord_hosts + # systemd 255: LoadCredential= files arrive 0440 there (0400 on Debian 12's 252), + # which is what the unit's keystore copy into its RuntimeDirectory exists for. The + # stub enforces upstream's keystore mode check, so a regression fails here. + - name: sponsord-molecule-u2404 + # Same digest-pinned image as ../os-matrix (repo convention). + image: geerlingguy/docker-ubuntu2404-ansible@sha256:3394e335df58c961c00b78ae601ef30cdd7463b112f21b0c5f78674a70526302 + pre_build_image: true + command: /usr/lib/systemd/systemd + privileged: true + cgroupns_mode: host + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + groups: + - sponsord_hosts +provisioner: + name: ansible + options: + skip-tags: baseline + env: + ANSIBLE_ROLES_PATH: "${MOLECULE_PROJECT_DIRECTORY}/roles" + ANSIBLE_COLLECTIONS_PATH: "${MOLECULE_PROJECT_DIRECTORY}/collections" + # Same pipelining note as ../default/molecule.yml. + ANSIBLE_PIPELINING: "true" + inventory: + group_vars: + # Inventory group_vars, not converge vars: the converge is an + # import_playbook, which takes no vars of its own. + sponsord_hosts: + sponsord_install_method: manual + sponsord_manual_bin_src: "${MOLECULE_SCENARIO_DIRECTORY}/files/sponsord-stub" + sponsord_version: "0.0.0-molecule-stub" + sponsord_network: arbitrum-sepolia + sponsord_pool_id: "0x00000000000000000000000000000000000000000000000000000000000000aa" + # Must equal EXPECTED_RPC_URL in files/sponsord-stub. The doubled dollar + # signs are molecule's escape: it interpolates variables in this file. + sponsord_rpc_url: "https://rpc.example.invalid/?key=$$HOME$$(touch /tmp/sponsord-rpc-expanded)&q=o'brien \"x\"" + sponsord_max_ttl_secs: 3600 + sponsord_pool_low_water_micro_usdc: 0 + sponsord_readiness_retries: 10 + sponsord_readiness_delay: 1 + decdn_grafana_cloud_enabled: true + grafana_alloy_install_method: manual + grafana_alloy_manual_bin_src: "${MOLECULE_PROJECT_DIRECTORY}/molecule/grafana-cloud/files/alloy-stub" + grafana_alloy_region: US + grafana_alloy_loki_url: https://logs-prod-xx.molecule.invalid/loki/api/v1/push +verifier: + name: ansible +scenario: + test_sequence: + # Leading destroy: see ../default/molecule.yml. + - destroy + - create + - prepare + - converge + - idempotence + - side_effect + - verify + - destroy diff --git a/ansible/molecule/sponsord/prepare.yml b/ansible/molecule/sponsord/prepare.yml new file mode 100644 index 0000000..902996c --- /dev/null +++ b/ansible/molecule/sponsord/prepare.yml @@ -0,0 +1,158 @@ +--- +# Stage what an operator provisions before the first deploy: +# 1. the treasury keystore + password (placeholders, staged at 0644 so verify.yml +# can attribute the 0600 lock-down to the role); +# 2. the Grafana Cloud credential env file (as ../grafana-cloud/prepare.yml); +# 3. a unit drop-in that switches the stub's byte-for-byte RPC URL check on; +# 4. a release mirror on loopback (127.0.0.1:8081): the stub packaged the way +# upstream names a sponsord release, a SHA256SUMS signed by a throwaway key, and +# a second release whose binary reports the wrong version. side_effect.yml +# installs from it; the role verifies against the mirror's key (fetched to +# the controller, since sponsord_release_keyring is a controller-side file). +# The API token is NOT staged: the role must generate it. +# +# CONTAINER-ONLY step first: systemd builds each unit's LoadCredential= ramfs in a +# helper mount namespace and moves it into /run/credentials/, which reaches +# PID 1's namespace only when / has shared propagation. systemd sets that itself +# when it boots a real host; under docker the container's / is private, so without +# this every credential directory comes up EMPTY and sponsord cannot start. +- name: Prepare + hosts: all + become: true + tasks: + # ansible.posix.mount cannot change propagation, hence the command. + - name: Give / shared mount propagation, as systemd does on a real host # noqa: command-instead-of-module + ansible.builtin.command: + cmd: mount --make-rshared / + changed_when: true + + - name: Create the sponsord config directory + ansible.builtin.file: + path: /etc/sponsord + state: directory + mode: "0755" + + - name: Stage a placeholder treasury keystore + ansible.builtin.copy: + dest: /etc/sponsord/treasury-keystore.json + content: "{{ '{}' }}\n" + mode: "0644" + + - name: Stage a placeholder treasury password + ansible.builtin.copy: + dest: /etc/sponsord/treasury-password + content: "molecule-placeholder\n" + mode: "0644" + + - name: Stage the Grafana Cloud credential fixture on the host + ansible.builtin.copy: + dest: /etc/grafana-alloy.env + owner: root + group: root + mode: "0600" + content: | + GC_PROM_REMOTE_WRITE_URL=https://prometheus-prod-xx.molecule.invalid/api/prom/push + GC_OTLP_ENDPOINT=https://otlp-gateway-prod-xx.molecule.invalid/otlp + GC_PROM_USERNAME=999888777 + GC_OTLP_USERNAME=999888779 + GC_LOKI_USERNAME=999888778 + GC_API_TOKEN=molecule-test-token + + - name: Create the sponsord unit drop-in directory + ansible.builtin.file: + path: /etc/systemd/system/sponsord.service.d + state: directory + mode: "0755" + + - name: Switch on the stub's RPC literal check + ansible.builtin.copy: + dest: /etc/systemd/system/sponsord.service.d/molecule.conf + mode: "0644" + content: | + [Service] + Environment=SPONSORD_MOLECULE_RPC_LITERAL=1 + + # --- The release mirror ------------------------------------------------------- + - name: Install gnupg and tar for the release mirror + ansible.builtin.apt: + name: [gnupg, tar] + state: present + update_cache: true + + - name: Create the mirror's served tree and private directories + ansible.builtin.file: + path: "{{ item.path }}" + state: directory + owner: root + group: root + mode: "{{ item.mode }}" + loop: + - {path: /srv/sponsord-mirror, mode: "0755"} + - {path: /srv/sponsord-mirror/www, mode: "0755"} + - {path: /srv/sponsord-mirror/www/releases, mode: "0755"} + - {path: /srv/sponsord-mirror/www/releases/sponsord-v0.0.0-molecule-stub, mode: "0755"} + - {path: /srv/sponsord-mirror/www/releases/sponsord-v0.0.9, mode: "0755"} + - {path: /srv/sponsord-mirror/gnupg, mode: "0700"} + - {path: /srv/sponsord-mirror/stage, mode: "0700"} + loop_control: + label: "{{ item.path }}" + + - name: Stage the stub as the release binary + ansible.builtin.copy: + src: "{{ lookup('ansible.builtin.env', 'MOLECULE_SCENARIO_DIRECTORY') }}/files/sponsord-stub" + dest: /srv/sponsord-mirror/stage/sponsord + owner: root + group: root + mode: "0755" + + # sponsord-v/sponsord--.tar.gz with the binary at the archive + # root, as upstream's release.yml publishes. v0.0.9 holds the same stub, which + # reports 0.0.0-molecule-stub: a release whose binary is not the pinned version. + - name: Build, checksum and sign the mirror's releases + ansible.builtin.shell: + cmd: | + set -euo pipefail + export GNUPGHOME=/srv/sponsord-mirror/gnupg + gpg --batch --pinentry-mode loopback --passphrase '' \ + --quick-gen-key 'molecule sponsord mirror ' ed25519 sign 1d + gpg --batch --armor --export > /srv/sponsord-mirror/KEYS.asc + for ver in 0.0.0-molecule-stub 0.0.9; do + cd /srv/sponsord-mirror/www/releases/sponsord-v$ver + tar -C /srv/sponsord-mirror/stage -czf "sponsord-$ver-{{ _sp_triple }}.tar.gz" sponsord + sha256sum -- *.tar.gz > SHA256SUMS + gpg --batch --armor --detach-sign --output SHA256SUMS.asc SHA256SUMS + chmod 0644 -- * + done + executable: /bin/bash + creates: /srv/sponsord-mirror/KEYS.asc + vars: + _sp_triple: >- + {{ {'x86_64': 'x86_64-unknown-linux-gnu', + 'aarch64': 'aarch64-unknown-linux-gnu'}[ansible_facts['architecture']] }} + + - name: Fetch the mirror's public key to the controller + ansible.builtin.fetch: + src: /srv/sponsord-mirror/KEYS.asc + dest: "{{ _sp_eph }}/sponsord-mirror-KEYS-{{ inventory_hostname }}.asc" + flat: true + vars: + _sp_eph: "{{ lookup('ansible.builtin.env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}" + + - name: Install the mirror's HTTP server unit + ansible.builtin.copy: + dest: /etc/systemd/system/molecule-sponsord-mirror.service + owner: root + group: root + mode: "0644" + content: | + [Unit] + Description=Loopback sponsord release mirror for the molecule scenario + [Service] + ExecStart=/usr/bin/python3 -m http.server 8081 --bind 127.0.0.1 --directory /srv/sponsord-mirror/www + DynamicUser=yes + + - name: Start the mirror + ansible.builtin.systemd_service: + name: molecule-sponsord-mirror + state: started + daemon_reload: true diff --git a/ansible/molecule/sponsord/side_effect.yml b/ansible/molecule/sponsord/side_effect.yml new file mode 100644 index 0000000..b043cce --- /dev/null +++ b/ansible/molecule/sponsord/side_effect.yml @@ -0,0 +1,493 @@ +--- +# Four behaviours a green converge + idempotence cannot prove: +# 1. Rotating a credential OUT OF BAND (the operator replaces the treasury +# password file) and re-running the role restarts the daemon onto it. Without +# this the daemon keeps the old credential behind a green deploy. +# 2. The /healthz readiness gate is FATAL: a daemon that never binds (the stub +# refuses to start while its marker exists) fails the converge at the role's +# report task, rather than passing on a Type=simple unit stuck restarting. +# 3. secret.env's two homes and the provenance guards between them: an empty +# sponsord_rpc_url must not adopt the role's own file; a handed-over file is +# recorded `host` and locked to 0600; a host edit restarts the daemon; an +# inventory URL must not overwrite the host's file until confirmed. +# 4. release mode against prepare.yml's signed loopback mirror: a verified +# install writes the version stamp; a re-run needs no download; a bad +# checksum, a bad signature and a binary that is not the pinned version all +# fail at their own task and leave no stamp; manual mode clears the stamp. +# The host is left healthy, back on the inventory-authored secret.env and the +# manual install, for verify.yml. +- name: Side effect — credential rotation and the fatal readiness gate + hosts: all + become: true + tasks: + - name: Read the unit state before the rotation + ansible.builtin.systemd_service: + name: sponsord + register: _sp_before + + - name: Assert sponsord was running cleanly before the rotation + ansible.builtin.assert: + that: + - _sp_before.status.ActiveState == 'active' + - _sp_before.status.MainPID | int > 0 + - _sp_before.status.NRestarts == '0' + fail_msg: >- + Expected sponsord active with no automatic restarts before the rotation (got + ActiveState={{ _sp_before.status.ActiveState }}, + NRestarts={{ _sp_before.status.NRestarts }}): journalctl -u sponsord -e + + - name: Hash the generated API token before the second converge + ansible.builtin.stat: + path: /etc/sponsord/api-token + checksum_algorithm: sha256 + register: _sp_token_before + + - name: Save the token hash for verify.yml + ansible.builtin.copy: + dest: /root/molecule-sponsord-token-before + content: "{{ _sp_token_before.stat.checksum }}\n" + mode: "0600" + + - name: Rotate the treasury password out of band + ansible.builtin.copy: + dest: /etc/sponsord/treasury-password + content: "molecule-rotated\n" + mode: "0600" + + - name: Converge again after the rotation + ansible.builtin.include_role: + name: sponsord + + - name: Read the unit state after the rotation converge + ansible.builtin.systemd_service: + name: sponsord + register: _sp_after + + # Checked here, before the readiness case below stops the daemon (which changes + # the PID regardless). + - name: Require the rotation to have restarted sponsord + ansible.builtin.assert: + that: + - _sp_after.status.ActiveState == 'active' + - _sp_after.status.MainPID != _sp_before.status.MainPID + fail_msg: >- + sponsord kept PID {{ _sp_before.status.MainPID }} after its treasury password + was rotated and the role re-ran: the daemon is still on the old credential. + + - name: Stop the daemon, then make it refuse to start + ansible.builtin.systemd_service: + name: sponsord + state: stopped + + - name: Stage the stub's refuse-to-start marker + ansible.builtin.copy: + dest: /etc/sponsord/stub-unhealthy + content: "" + mode: "0644" + + - name: Converge against a daemon that never becomes healthy + block: + - name: Run the role with a short readiness window + ansible.builtin.include_role: + name: sponsord + vars: + sponsord_readiness_retries: 3 + sponsord_readiness_delay: 1 + + - name: Fail — the readiness gate let an unhealthy daemon through + ansible.builtin.fail: + msg: The role finished although sponsord never answered /healthz. + rescue: + - name: Require the failure to come from the readiness gate itself + ansible.builtin.assert: + that: + - ansible_failed_task.name == 'Report a sponsord that did not become healthy' + fail_msg: >- + Expected the role to fail at its readiness report, but it failed at + "{{ ansible_failed_task.name }}". + always: + - name: Remove the refuse-to-start marker + ansible.builtin.file: + path: /etc/sponsord/stub-unhealthy + state: absent + + # The failed run wrote no records (they come after the gate), so this run sees + # the same inputs and only has to bring the daemon back. + - name: Converge once more to leave the host healthy + ansible.builtin.include_role: + name: sponsord + +- name: Side effect — secret.env provenance + hosts: all + become: true + vars: + _sp_env: /etc/sponsord/secret.env + _sp_rec: /etc/sponsord/.secret.env.sha256 + tasks: + # 1. secret.yml "went missing": the file is byte-for-byte the role's own. + - name: Hash secret.env before the adoption attempt + ansible.builtin.stat: + path: "{{ _sp_env }}" + checksum_algorithm: sha256 + register: _sp_env_before + + - name: Run with no inventory RPC URL against the role-authored file + block: + - name: Run the role with sponsord_rpc_url empty + ansible.builtin.include_role: + name: sponsord + vars: + sponsord_rpc_url: "" + + - name: Fail — the role adopted its own file as host-provisioned + ansible.builtin.fail: + msg: An empty sponsord_rpc_url silently adopted the role-authored secret.env. + rescue: + - name: Require the adoption guard to be what stopped it + ansible.builtin.assert: + that: + - ansible_failed_task.name == 'Refuse to adopt a role-authored secret.env as host-provisioned' + fail_msg: "Expected the adoption guard, got \"{{ ansible_failed_task.name }}\"." + + - name: Hash secret.env after the refused adoption + ansible.builtin.stat: + path: "{{ _sp_env }}" + checksum_algorithm: sha256 + register: _sp_env_after + + - name: Require the refused run to leave secret.env untouched + ansible.builtin.assert: + that: + - _sp_env_after.stat.checksum == _sp_env_before.stat.checksum + + # 2. A deliberate handover: drop the record, loosen the mode. The role must take + # the file as the operator's, lock it down, and owe no restart (same bytes). + - name: Read the unit state before the handover + ansible.builtin.systemd_service: + name: sponsord + register: _sp_unit_a + + - name: Hand secret.env to the host (discard the record, loosen the mode) + ansible.builtin.shell: + cmd: rm -f {{ _sp_rec }} && chmod 0644 {{ _sp_env }} + changed_when: true + + - name: Converge with the host-provisioned secret.env + ansible.builtin.include_role: + name: sponsord + vars: + sponsord_rpc_url: "" + + - name: Read secret.env, its record and the unit after the handover + ansible.builtin.stat: + path: "{{ _sp_env }}" + checksum_algorithm: sha256 + register: _sp_env_host + + - name: Read the provenance record after the handover + ansible.builtin.slurp: + src: "{{ _sp_rec }}" + register: _sp_rec_host + + - name: Read the unit state after the handover + ansible.builtin.systemd_service: + name: sponsord + register: _sp_unit_b + + - name: Require a locked, host-recorded file and no restart + ansible.builtin.assert: + that: + - _sp_env_host.stat.mode == '0600' + - (_sp_rec_host.content | b64decode).split() == ['host', _sp_env_host.stat.checksum] + - _sp_unit_b.status.MainPID == _sp_unit_a.status.MainPID + fail_msg: >- + Expected secret.env 0600 (got {{ _sp_env_host.stat.mode }}), record + "host {{ _sp_env_host.stat.checksum }}" (got "{{ _sp_rec_host.content | b64decode | trim }}") + and no restart (PID {{ _sp_unit_a.status.MainPID }} -> {{ _sp_unit_b.status.MainPID }}). + + # 3. The operator edits the host file: the next converge must restart onto it. + - name: Edit secret.env on the host + ansible.builtin.lineinfile: + path: "{{ _sp_env }}" + line: "# molecule host edit" + mode: "0600" + + - name: Converge after the host edit + ansible.builtin.include_role: + name: sponsord + vars: + sponsord_rpc_url: "" + + - name: Read the unit state after the host edit + ansible.builtin.systemd_service: + name: sponsord + register: _sp_unit_c + + - name: Require the host edit to have restarted sponsord + ansible.builtin.assert: + that: + - _sp_unit_c.status.ActiveState == 'active' + - _sp_unit_c.status.MainPID != _sp_unit_b.status.MainPID + + # 4. An inventory URL against the host's (edited) file: refused, file kept. + - name: Run with the inventory URL against the host's file + block: + - name: Run the role with the inventory URL (group_vars) + ansible.builtin.include_role: + name: sponsord + + - name: Fail — the role overwrote the host's file unconfirmed + ansible.builtin.fail: + msg: An inventory sponsord_rpc_url overwrote a host-provisioned secret.env without confirmation. + rescue: + - name: Require the overwrite guard to be what stopped it + ansible.builtin.assert: + that: + - ansible_failed_task.name == 'Require confirmation to overwrite a secret.env this role did not write' + fail_msg: "Expected the overwrite guard, got \"{{ ansible_failed_task.name }}\"." + + - name: Require the host edit to survive the refused overwrite + ansible.builtin.command: + cmd: grep -qx '# molecule host edit' {{ _sp_env }} + changed_when: false + + # 5. Confirmed: the role takes the file back and restarts onto it. + - name: Converge with the overwrite confirmed + ansible.builtin.include_role: + name: sponsord + vars: + sponsord_secret_env_overwrite_host_file: true + + - name: Read the provenance record after the takeover + ansible.builtin.slurp: + src: "{{ _sp_rec }}" + register: _sp_rec_inv + + - name: Read the unit state after the takeover + ansible.builtin.systemd_service: + name: sponsord + register: _sp_unit_d + + - name: Require an inventory record, the edit gone and a restart + ansible.builtin.assert: + that: + - (_sp_rec_inv.content | b64decode).split()[0] == 'inventory' + - _sp_unit_d.status.MainPID != _sp_unit_c.status.MainPID + fail_msg: >- + Expected record source "inventory" (got + "{{ _sp_rec_inv.content | b64decode | trim }}") and a restart. + + - name: Require the host edit to be gone + ansible.builtin.command: + cmd: grep -qx '# molecule host edit' {{ _sp_env }} + register: _sp_edit_gone + changed_when: false + failed_when: _sp_edit_gone.rc != 1 + +- name: Side effect — release mode against the signed loopback mirror + hosts: all + become: true + vars: + _sp_ver: 0.0.0-molecule-stub + _sp_base: http://127.0.0.1:8081/releases + _sp_eph: "{{ lookup('ansible.builtin.env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}" + _sp_keys: "{{ _sp_eph }}/sponsord-mirror-KEYS-{{ inventory_hostname }}.asc" + _sp_stamp: /usr/local/lib/sponsord/installed-version + _sp_rel: /srv/sponsord-mirror/www/releases/sponsord-v0.0.0-molecule-stub + tasks: + - name: Wait for the mirror to serve the signed manifest + ansible.builtin.uri: + url: "{{ _sp_base }}/sponsord-v{{ _sp_ver }}/SHA256SUMS.asc" + register: _sp_mirror_up + until: _sp_mirror_up.status == 200 + retries: 10 + delay: 1 + + # 1. A verified install records the stamp. + - name: Install the pinned release + ansible.builtin.include_role: + name: sponsord + vars: + sponsord_install_method: release + sponsord_version: "{{ _sp_ver }}" + sponsord_release_base: "{{ _sp_base }}" + sponsord_release_keyring: "{{ _sp_keys }}" + + - name: Read the version stamp + ansible.builtin.slurp: + src: "{{ _sp_stamp }}" + register: _sp_stamp_1 + + - name: Require the stamp to name the installed release + ansible.builtin.assert: + that: + - (_sp_stamp_1.content | b64decode | trim) == _sp_ver + + # 2. The stamp is what makes a re-run need no download. + - name: Stop the mirror + ansible.builtin.systemd_service: + name: molecule-sponsord-mirror + state: stopped + + - name: Re-run the release install with the mirror down + ansible.builtin.include_role: + name: sponsord + vars: + sponsord_install_method: release + sponsord_version: "{{ _sp_ver }}" + sponsord_release_base: "{{ _sp_base }}" + sponsord_release_keyring: "{{ _sp_keys }}" + + - name: Start the mirror again + ansible.builtin.systemd_service: + name: molecule-sponsord-mirror + state: started + + # 3. A manifest with a valid signature but the wrong hash. + - name: Keep the good manifest and signature aside, then publish a signed bad hash + ansible.builtin.shell: + cmd: | + set -euo pipefail + cd {{ _sp_rel | quote }} + cp SHA256SUMS /root/SHA256SUMS.good + cp SHA256SUMS.asc /root/SHA256SUMS.asc.good + f=$(awk '{print $2}' SHA256SUMS) + printf '%064d %s\n' 0 "$f" > SHA256SUMS + rm -f SHA256SUMS.asc + GNUPGHOME=/srv/sponsord-mirror/gnupg gpg --batch --armor --detach-sign --output SHA256SUMS.asc SHA256SUMS + cp SHA256SUMS.asc /root/SHA256SUMS.asc.bad + chmod 0644 SHA256SUMS SHA256SUMS.asc + rm -f {{ _sp_stamp | quote }} + executable: /bin/bash + changed_when: true + + - name: Install against the bad checksum + block: + - name: Run the release install + ansible.builtin.include_role: + name: sponsord + vars: + sponsord_install_method: release + sponsord_version: "{{ _sp_ver }}" + sponsord_release_base: "{{ _sp_base }}" + sponsord_release_keyring: "{{ _sp_keys }}" + + - name: Fail — a tarball that does not match the manifest was installed + ansible.builtin.fail: + msg: The checksum check let a mismatching tarball through. + rescue: + - name: Require the checksum check to be what stopped it + ansible.builtin.assert: + that: + - ansible_failed_task.name == 'Verify the tarball against SHA256SUMS' + fail_msg: "Expected the checksum check, got \"{{ ansible_failed_task.name }}\"." + + # 4. The good manifest with a signature over a different one. + - name: Publish the good manifest with the bad manifest's signature + ansible.builtin.shell: + cmd: | + set -euo pipefail + cd {{ _sp_rel | quote }} + cp /root/SHA256SUMS.good SHA256SUMS + cp /root/SHA256SUMS.asc.bad SHA256SUMS.asc + chmod 0644 SHA256SUMS SHA256SUMS.asc + executable: /bin/bash + changed_when: true + + - name: Install against the mismatched signature + block: + - name: Run the release install + ansible.builtin.include_role: + name: sponsord + vars: + sponsord_install_method: release + sponsord_version: "{{ _sp_ver }}" + sponsord_release_base: "{{ _sp_base }}" + sponsord_release_keyring: "{{ _sp_keys }}" + + - name: Fail — a manifest with a bad signature was trusted + ansible.builtin.fail: + msg: The signature check let a mismatched signature through. + rescue: + - name: Require the signature check to be what stopped it + ansible.builtin.assert: + that: + - ansible_failed_task.name == 'Verify the SHA256SUMS signature against the release keyring' + fail_msg: "Expected the signature check, got \"{{ ansible_failed_task.name }}\"." + + - name: Restore the good signature + ansible.builtin.copy: + src: /root/SHA256SUMS.asc.good + dest: "{{ _sp_rel }}/SHA256SUMS.asc" + remote_src: true + mode: "0644" + + # Both failures keep their staging directory as evidence; remove it. + - name: Find the retained staging directories + ansible.builtin.find: + paths: /tmp + patterns: sponsord-install-* + file_type: directory + register: _sp_stages + + - name: Require one retained staging directory per failed verification + ansible.builtin.assert: + that: + - _sp_stages.matched == 2 + fail_msg: "Expected 2 retained staging dirs, found {{ _sp_stages.files | map(attribute='path') | list }}" + + - name: Remove the retained staging directories + ansible.builtin.file: + path: "{{ item.path }}" + state: absent + loop: "{{ _sp_stages.files }}" + loop_control: + label: "{{ item.path }}" + + # 5. A signed, verified release whose binary is not the pinned version. + - name: Install a release whose binary reports another version + block: + - name: Run the release install of 0.0.9 + ansible.builtin.include_role: + name: sponsord + vars: + sponsord_install_method: release + sponsord_version: 0.0.9 + sponsord_release_base: "{{ _sp_base }}" + sponsord_release_keyring: "{{ _sp_keys }}" + + - name: Fail — a binary that is not the pinned version was accepted + ansible.builtin.fail: + msg: The version backstop accepted a binary reporting another version. + rescue: + - name: Require the version backstop to be what stopped it + ansible.builtin.assert: + that: + - ansible_failed_task.name == 'Require the installed sponsord to run and match the pinned version' + fail_msg: "Expected the version backstop, got \"{{ ansible_failed_task.name }}\"." + + - name: Check for a version stamp after the failed installs + ansible.builtin.stat: + path: "{{ _sp_stamp }}" + register: _sp_stamp_failed + + - name: Require no stamp after the failed installs + ansible.builtin.assert: + that: + - not _sp_stamp_failed.stat.exists + fail_msg: A failed release install left a version stamp, so the next run would skip the download. + + # 6. Back to manual (the scenario's group_vars): the stamp must go. + - name: Converge back to the manual install + ansible.builtin.include_role: + name: sponsord + + - name: Check for a version stamp in manual mode + ansible.builtin.stat: + path: "{{ _sp_stamp }}" + register: _sp_stamp_manual + + - name: Require manual mode to clear the stamp + ansible.builtin.assert: + that: + - not _sp_stamp_manual.stat.exists diff --git a/ansible/molecule/sponsord/verify.yml b/ansible/molecule/sponsord/verify.yml new file mode 100644 index 0000000..73108e9 --- /dev/null +++ b/ansible/molecule/sponsord/verify.yml @@ -0,0 +1,212 @@ +--- +- name: Verify + hosts: all + become: true + gather_facts: false + tasks: + # --- Files and modes ------------------------------------------------------------ + - name: Stat the sponsord files + ansible.builtin.stat: + path: "{{ item }}" + follow: false + register: _sp_files + loop: + - /etc/sponsord/api-token + - /etc/sponsord/treasury-keystore.json + - /etc/sponsord/treasury-password + - /etc/sponsord/secret.env + - /etc/sponsord/.secret.env.sha256 + - /etc/sponsord/.inputs.sha256 + + - name: Require every secret and record to be a root-only regular file + ansible.builtin.assert: + that: + - item.stat.isreg + - item.stat.pw_name == 'root' + - item.stat.mode == '0600' + fail_msg: "{{ item.item }}: expected root 0600 regular file, got {{ item.stat }}" + quiet: true + loop: "{{ _sp_files.results }}" + loop_control: + label: "{{ item.item }}" + + - name: Read the generated API token's length + ansible.builtin.shell: + executable: /bin/bash + cmd: set -o pipefail; tr -d '\n' < /etc/sponsord/api-token | wc -c + register: _sp_token_len + changed_when: false + + - name: Require a 64-hex-character token + ansible.builtin.assert: + that: + - _sp_token_len.stdout | int == 64 + + - name: Read the token hash side_effect saved before its converges + ansible.builtin.slurp: + src: /root/molecule-sponsord-token-before + register: _sp_token_before + + - name: Hash the API token now + ansible.builtin.stat: + path: /etc/sponsord/api-token + checksum_algorithm: sha256 + register: _sp_token_now + + - name: Read the unit state now + ansible.builtin.systemd_service: + name: sponsord + register: _sp_unit + + - name: Require sponsord running on an API token that was never regenerated + ansible.builtin.assert: + that: + - _sp_unit.status.ActiveState == 'active' + - _sp_token_now.stat.checksum == (_sp_token_before.content | b64decode | trim) + fail_msg: >- + Expected sponsord active (it is {{ _sp_unit.status.ActiveState }}) and the + generated API token unchanged across three converges. + + # --- Config --------------------------------------------------------------------- + - name: Read sponsord.env + ansible.builtin.slurp: + src: /etc/sponsord/sponsord.env + register: _sp_env + + - name: Require the non-secret settings and nothing secret + ansible.builtin.assert: + that: + - "'SPONSORD_BIND=127.0.0.1:8090' in _sp_env_text" + # From the arbitrum-sepolia profile (vars/main/networks.yml). + - "'SPONSORD_CHAIN_ID=421614' in _sp_env_text" + - "'SPONSORD_PAYMENT_POOL_ADDR=0x351E5544471e11638b89DF8E3A3299c28E00cEc3' in _sp_env_text" + - "'SPONSORD_POOL_ID=0x' in _sp_env_text" + # Set knobs are emitted (0 included), unset ones are not. + - "'SPONSORD_MAX_TTL_SECS=3600' in _sp_env_text" + - "'SPONSORD_POOL_LOW_WATER_MICRO_USDC=0' in _sp_env_text" + - "'SPONSORD_MAX_SPENDING_CAP' not in _sp_env_text" + - "'SPONSORD_POOL_REFILL' not in _sp_env_text" + - "'RPC_URL' not in _sp_env_text" + - "'TOKEN' not in _sp_env_text" + - "'PASSWORD' not in _sp_env_text" + fail_msg: "Unexpected sponsord.env:\n{{ _sp_env_text }}" + vars: + _sp_env_text: "{{ _sp_env.content | b64decode }}" + + - name: Require the RPC URL not to have been shell-expanded anywhere + ansible.builtin.stat: + path: /tmp/sponsord-rpc-expanded + register: _sp_expanded + failed_when: _sp_expanded.stat.exists + + - name: Hash secret.env + ansible.builtin.stat: + path: /etc/sponsord/secret.env + checksum_algorithm: sha256 + register: _sp_secret_env + + - name: Read the secret.env provenance record + ansible.builtin.slurp: + src: /etc/sponsord/.secret.env.sha256 + register: _sp_secret_rec + + # A wrong source token would silently disable the adoption guard. + - name: Require the record to name the role and the current file + ansible.builtin.assert: + that: + - (_sp_secret_rec.content | b64decode).split() == ['inventory', _sp_secret_env.stat.checksum] + fail_msg: "Unexpected record: {{ _sp_secret_rec.content | b64decode | trim }}" + + # --- Unit ----------------------------------------------------------------------- + - name: Read the unit + ansible.builtin.slurp: + src: /etc/systemd/system/sponsord.service + register: _sp_unit_file + + - name: Require credentials and hardening in the unit + ansible.builtin.assert: + that: + - item in (_sp_unit_file.content | b64decode) + fail_msg: "sponsord.service lacks: {{ item }}" + quiet: true + loop: + - DynamicUser=true + # kics-scan ignore-line (a credential file PATH in the unit, not a secret) + - LoadCredential=api-token:/etc/sponsord/api-token + # kics-scan ignore-line (a credential file PATH in the unit, not a secret) + - LoadCredential=treasury-password:/etc/sponsord/treasury-password + - LoadCredential=treasury-keystore.json:/etc/sponsord/treasury-keystore.json + - Environment=SPONSORD_TREASURY_KEYSTORE=%t/sponsord/treasury-keystore.json + - RuntimeDirectoryMode=0700 + - EnvironmentFile=/etc/sponsord/secret.env + - NoNewPrivileges=true + - ProtectSystem=strict + - CapabilityBoundingSet= + - MemoryDenyWriteExecute=true + # The plain-text level parsing in grafana_alloy depends on it. + - Environment=NO_COLOR=1 + - SystemCallFilter=@system-service + - TimeoutStopSec=120 + + # --- Listener ------------------------------------------------------------------- + - name: List TCP listeners + ansible.builtin.command: + cmd: ss -Hltn + register: _sp_ss + changed_when: false + + - name: Require sponsord on loopback only + ansible.builtin.assert: + that: + - _sp_ss.stdout is search('127\.0\.0\.1:8090\s') + - _sp_ss.stdout is not search('(0\.0\.0\.0|\*|\[::\]):8090\s') + fail_msg: "{{ _sp_ss.stdout }}" + + # The keystore the daemon reads: a 0600 copy, in the unit's private 0700 runtime + # directory, of the operator's file (not of some other credential). + # With DynamicUser=, /run/sponsord is a symlink into the 0700 /run/private/. + - name: Stat the runtime directory and the keystore copy + ansible.builtin.stat: + path: "{{ item }}" + follow: true + checksum_algorithm: sha256 + register: _sp_runtime + loop: + - /run/sponsord + - /run/sponsord/treasury-keystore.json + - /etc/sponsord/treasury-keystore.json + + - name: Require the private runtime copy of the keystore + ansible.builtin.assert: + that: + - _sp_runtime.results[0].stat.isdir and _sp_runtime.results[0].stat.mode == '0700' + - _sp_runtime.results[1].stat.isreg and _sp_runtime.results[1].stat.mode == '0600' + - _sp_runtime.results[1].stat.checksum == _sp_runtime.results[2].stat.checksum + fail_msg: "{{ _sp_runtime.results | map(attribute='stat') | list | to_json }}" + + - name: Require /healthz to answer + ansible.builtin.uri: + url: http://127.0.0.1:8090/healthz + return_content: true + register: _sp_health + failed_when: _sp_health.json.ok is not true + + # --- Alloy (standalone host) ---------------------------------------------------- + - name: Read the rendered Alloy config + ansible.builtin.slurp: + src: /etc/alloy/config.alloy + register: _sp_alloy + + - name: Require the sponsord scrape and no decdn-node scrape + ansible.builtin.assert: + that: + - "'prometheus.scrape \"sponsord\"' in _sp_alloy_text" + - "'\"__address__\" = \"127.0.0.1:8090\"' in _sp_alloy_text" + - "'prometheus.scrape \"decdn_node\"' not in _sp_alloy_text" + # The journald service_name rule (3 spaces; the metric rule has 2). + - "'replacement = \"sponsord\"' in _sp_alloy_text" + fail_msg: >- + The Alloy config on a sponsord-only host must scrape sponsord and not the + node (playbooks/group_vars/all.yml toggles). + vars: + _sp_alloy_text: "{{ _sp_alloy.content | b64decode }}" diff --git a/ansible/molecule/validation/converge.yml b/ansible/molecule/validation/converge.yml index 26f1230..948c41b 100644 --- a/ansible/molecule/validation/converge.yml +++ b/ansible/molecule/validation/converge.yml @@ -1,8 +1,8 @@ --- # NEGATIVE-path scenario: proves the role's fail-loud tuning-knob validation actually # REJECTS bad values (AGENTS.md hard-rule 4), so a regression that weakens an assert -# fails CI. Each case runs decdn_node with a VALID required config plus ONE bad -# optional knob, wrapped in block/rescue. The role aborts at its validation asserts +# fails CI. Each case runs one role (decdn_node, grafana_alloy or sponsord) with a +# VALID required config plus ONE bad input, wrapped in block/rescue. The role aborts at its validation asserts # (which for the tuning-knob cases run before any host mutation), so those cases # don't interfere and no service starts. The env-file cases below are the # exception — see the ordering note. @@ -22,7 +22,9 @@ # config and has to reach the END of the role, because what it proves is that the # post-template `decdn config validate` gate still fails the deploy. It therefore # mutates the host, which is why this scenario has a prepare.yml staging wallet -# material. It runs last so it cannot interfere with the assert-only cases above. +# material. It runs after the decdn_node assert-only cases so it cannot interfere +# with them; the grafana_alloy and sponsord cases follow it and clean up after +# themselves. - name: Converge (expect validation failures) hosts: all become: true @@ -43,6 +45,12 @@ decdn_slash_judge_address: "0x3333333333333333333333333333333333333333" # REQUIRED upstream (ADR 011/031) — the role asserts it with the other three. decdn_content_blacklist_address: "0x4444444444444444444444444444444444444444" + # Valid sponsord baseline for the sponsord-* cases (unused by the others). + sponsord_manual_bin_src: >- + {{ lookup('ansible.builtin.env', 'MOLECULE_PROJECT_DIRECTORY') }}/molecule/sponsord/files/sponsord-stub + sponsord_network: arbitrum-sepolia + sponsord_pool_id: "0x00000000000000000000000000000000000000000000000000000000000000aa" + sponsord_rpc_url: "https://rpc.example.invalid/" tasks: - name: Initialize the rejected-by-validation tracker ansible.builtin.set_fact: @@ -1622,6 +1630,484 @@ - /etc/grafana-alloy.env.sha256 - /var/lib/grafana-alloy-record-fixture + # --- sponsord: each case breaks ONE input over the valid baseline in play vars --- + # All but the last two abort at an assert before any host mutation. + - name: "Case sponsord-release-version — release mode with no pinned version" + block: + - name: Run sponsord with release mode with no pinned version + ansible.builtin.include_role: + name: sponsord + vars: + sponsord_install_method: release + sponsord_version: "" + rescue: + - name: Record sponsord-release-version rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['sponsord-release-version'] }}" + when: ansible_failed_task.name == "Require a pinned release version" + + - name: "Case sponsord-manual-src — manual mode with no binary source" + block: + - name: Run sponsord with manual mode with no binary source + ansible.builtin.include_role: + name: sponsord + vars: + sponsord_manual_bin_src: "" + rescue: + - name: Record sponsord-manual-src rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['sponsord-manual-src'] }}" + when: ansible_failed_task.name == "Require a manual binary source" + + - name: "Case sponsord-network-unknown — an unknown network profile" + block: + - name: Run sponsord with an unknown network profile + ansible.builtin.include_role: + name: sponsord + vars: + sponsord_network: nope + rescue: + - name: Record sponsord-network-unknown rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['sponsord-network-unknown'] }}" + when: ansible_failed_task.name == "Require a known sponsord_network" + + - name: "Case sponsord-network-chain — a chain_id that contradicts the profile" + block: + - name: Run sponsord with a chain_id that contradicts the profile + ansible.builtin.include_role: + name: sponsord + vars: + sponsord_chain_id: 1 + rescue: + - name: Record sponsord-network-chain rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['sponsord-network-chain'] }}" + when: ansible_failed_task.name == "Require chain_id to match the network profile" + + - name: "Case sponsord-pool-id — a pool id that is not 32 bytes" + block: + - name: Run sponsord with a pool id that is not 32 bytes + ansible.builtin.include_role: + name: sponsord + vars: + sponsord_pool_id: "0x1234" + rescue: + - name: Record sponsord-pool-id rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['sponsord-pool-id'] }}" + when: ansible_failed_task.name == "Require chain, PaymentPool and pool id" + + - name: "Case sponsord-zero-address — the zero PaymentPool address" + block: + - name: Run sponsord with the zero PaymentPool address + ansible.builtin.include_role: + name: sponsord + vars: + sponsord_payment_pool_address: "0x0000000000000000000000000000000000000000" + rescue: + - name: Record sponsord-zero-address rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['sponsord-zero-address'] }}" + when: ansible_failed_task.name == "Require chain, PaymentPool and pool id" + + - name: "Case sponsord-bind — a wildcard listener" + block: + - name: Run sponsord with a wildcard listener + ansible.builtin.include_role: + name: sponsord + vars: + sponsord_bind_address: "0.0.0.0" + rescue: + - name: Record sponsord-bind rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['sponsord-bind'] }}" + when: ansible_failed_task.name == "Require the listener to stay on loopback" + + - name: "Case sponsord-limit — a zero TTL" + block: + - name: Run sponsord with a zero TTL + ansible.builtin.include_role: + name: sponsord + vars: + sponsord_max_ttl_secs: 0 + rescue: + - name: Record sponsord-limit rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['sponsord-limit'] }}" + when: ansible_failed_task.name == "Validate the issuance and keeper limits" + + - name: "Case sponsord-limit — a refill in float notation" + block: + - name: Run sponsord with a refill in float notation + ansible.builtin.include_role: + name: sponsord + vars: + sponsord_pool_refill_micro_usdc: "1e6" + rescue: + - name: Record sponsord-limit rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['sponsord-limit'] }}" + when: ansible_failed_task.name == "Validate the issuance and keeper limits" + + - name: "Case sponsord-bool — a quoted boolean" + block: + - name: Run sponsord with a quoted boolean + ansible.builtin.include_role: + name: sponsord + vars: + # kics-scan ignore-line (a quoted boolean under test, not a secret) + sponsord_generate_api_token: "yes" + rescue: + - name: Record sponsord-bool rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['sponsord-bool'] }}" + when: ansible_failed_task.name == "Validate the role's boolean knobs" + + - name: "Case sponsord-rpc-newline — an RPC URL that injects a second variable" + block: + - name: Run sponsord with an RPC URL that injects a second variable + ansible.builtin.include_role: + name: sponsord + vars: + sponsord_rpc_url: "https://rpc.example.invalid/\nSPONSORD_BIND=0.0.0.0:8090" + rescue: + - name: Record sponsord-rpc-newline rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['sponsord-rpc-newline'] }}" + when: ansible_failed_task.name == "Validate the inventory RPC URL is one-line EnvironmentFile data" + + - name: "Case sponsord-rpc-gate — no RPC URL and no host secret.env" + block: + - name: Run sponsord with no RPC URL and no host secret.env + ansible.builtin.include_role: + name: sponsord + vars: + sponsord_rpc_url: "" + rescue: + - name: Record sponsord-rpc-gate rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['sponsord-rpc-gate'] }}" + when: ansible_failed_task.name == "Require an RPC endpoint — inventory value or a host-provisioned secret.env" + + - name: "Case sponsord-treasury — no treasury keystore on the host" + block: + - name: Run sponsord with no treasury keystore on the host + ansible.builtin.include_role: + name: sponsord + rescue: + - name: Record sponsord-treasury rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['sponsord-treasury'] }}" + when: ansible_failed_task.name == "Require the treasury keystore and password as regular files" + + # The role's root lock-down would otherwise chmod the link's target. + - name: "Case sponsord-treasury — a symlinked treasury keystore" + block: + - name: Stage a password file and a keystore that is a symlink + ansible.builtin.shell: + cmd: | + set -eu + mkdir -p /etc/sponsord + echo placeholder > /etc/sponsord/treasury-password + echo '{}' > /root/sponsord-keystore-target.json + ln -sfn /root/sponsord-keystore-target.json /etc/sponsord/treasury-keystore.json + changed_when: true + + - name: Run sponsord against the symlinked keystore + ansible.builtin.include_role: + name: sponsord + rescue: + - name: Record sponsord-treasury (symlink) rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['sponsord-treasury'] }}" + when: ansible_failed_task.name == "Require the treasury keystore and password as regular files" + always: + - name: Remove the symlink fixtures + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - /etc/sponsord + - /root/sponsord-keystore-target.json + + # Reaches past every pure-input assert and stops at the host-side token length + # check, before the binary is installed (so no restart is left pending for a + # unit that does not exist). It still touches /etc/sponsord, so it cleans up. + - name: "Case sponsord-token — an operator-provisioned API token under 32 bytes" + block: + - name: Stage the treasury files and a short token + ansible.builtin.shell: + cmd: | + set -eu + umask 077 + mkdir -p /etc/sponsord + echo '{}' > /etc/sponsord/treasury-keystore.json + echo placeholder > /etc/sponsord/treasury-password + echo too-short > /etc/sponsord/api-token + changed_when: true + + - name: Run sponsord with the short token and generation off + ansible.builtin.include_role: + name: sponsord + vars: + # kics-scan ignore-line (a boolean switch, not a secret) + sponsord_generate_api_token: false + rescue: + - name: Record sponsord-token rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['sponsord-token'] }}" + when: ansible_failed_task.name == "Require a single-line API token of at least 32 bytes" + always: + - name: Remove the sponsord fixtures + ansible.builtin.file: + path: /etc/sponsord + state: absent + + # Every octet must be 0-255, not merely 1-3 digits: 127.999.0.1 would pass a + # digit-count pattern and only fail later, when the daemon cannot parse it. + - name: "Case sponsord-bind — a loopback-looking address with an octet above 255" + block: + - name: Run sponsord with bind address 127.999.0.1 + ansible.builtin.include_role: + name: sponsord + vars: + sponsord_bind_address: "127.999.0.1" + rescue: + - name: Record sponsord-bind (octet) rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['sponsord-bind'] }}" + when: ansible_failed_task.name == "Require the listener to stay on loopback" + + # An existing secret.env with NO provenance record is the operator's: an + # inventory URL must not replace it without sponsord_secret_env_overwrite_host_file. + - name: "Case sponsord-overwrite-untracked — inventory URL over an unrecorded host file" + block: + - name: Stage a host secret.env with no provenance record + ansible.builtin.shell: + cmd: | + set -eu + umask 077 + mkdir -p /etc/sponsord + echo 'SPONSORD_RPC_URL=https://operator.example.invalid/' > /etc/sponsord/secret.env + changed_when: true + + - name: Run sponsord with the inventory URL + ansible.builtin.include_role: + name: sponsord + rescue: + - name: Record sponsord-overwrite-untracked rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['sponsord-overwrite-untracked'] }}" + when: ansible_failed_task.name == "Require confirmation to overwrite a secret.env this role did not write" + always: + - name: Remove the unrecorded secret.env fixture + ansible.builtin.file: + path: /etc/sponsord + state: absent + + # Long enough, but two lines: upstream would start on it, and no client could + # send it in an Authorization header. + - name: "Case sponsord-token — a provisioned API token with a line break inside" + block: + - name: Stage the treasury files and a two-line token + ansible.builtin.shell: + cmd: | + set -eu + umask 077 + mkdir -p /etc/sponsord + echo '{}' > /etc/sponsord/treasury-keystore.json + echo placeholder > /etc/sponsord/treasury-password + printf '%s\n%s\n' 0123456789abcdef0123456789abcdef 0123456789abcdef > /etc/sponsord/api-token + changed_when: true + + - name: Run sponsord with the two-line token and generation off + ansible.builtin.include_role: + name: sponsord + vars: + # kics-scan ignore-line (a boolean switch, not a secret) + sponsord_generate_api_token: false + rescue: + - name: Record sponsord-token (multi-line) rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['sponsord-token'] }}" + when: ansible_failed_task.name == "Require a single-line API token of at least 32 bytes" + always: + - name: Remove the multi-line token fixtures + ansible.builtin.file: + path: /etc/sponsord + state: absent + + # --- sponsord: sponsord_release_target_dir resolution (control machine) --------- + # One per-run controller tempdir for the three dir-mode cases (see binary-format + # above for why not a fixed path). The role resolves into an internal fact, so + # sponsord_manual_bin_src is passed as "" per case with nothing to restore. + - name: Create a per-run target-dir root on the control machine (sponsord-dir-*) + ansible.builtin.tempfile: + state: directory + path: "{{ _decdn_ctl_tmp }}" + prefix: sponsord-molecule-dir. + register: _sp_dir_tmp + delegate_to: localhost + become: false + + - name: Run the sponsord dir-mode cases + vars: + _sp_triple: >- + {{ {'x86_64': 'x86_64-unknown-linux-gnu', + 'aarch64': 'aarch64-unknown-linux-gnu'}[ansible_facts['architecture']] }} + block: + - name: "Case sponsord-dir-missing — no binary in the target dir or its cross dir" + block: + - name: Create an empty target/release on the control machine + ansible.builtin.file: + path: "{{ _sp_dir_tmp.path }}/empty/target/release" + state: directory + mode: "0755" + delegate_to: localhost + become: false + + - name: Run sponsord pointed at the empty target dir + ansible.builtin.include_role: + name: sponsord + vars: + sponsord_manual_bin_src: "" + sponsord_release_target_dir: "{{ _sp_dir_tmp.path }}/empty/target/release" + rescue: + - name: Record sponsord-dir-missing rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['sponsord-dir-missing'] }}" + when: ansible_failed_task.name == "Fail if no candidate directory held the binary" + + - name: "Case sponsord-dir-format — the target dir holds a non-ELF binary" + block: + - name: Create target/release on the control machine + ansible.builtin.file: + path: "{{ _sp_dir_tmp.path }}/script/target/release" + state: directory + mode: "0755" + delegate_to: localhost + become: false + + - name: Stage the Python stub as the binary + ansible.builtin.copy: + src: "{{ sponsord_manual_bin_src }}" + dest: "{{ _sp_dir_tmp.path }}/script/target/release/sponsord" + mode: "0755" + delegate_to: localhost + become: false + + - name: Run sponsord pointed at the non-ELF target dir + ansible.builtin.include_role: + name: sponsord + vars: + sponsord_manual_bin_src: "" + sponsord_release_target_dir: "{{ _sp_dir_tmp.path }}/script/target/release" + rescue: + - name: Record sponsord-dir-format rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['sponsord-dir-format'] }}" + when: ansible_failed_task.name == "Require the binary to be an ELF for the host's architecture" + + # POSITIVE: the plain dir is empty, the cross output dir holds an ELF for this + # host, and the dir is given with a trailing slash (which must not shift the + # cross dir one level down). Resolution passes, so the role runs on to the + # treasury gate — counted only if it resolved the cross path exactly. + - name: "Case sponsord-dir-cross — falls back to target//release" + block: + - name: Create target/release and the cross output dir on the control machine + ansible.builtin.file: + path: "{{ item }}" + state: directory + mode: "0755" + loop: + - "{{ _sp_dir_tmp.path }}/cross/target/release" + - "{{ _sp_dir_tmp.path }}/cross/target/{{ _sp_triple }}/release" + delegate_to: localhost + become: false + + - name: Stage an ELF for this host only in the cross output dir + ansible.builtin.copy: + src: /bin/true + dest: "{{ _sp_dir_tmp.path }}/cross/target/{{ _sp_triple }}/release/sponsord" + mode: "0755" + delegate_to: localhost + become: false + + - name: Run sponsord pointed at target/release/ (trailing slash) + ansible.builtin.include_role: + name: sponsord + vars: + sponsord_manual_bin_src: "" + sponsord_release_target_dir: "{{ _sp_dir_tmp.path }}/cross/target/release/" + rescue: + - name: Record sponsord-dir-cross (only if it resolved the cross dir and stopped at the treasury) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['sponsord-dir-cross'] }}" + when: + - ansible_failed_task.name == "Require the treasury keystore and password as regular files" + - _sponsord_bin_src == _sp_dir_tmp.path ~ '/cross/target/' ~ _sp_triple ~ '/release/sponsord' + always: + - name: Remove the target-dir fixtures from the control machine + ansible.builtin.file: + path: "{{ _sp_dir_tmp.path }}" + state: absent + delegate_to: localhost + become: false + + # --- sponsord: host-provisioned secret.env (stop before install) ---------------- + # Each stages the treasury files so the role reaches its secret.env content + # checks, which run before the binary is installed, and removes /etc/sponsord + # (the token the role generated included) afterwards. + - name: "Case sponsord-secret-env — host-provisioned file cases" + vars: + _sp_cases: + - tag: sponsord-rpc-unusable + content: "SPONSORD_RPC_URL=\n" + task: Report a host-provisioned secret.env with no usable SPONSORD_RPC_URL + - tag: sponsord-secret-env-keys + content: "SPONSORD_RPC_URL=https://rpc.example.invalid/\nSPONSORD_BIND=0.0.0.0:8090\n" + task: Refuse keys besides SPONSORD_RPC_URL in a host-provisioned secret.env + - tag: sponsord-secret-env-keys + content: "SPONSORD_RPC_URL=https://rpc.example.invalid/\nexport SPONSORD_POOL_ID=0x01\n" + task: Refuse keys besides SPONSORD_RPC_URL in a host-provisioned secret.env + ansible.builtin.include_tasks: includes/sponsord-secret-env-case.yml + loop: "{{ _sp_cases }}" + loop_control: + loop_var: _sp_case + label: "{{ _sp_case.tag }}" + + # A symlinked secret.env: the host lock-down would chmod the link's target. + - name: "Case sponsord-secret-env-symlink — secret.env is a symlink" + block: + - name: Stage secret.env as a symlink + ansible.builtin.shell: + cmd: | + set -eu + mkdir -p /etc/sponsord + echo 'SPONSORD_RPC_URL=https://rpc.example.invalid/' > /root/sponsord-secret-target + ln -sfn /root/sponsord-secret-target /etc/sponsord/secret.env + changed_when: true + + - name: Run sponsord against the symlinked secret.env + ansible.builtin.include_role: + name: sponsord + vars: + sponsord_rpc_url: "" + rescue: + - name: Record sponsord-secret-env-symlink rejection (only if its assert failed) + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + ['sponsord-secret-env-symlink'] }}" + when: ansible_failed_task.name == "Require secret.env to be a regular file" + always: + - name: Remove the secret.env symlink fixtures + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - /etc/sponsord + - /root/sponsord-secret-target + - name: Confirm every bad value was rejected by the role's own validation ansible.builtin.assert: that: @@ -1629,8 +2115,8 @@ # lets one broken sibling hide behind another case that uses the same gate. - decdn_rejected | sort == _decdn_expected | sort fail_msg: >- - Neither decdn_node nor grafana_alloy rejected every bad value at their - validation asserts. + decdn_node, grafana_alloy and sponsord did not reject every bad value at + their validation asserts. Expected {{ _decdn_expected | sort }}, got {{ decdn_rejected | sort }} — a missing tag means that bad value slipped past validation (it would reach the daemon/crash-loop it at config load, or ship credentials insecurely). @@ -1653,4 +2139,12 @@ "otlp-username-env", "token-shape", "token-shape", "token-collision", "credential-knobs", "token-all-or-nothing", "token-all-or-nothing", "provenance-record", "provenance-record", - "managed-paths", "managed-paths", "managed-paths", "managed-paths", "managed-paths"] + "managed-paths", "managed-paths", "managed-paths", "managed-paths", "managed-paths", + "sponsord-release-version", "sponsord-manual-src", "sponsord-network-unknown", + "sponsord-network-chain", "sponsord-pool-id", "sponsord-zero-address", "sponsord-bind", + "sponsord-limit", "sponsord-limit", "sponsord-bool", "sponsord-rpc-newline", + "sponsord-rpc-gate", "sponsord-treasury", "sponsord-treasury", "sponsord-token", + "sponsord-dir-missing", "sponsord-dir-format", "sponsord-dir-cross", + "sponsord-bind", "sponsord-overwrite-untracked", "sponsord-token", + "sponsord-rpc-unusable", "sponsord-secret-env-keys", "sponsord-secret-env-keys", + "sponsord-secret-env-symlink"] diff --git a/ansible/molecule/validation/includes/sponsord-secret-env-case.yml b/ansible/molecule/validation/includes/sponsord-secret-env-case.yml new file mode 100644 index 0000000..ef8446c --- /dev/null +++ b/ansible/molecule/validation/includes/sponsord-secret-env-case.yml @@ -0,0 +1,39 @@ +--- +# One host-provisioned secret.env case for converge.yml (looped over _sp_case: tag, +# content, task). Stages the treasury files and secret.env, runs sponsord with no +# inventory RPC URL, and records the tag only if the role failed at _sp_case.task. +- name: "Host-provisioned secret.env case {{ _sp_case.tag }}" + block: + - name: Create /etc/sponsord + ansible.builtin.file: + path: /etc/sponsord + state: directory + mode: "0755" + + - name: Stage the treasury files and the host secret.env + ansible.builtin.copy: + dest: "{{ item.dest }}" + content: "{{ item.content }}" + mode: "0600" + loop: + - {dest: /etc/sponsord/treasury-keystore.json, content: "{}\n"} + - {dest: /etc/sponsord/treasury-password, content: "placeholder\n"} + - {dest: /etc/sponsord/secret.env, content: "{{ _sp_case.content }}"} + loop_control: + label: "{{ item.dest }}" + + - name: Run sponsord against the host secret.env + ansible.builtin.include_role: + name: sponsord + vars: + sponsord_rpc_url: "" + rescue: + - name: Record the rejection (only if its assert failed) of {{ _sp_case.tag }} + ansible.builtin.set_fact: + decdn_rejected: "{{ decdn_rejected + [_sp_case.tag] }}" + when: ansible_failed_task.name == _sp_case.task + always: + - name: Remove /etc/sponsord + ansible.builtin.file: + path: /etc/sponsord + state: absent diff --git a/ansible/playbooks/group_vars/all.yml b/ansible/playbooks/group_vars/all.yml new file mode 100644 index 0000000..e003d87 --- /dev/null +++ b/ansible/playbooks/group_vars/all.yml @@ -0,0 +1,17 @@ +--- +# Playbook-adjacent group_vars for every host (see decdn_nodes.yml beside this for +# why playbook-adjacent: they load for any INVENTORY=, an overlay included). +# +# Which daemons the opt-in Grafana Alloy agent scrapes, derived from the groups a +# host is in, so a sponsord-only host does not scrape a decdn-node /metrics that +# does not exist, and a co-located host gets both. Host-scoped on purpose: site.yml +# runs grafana_alloy once per play on a co-located host, and play vars would make +# the two runs render different configs. +# +# Precedence: these beat role defaults, and ALSO the inventory's group_vars/all.yml +# and group vars written inline in hosts.yml, so an override placed in either of +# those is silently ignored. They lose only to the inventory's +# group_vars/.yml for a named group (decdn_nodes, sponsord_hosts, …) and to +# host_vars, so override them there. +grafana_alloy_node_enabled: "{{ 'decdn_nodes' in group_names }}" +grafana_alloy_sponsord_enabled: "{{ 'sponsord_hosts' in group_names }}" diff --git a/ansible/playbooks/site.yml b/ansible/playbooks/site.yml index 8340098..aedaf0f 100644 --- a/ansible/playbooks/site.yml +++ b/ansible/playbooks/site.yml @@ -17,3 +17,7 @@ tags: [observability, decdn_node, node] - role: decdn_node tags: [decdn_node, node] + +# sponsord on hosts in sponsord_hosts (none unless the inventory defines the group). +- name: Provision sponsord + ansible.builtin.import_playbook: sponsord.yml diff --git a/ansible/playbooks/sponsord.yml b/ansible/playbooks/sponsord.yml new file mode 100644 index 0000000..befb125 --- /dev/null +++ b/ansible/playbooks/sponsord.yml @@ -0,0 +1,21 @@ +--- +# sponsord: the deCDN onboarding sponsor (treasury signer + PaymentPool keeper). +# baseline -> grafana_alloy (opt-in, off by default) -> sponsord +# Standalone entry point (`make deploy-sponsord`), also imported by site.yml. It +# needs no decdn-node on the host and opens no firewall hole: its only listener is +# loopback, for the onramp on the same host. A host in both decdn_nodes and +# sponsord_hosts runs baseline and Alloy once per play; both converge to the same +# state, because Alloy's per-daemon toggles are host-scoped +# (playbooks/group_vars/all.yml), not play-scoped. +# Opening and funding the treasury's pool is an operator action — see +# roles/sponsord/README.md. +- name: Provision sponsord + hosts: sponsord_hosts + become: true + roles: + - role: baseline + tags: [baseline] + - role: grafana_alloy + tags: [observability, sponsord] + - role: sponsord + tags: [sponsord] diff --git a/ansible/roles/grafana_alloy/README.md b/ansible/roles/grafana_alloy/README.md index 8ebfa8d..c44b996 100644 --- a/ansible/roles/grafana_alloy/README.md +++ b/ansible/roles/grafana_alloy/README.md @@ -195,9 +195,10 @@ Binary/package removal is manual. | Signal | Job label | Where it shows up | | --- | --- | --- | -| Node `/metrics` | `decdn-node` | your own dashboards/queries | +| Node `/metrics` | `decdn-node` | your own dashboards/queries (`grafana_alloy_node_enabled`) | +| sponsord `/metrics` | `sponsord` | pool balance, top-ups, keeper failures, issued capabilities (`grafana_alloy_sponsord_enabled`); `service_name=sponsord` | | Machine metrics | `integrations/node_exporter` | Grafana Cloud's **Linux Server** integration dashboards + alerts, unmodified | -| journald | `integrations/node_exporter` | the same integration's logs dashboards (correlated by `instance`); the daemon's own unit (`decdn-node.service`) also carries `service_name=` (default `decdn-node`) | +| journald | `integrations/node_exporter` | the same integration's logs dashboards (correlated by `instance`); the daemon's own unit (`decdn-node.service`) also carries `service_name=` (default `decdn-node`), and `sponsord.service` carries `service_name=sponsord` when that scrape is on | | Alloy self-metrics | `integrations/alloy` | agent health | | Traces | — | Application Observability | @@ -235,6 +236,12 @@ deliberately lean and every lever is a variable: regex is applied to that JSON level for this unit. Values follow journald's keywords, so query `level="warning"`, not `warn`; TRACE becomes `trace`. Lines that are not JSON keep the journald level and are never priority-dropped. +- sponsord, when `grafana_alloy_sponsord_enabled` is on, gets the same treatment + from its plain-text lines (` WARN : …`; its unit sets + `NO_COLOR=1`). The raw lines it writes when it exits on an error (`Error: …`, + `Caused by: …`, a panic) are labelled `error`. Other unlevelled lines keep + journald's level. With the toggle off, nothing re-levels its stream, so an + info-dropping priority regex would drop its warnings and errors too. ## Hardening: the two relaxations machine monitoring requires @@ -270,7 +277,7 @@ upstream version/sha256). Highlights: | `grafana_alloy_host_metrics_enabled` | `true` | Machine metrics (Alloy's in-process `node_exporter`) | | `grafana_alloy_host_collectors` | 13 collectors | Replacement set; `_host_extra_collectors` adds, `_host_disable_collectors` vetoes | | `grafana_alloy_host_scrape_interval` | `60s` | Host metrics only; the node keeps `30s` | -| `grafana_alloy_host_systemd_unit_include` | `(decdn-node\|alloy\|ssh\|sshd)\.service` | Scope of the per-unit `systemd` collector | +| `grafana_alloy_host_systemd_unit_include` | `(decdn-node\|sponsord\|alloy\|ssh\|sshd)\.service` | Scope of the per-unit `systemd` collector | | `grafana_alloy_logs_enabled` | `true` | journald → Grafana Cloud Loki | | `grafana_alloy_logs_max_age` | `12h` | Bounds the catch-up burst after an outage | | `grafana_alloy_self_metrics_enabled` | `true` | Alloy's own health | @@ -279,6 +286,9 @@ upstream version/sha256). Highlights: | `grafana_alloy_env_checksum_file` | `/etc/grafana-alloy.env.sha256` | Provenance record (` `) enabling the adoption/overwrite guards; must remain exactly `.sha256` | | `grafana_alloy_overwrite_host_file` | `false` | Explicit opt-in letting an inventory token rewrite a hand-edited/untracked env file (all-or-nothing: extra `GC_*` lines are discarded) | | `grafana_alloy_node_job` / `_host_job` / `_self_job` / `_logs_job` | see table above | Job labels; the `integrations/…` ones are what Grafana Cloud's dashboards match | +| `grafana_alloy_node_enabled` | `true` | Scrape decdn-node's `/metrics`. This repo's playbooks set it from membership of `decdn_nodes` (`playbooks/group_vars/all.yml`) | +| `grafana_alloy_sponsord_enabled` | `false` | Scrape sponsord's `/metrics`, label and re-level its journald stream. Set from membership of `sponsord_hosts` by the same file | +| `grafana_alloy_sponsord_job` / `_sponsord_service_name` | `sponsord` / `sponsord` | sponsord's job label and `service_name` | Label variables (`service_name`, `service_namespace`, `instance_id`, `deployment_environment`, `region`) ship on most series, spans and log lines — @@ -329,9 +339,10 @@ Three layers, because the first one cannot prove correctness on its own: container with no D-Bus and no real journal, so the `systemd` collector and journald collect nothing there — expected, and not what this layer proves. 2. **`make lint-alloy`** (CI job `alloy-config`) — renders these templates in - eight variable combinations (defaults, minimal identity, fully overridden, + twelve variable combinations (defaults, minimal identity, fully overridden, host-metrics-only, logs-only, every sub-knob off, inventory-supplied - endpoints, vetoed collector + blanked node job) and feeds them to the REAL pinned Alloy binary: `alloy validate` + endpoints, OTLP username fallback, vetoed collector + blanked node job, + info-dropping guardrail, sponsord-only, node + sponsord co-located) and feeds them to the REAL pinned Alloy binary: `alloy validate` (component graph, not just syntax), a check that every `ExecStart` flag exists in `alloy run --help`, and greps proving each Jinja branch actually switched — including that "every sub-knob off" reproduces the pre-machine-monitoring diff --git a/ansible/roles/grafana_alloy/defaults/main.yml b/ansible/roles/grafana_alloy/defaults/main.yml index 6e873e5..574d8e3 100644 --- a/ansible/roles/grafana_alloy/defaults/main.yml +++ b/ansible/roles/grafana_alloy/defaults/main.yml @@ -135,6 +135,21 @@ grafana_alloy_scrape_interval: 30s # dashboards, recording rules and alerts. grafana_alloy_node_job: decdn-node +# Which daemons on this host to scrape. The node scrape stays on by default (the +# role's original behaviour); a host that runs only sponsord turns it off. This +# repo's playbooks derive both from group membership (playbooks/group_vars/all.yml: +# decdn_nodes / sponsord_hosts), so set them here only when using the role from +# your own playbook. +grafana_alloy_node_enabled: true +# sponsord's loopback /metrics (roles/sponsord: sponsord_bind_address:sponsord_port, +# default 127.0.0.1:8090), with its own job and service_name. It also labels +# sponsord's journald stream: a service_name, and a `level` parsed from each line. +# That exempts sponsord.service from the journald-priority guardrail, which is then +# applied to the parsed level instead. +grafana_alloy_sponsord_enabled: false +grafana_alloy_sponsord_job: sponsord +grafana_alloy_sponsord_service_name: sponsord + # --- Host (machine) metrics --------------------------------------------------- # Alloy's built-in node_exporter. Runs IN-PROCESS: no extra listener, no extra # port, no firewall hole (AGENTS.md hard rule 2 is unaffected). Stamped with the @@ -196,7 +211,7 @@ grafana_alloy_host_disk_device_exclude: '^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n # Per-unit state, deliberately scoped: one series set per unit on the box is a # real cost, and only these units matter. Units must match include AND NOT match # exclude. -grafana_alloy_host_systemd_unit_include: '(decdn-node|alloy|ssh|sshd)\.service' +grafana_alloy_host_systemd_unit_include: '(decdn-node|sponsord|alloy|ssh|sshd)\.service' grafana_alloy_host_systemd_unit_exclude: '.+\.(automount|device|mount|scope|slice)' # --- Agent self-monitoring ---------------------------------------------------- diff --git a/ansible/roles/grafana_alloy/tasks/preflight.yml b/ansible/roles/grafana_alloy/tasks/preflight.yml index 32b18fe..19fb7ec 100644 --- a/ansible/roles/grafana_alloy/tasks/preflight.yml +++ b/ansible/roles/grafana_alloy/tasks/preflight.yml @@ -52,10 +52,13 @@ - grafana_alloy_self_metrics_enabled is boolean - grafana_alloy_logs_enabled is boolean - grafana_alloy_host_include_exporter_metrics is boolean + - grafana_alloy_node_enabled is boolean + - grafana_alloy_sponsord_enabled is boolean fail_msg: >- - grafana_alloy_host_metrics_enabled, _self_metrics_enabled, _logs_enabled - and _host_include_exporter_metrics must be booleans (true/false), not - quoted strings — each gates whole component groups in config.alloy. + grafana_alloy_host_metrics_enabled, _self_metrics_enabled, _logs_enabled, + _host_include_exporter_metrics, _node_enabled and _sponsord_enabled must be + booleans (true/false), not quoted strings — each gates whole component + groups in config.alloy. # An unknown collector name is NOT a startup crash: Alloy resolves collectors by # map lookup and silently ignores names it does not know, so a typo ships an @@ -123,6 +126,7 @@ - {name: grafana_alloy_host_job, value: "{{ grafana_alloy_host_job | string }}", allow_empty: false} - {name: grafana_alloy_self_job, value: "{{ grafana_alloy_self_job | string }}", allow_empty: false} - {name: grafana_alloy_logs_job, value: "{{ grafana_alloy_logs_job | string }}", allow_empty: false} + - {name: grafana_alloy_sponsord_job, value: "{{ grafana_alloy_sponsord_job | string }}", allow_empty: false} loop_control: label: "{{ item.name }}" @@ -153,10 +157,11 @@ resource attributes on most series/spans/log lines, so they are restricted to a low-cardinality charset (letters, digits, dot, underscore, slash, ≤ 63 chars): {{ item }} does not fit. Offending variables: - grafana_alloy_service_name / service_namespace / instance_id / - deployment_environment / region. + grafana_alloy_service_name / sponsord_service_name / service_namespace / + instance_id / deployment_environment / region. loop: - "{{ grafana_alloy_service_name }}" + - "{{ grafana_alloy_sponsord_service_name }}" - "{{ grafana_alloy_service_namespace }}" # _ga_instance_id is REQUIRED (never "") — it has a derived non-empty default. - "{{ _ga_instance_id }}" diff --git a/ansible/roles/grafana_alloy/templates/config.alloy.j2 b/ansible/roles/grafana_alloy/templates/config.alloy.j2 index 0d205fb..42cabe9 100644 --- a/ansible/roles/grafana_alloy/templates/config.alloy.j2 +++ b/ansible/roles/grafana_alloy/templates/config.alloy.j2 @@ -50,6 +50,9 @@ {% if ga_otlp_user | length > 0 %}{{ ga_otlp_user | tojson }}{% else %}coalesce(sys.env("GC_OTLP_USERNAME"), {{ ga_endpoint(grafana_alloy_prom_username, 'GC_PROM_USERNAME') }}){% endif %} {%- endmacro %} +{# The decdn-node scrape, off on a host that runs no node (grafana_alloy_node_enabled; + playbooks/group_vars/all.yml derives it from the decdn_nodes group). #} +{% if grafana_alloy_node_enabled | bool %} prometheus.scrape "decdn_node" { targets = [ { "__address__" = "127.0.0.1:{{ decdn_metrics_port | default(9090) }}", @@ -85,6 +88,40 @@ prometheus.relabel "decdn_identity" { {% endfor %} } +{% endif %} +{% if grafana_alloy_sponsord_enabled | bool %} +{# sponsord's own /metrics (pool balance, keeper top-ups, issued capabilities) on + its loopback listener — the same port as its API, but /metrics needs no token. + Its own relabel, because the shared identity stamps service_name=decdn-node. #} +prometheus.scrape "sponsord" { + targets = [ + { "__address__" = {{ (sponsord_bind_address | default('127.0.0.1') ~ ':' ~ sponsord_port | default(8090)) | tojson }}, + "instance" = "{{ _ga_instance_id }}", + "job" = {{ grafana_alloy_sponsord_job | tojson }} }, + ] + forward_to = [prometheus.relabel.sponsord_identity.receiver] + scrape_interval = {{ grafana_alloy_scrape_interval | to_json }} +} + +{% set ga_sponsord_labels = [ + ["service_name", grafana_alloy_sponsord_service_name], + ["service_namespace", grafana_alloy_service_namespace], + ["service_instance_id", _ga_instance_id], + ["deployment_environment", grafana_alloy_deployment_environment], + ["region", grafana_alloy_region], + ] %} +prometheus.relabel "sponsord_identity" { + forward_to = [prometheus.remote_write.cloud.receiver] +{% for name, value in ga_sponsord_labels if value | length > 0 %} + + rule { + target_label = {{ name | tojson }} + replacement = {{ value | tojson }} + } +{% endfor %} +} + +{% endif %} prometheus.remote_write "cloud" { endpoint { url = {{ ga_endpoint(grafana_alloy_prom_url, 'GC_PROM_REMOTE_WRITE_URL') }} @@ -269,10 +306,10 @@ loki.relabel "journal_rules" { // Cost guardrail: the regex is anchored to the WHOLE value by Prometheus // relabel semantics, so "debug" drops debug and nothing else. // - // decdn-node is exempt: journald reports every line it writes as info, so + // decdn-node{% if grafana_alloy_sponsord_enabled | bool %} and sponsord are{% else %} is{% endif %} exempt: journald reports every line it writes as info, so // judging it by priority would be judging it by a level it never set (a // regex containing "info" would drop its errors). loki.process - // "daemon_level" below applies the same regex to its real, JSON level. The + // "daemon_level" below applies the same regex to its real{% if not grafana_alloy_sponsord_enabled | bool %}, JSON{% endif %} level. The // exemption blanks a scratch copy of the keyword for that one unit; like // every __-prefixed label it never leaves this ruleset. rule { @@ -282,7 +319,7 @@ loki.relabel "journal_rules" { rule { source_labels = ["__journal__systemd_unit"] - regex = "decdn-node\\.service" + regex = {{ ('(?:decdn-node|sponsord)\\.service' if grafana_alloy_sponsord_enabled | bool else 'decdn-node\\.service') | tojson }} target_label = "__priority_guardrail" replacement = "" } @@ -337,16 +374,9 @@ loki.source.journal "host" { with no level, or an unexpected value — leaves decdn_level unset (the template stage deletes a key it renders empty), so stage.labels and stage.drop skip the line and its journald level stands. #} -loki.process "daemon_level" { - forward_to = [loki.relabel.journal_identity.receiver] - - stage.match { - selector = `{unit="decdn-node.service"}` - - stage.json { - expressions = { decdn_level = "level" } - } - +{# The stages after a daemon's own level extraction are identical for both + daemons: normalise, label, and apply the guardrail to the real level. #} +{% macro ga_daemon_level_stages() %} stage.template { source = "decdn_level" template = {% raw %}`{{ $l := ToLower (TrimSpace (print .Value)) }}{{ if eq $l "warn" "warning" }}warning{{ else if eq $l "trace" "debug" "info" "error" }}{{ $l }}{{ end }}`{% endraw +%} @@ -365,7 +395,49 @@ loki.process "daemon_level" { expression = {{ ('^(?:' ~ grafana_alloy_logs_drop_priority_regex ~ ')$') | tojson }} } {% endif %} +{%- endmacro %} +loki.process "daemon_level" { + forward_to = [loki.relabel.journal_identity.receiver] + + stage.match { + selector = `{unit="decdn-node.service"}` + + stage.json { + expressions = { decdn_level = "level" } + } + +{{ ga_daemon_level_stages() }} } +{% if grafana_alloy_sponsord_enabled | bool %} + + // sponsord logs tracing-subscriber's plain-text format with colour off + // (NO_COLOR=1 in its unit): " WARN : ". The + // level is the second whitespace-separated field. + stage.match { + selector = `{unit="sponsord.service"}` + + stage.regex { + expression = `^\S+\s+(?PTRACE|DEBUG|INFO|WARN|ERROR)\s` + } + +{{ ga_daemon_level_stages() }} } + + // What sponsord writes raw when it exits on an error carries no level field: + // anyhow's "Error: ..." and "Caused by:" lines, and a panic message. They are + // the lines that say WHY it stopped, so they are errors. Any other unlevelled + // line keeps its journald level. + stage.match { + selector = `{unit="sponsord.service"} |~ "^(Error: |Caused by:|thread '.+' panicked)"` + + stage.template { + source = "decdn_level" + template = "error" + } + + stage.labels { + values = { level = "decdn_level" } + } } +{% endif %} } {# Same job/instance pair as the host metrics above — Grafana Cloud's Linux @@ -409,6 +481,17 @@ loki.relabel "journal_identity" { replacement = {{ grafana_alloy_service_name | tojson }} } {% endif %} +{% if grafana_alloy_sponsord_enabled | bool and grafana_alloy_sponsord_service_name | length > 0 %} +{# The same for sponsord's stream (the unit roles/sponsord installs; also named in + the journal_rules exemption and the daemon_level selector above). #} + + rule { + source_labels = ["unit"] + regex = "sponsord\\.service" + target_label = "service_name" + replacement = {{ grafana_alloy_sponsord_service_name | tojson }} + } +{% endif %} } loki.write "cloud" { diff --git a/ansible/roles/sponsord/README.md b/ansible/roles/sponsord/README.md new file mode 100644 index 0000000..d7d544d --- /dev/null +++ b/ansible/roles/sponsord/README.md @@ -0,0 +1,147 @@ +# Role: `sponsord` + +Deploys [sponsord](https://github.com/decdn/sponsord), the deCDN onboarding sponsor. +It holds the **treasury wallet**, which owns one `PaymentPool` pool, and does two +things: + +- signs capped, expiring capabilities for callers that present its API token; +- tops the pool up from the treasury whenever it runs low. + +It is independent of the node. It needs an RPC endpoint and the PaymentPool contract, +never a local `decdn-node`, so it can run on its own host or beside a node. + +The role covers the daemon only. Its public companion, `sponsord-onramp` (Turnstile +gate, installers, behind a TLS reverse proxy), is not deployed here yet. The onramp +expects the daemon on the same host and reads the API token from +`/etc/sponsord/api-token`. + +## What it does + +- **Installs the binary.** + - `manual` (the default) copies a binary built from a `decdn/sponsord` checkout + on the control machine (`sponsord_manual_bin_src`, or + `sponsord_release_target_dir` with an ELF/arch check). + - `release` downloads `sponsord-v` and verifies it against the + GPG-signed `SHA256SUMS`, using `files/sponsord-release-KEYS.asc`, a copy of + upstream's `KEYS`. + - Upstream has cut no release yet, so `manual` is the only working method today. +- **Places the secrets.** All of them live under `/etc/sponsord`, root 0600: + + | File | Who provides it | + |------|-----------------| + | `api-token` | The role generates it on the host when absent and never replaces it, because the onramp holds a copy. | + | `treasury-keystore.json`, `treasury-password` | **You** do. The role never creates the treasury wallet. | + | `secret.env` (`SPONSORD_RPC_URL`) | `sponsord_rpc_url` from inventory, or a file you write on the host. The two-way rules match `decdn_rpc_url`: an empty `sponsord_rpc_url` with a file the role wrote earlier fails the deploy, because that means `secret.yml` went missing. A host-written file may hold `SPONSORD_RPC_URL` **only**: as an `EnvironmentFile` it would override every other setting, so the role refuses any other key in it. | + +- **Writes non-secret settings** to `/etc/sponsord/sponsord.env` (0644): bind + address, chain, PaymentPool, pool id and any limits you set. +- **Runs a hardened unit.** `sponsord.service` runs with `DynamicUser=`, which gives + it no persistent uid, no state directory and a read-only filesystem. systemd hands + it the three secret files as `LoadCredential=` credentials, so they never sit in + its environment. + - The keystore needs one extra step. Newer systemd writes credentials 0440 + (seen on Ubuntu 24.04's systemd 255; Debian 12's 252 still writes 0400), and + sponsord refuses a keystore with any group bit. So `ExecStartPre` copies that + one credential at 0600 into the unit's private, tmpfs `RuntimeDirectory`. + - Upstream's reference unit reads it straight from the credentials directory, and + fails on Ubuntu 24.04 + ([decdn/sponsord#36](https://github.com/decdn/sponsord/issues/36)). +- **Gates the deploy on `/healthz`.** sponsord binds only after it has decrypted the + keystore and confirmed on-chain that the treasury owns `sponsord_pool_id`. If + `/healthz` does not answer 200 within the readiness window, the deploy fails. + sponsord has no `config validate` command, so this is the config check. +- **Restarts on out-of-band changes.** The role records a hash of everything the + daemon starts from (credentials, both env files, the unit, the binary), once the + daemon is healthy. If any of them differs on the next run, it restarts the daemon: + - you replaced a credential or edited a host-provisioned `secret.env`; + - an earlier run changed a file and then failed before its restart handler ran. + +## Before the first deploy + +1. **Create the treasury wallet** with the decdn CLI (`decdn key-gen`). **Open its + pool** from it (`decdn pool open`), which prints the pool id. Fund the wallet with + USDC plus gas. + - This is a hot key: hold only a few top-ups' worth. +2. **Copy the wallet files onto the host**, as root: + + ```bash + umask 077 + sudo mkdir -p /etc/sponsord + sudo install -m 0600 treasury-keystore.json /etc/sponsord/treasury-keystore.json + sudo install -m 0600 treasury-password /etc/sponsord/treasury-password + ``` + +3. **Provide the RPC URL**, either way: + - `/etc/sponsord/secret.env` on the host (see `files/secret.env.example`); + - `sponsord_rpc_url` in the git-ignored `host_vars//secret.yml`. +4. **Set inventory:** `sponsord_network: arbitrum-sepolia` (or `sponsord_chain_id` + + `sponsord_payment_pool_address`), plus `sponsord_pool_id`. + +Then run `make deploy-sponsord` from `ansible/`. Hosts in `sponsord_hosts` are also +deployed by `make deploy`, through `site.yml`. + +## Variables + +See [`defaults/main.yml`](defaults/main.yml) for the full list with comments. + +| Variable | Default | Notes | +|----------|---------|-------| +| `sponsord_install_method` | `manual` | `manual` or `release` | +| `sponsord_manual_bin_src` / `sponsord_release_target_dir` | `""` | `manual` only. Set one. | +| `sponsord_version` | `""` | Required in `release` mode, e.g. `0.1.0` → tag `sponsord-v0.1.0`. | +| `sponsord_network` | `""` | Network profile: `arbitrum-sepolia` supplies chain id and PaymentPool. | +| `sponsord_chain_id`, `sponsord_payment_pool_address` | from profile | Explicit values win over the profile. | +| `sponsord_pool_id` | `""` | **Required.** 0x + 64 hex. | +| `sponsord_rpc_url` | `""` | **Sensitive.** Leave empty to provision `secret.env` on the host. | +| `sponsord_bind_address` / `sponsord_port` | `127.0.0.1` / `8090` | IPv4 loopback only (asserted). | +| `sponsord_max_spending_cap_micro_usdc`, `_max_ttl_secs`, `_pool_low_water_micro_usdc`, `_pool_refill_micro_usdc`, `_pool_watch_interval_secs` | `""` | `""` uses the daemon's default. These are economic choices, not repo facts. | +| `sponsord_generate_api_token` | `true` | `false`: provision `/etc/sponsord/api-token` yourself (≥ 32 bytes). | +| `sponsord_secret_env_overwrite_host_file` | `false` | Confirm that `sponsord_rpc_url` may replace a `secret.env` the role did not write. | +| `sponsord_stop_timeout_sec` | `120` | Cap on the graceful stop. The daemon waits for a top-up it already sent. | +| `sponsord_readiness_retries` / `_delay` | `30` / `2` | `/healthz` window (about 60 s). | + +## Observability + +With `decdn_grafana_cloud_enabled: true`, the `grafana_alloy` role ships sponsord's +`/metrics` as `job="sponsord"` (pool balance, top-ups, keeper failures, issued +capabilities) and its journal with `service_name="sponsord"`. This repo's playbooks +turn that on for hosts in `sponsord_hosts`, and turn the node scrape off on a host +that is not in `decdn_nodes` (`playbooks/group_vars/all.yml`). From your own +playbook, set `grafana_alloy_sponsord_enabled` / `grafana_alloy_node_enabled` +yourself. + +## Day 2 + +- **Rotate the API token:** write the new token to `/etc/sponsord/api-token` and to + the onramp, then re-run the role, which restarts sponsord. +- **Rotate the treasury:** replace the keystore and password files, then re-run. + The new wallet must own `sponsord_pool_id`, or sponsord refuses to start and the + deploy fails at `/healthz`. +- **Logs:** `journalctl -u sponsord`. Plain text, with colour off. In Loki its `level` label comes from the line itself, not journald's priority, which is info for every line. +- **Removal** (no decommission playbook yet; the daemon itself keeps no state): + + ```bash + sudo systemctl disable --now sponsord + sudo rm /etc/systemd/system/sponsord.service + sudo systemctl daemon-reload + sudo rm -f /usr/local/bin/sponsord + sudo rm -rf /usr/local/lib/sponsord # the release version stamp + ``` + + Then delete `/etc/sponsord` (secrets, env files and the role's two `.sha256` + records) once the treasury files are safe elsewhere. + +## Testing + +- `molecule/sponsord` runs `playbooks/sponsord.yml` on a host with no node + (Debian 12 and Ubuntu 24.04), against a stub daemon. It checks: + - idempotence; + - a restart after a credential rotation; + - that the `/healthz` gate fails the deploy when the daemon will not start; + - each `secret.env` hand-off between inventory and host, with its guards; + - release mode against a locally signed mirror: the version stamp, a re-run + with the mirror down, and rejection of a bad checksum, a bad signature and + a binary that is not the pinned version; + - the Alloy toggles. +- `molecule/grafana-cloud` co-locates sponsord with a node. +- `molecule/validation` holds the negative cases. diff --git a/ansible/roles/sponsord/defaults/main.yml b/ansible/roles/sponsord/defaults/main.yml new file mode 100644 index 0000000..d9b5d3e --- /dev/null +++ b/ansible/roles/sponsord/defaults/main.yml @@ -0,0 +1,122 @@ +--- +# sponsord role defaults. sponsord is the deCDN onboarding sponsor: it holds the +# treasury wallet that owns one PaymentPool pool, signs capped, expiring +# capabilities for callers that present its API token, and tops the pool up from +# the treasury when it runs low. It is independent of the node: it needs an RPC +# endpoint and the PaymentPool contract, never a local decdn-node. +# +# Per-host values belong in host_vars// (non-secret in main.yml, the RPC URL +# — if the role should author it — in the git-ignored secret.yml). Upstream: +# github.com/decdn/sponsord, docs/operator.md. + +# --- Install method ----------------------------------------------------------- +# "release": download the pinned sponsord-v GitHub Release tarball and +# verify it against the GPG-signed SHA256SUMS. "manual": copy a locally-built +# binary from the control machine. Upstream has cut NO sponsord-v* tag yet, so +# "release" has nothing to fetch and the default is "manual"; flip it and pin +# sponsord_version once a release exists. +sponsord_install_method: manual # "release" | "manual" +sponsord_manual_bin_src: "" # manual: control-machine path to the sponsord binary +# Alternative to sponsord_manual_bin_src: the control-machine `target/release` dir. +# The role derives the binary path from it (falling back to the cross / `--target` +# output dir target//release) and checks it is an ELF for the host's +# architecture before shipping it. sponsord_manual_bin_src wins when both are set. +sponsord_release_target_dir: "" + +# --- Release to install (REQUIRED in "release" mode) --------------------------- +sponsord_version: "" # e.g. "0.1.0" -> tag sponsord-v0.1.0 +# Release target triple, derived from the host's gathered architecture (see +# vars/main/arch.yml). Set it explicitly only for a play with gather_facts: false. +sponsord_target: "{{ _sponsord_arch_targets.get(ansible_facts.get('architecture', ''), '') }}" +sponsord_release_base: "https://github.com/decdn/sponsord/releases/download" +# Verify SHA256SUMS.asc against sponsord_release_keyring before trusting the +# manifest. Set false only for an air-gapped mirror that strips the signature. +sponsord_verify_release_signature: true +# Defaults to the copy of upstream's KEYS vendored with this role. +sponsord_release_keyring: sponsord-release-KEYS.asc + +# --- Chain ---------------------------------------------------------------------- +# Network profile. Set sponsord_network to a chain upstream's `decdn config init +# --chain ` knows (today only "arbitrum-sepolia") and chain_id plus the +# PaymentPool address default to that deployment's manifest values, mirrored into +# vars/main/networks.yml by scripts/sync-network-profiles.py. An explicit inventory +# value still wins. Leave it "" to supply both yourself. +sponsord_network: "" +# The selected profile ({} when sponsord_network is "" or unknown — the role +# asserts the name first). Internal: set sponsord_network, not this. +_sponsord_profile: "{{ _sponsord_network_profiles.get(sponsord_network, {}) }}" +sponsord_chain_id: "{{ _sponsord_profile.get('chain_id', '') }}" +sponsord_payment_pool_address: "{{ _sponsord_profile.get('payment_pool_address', '') }}" +# REQUIRED. The pool this treasury owns: 0x + 64 hex, printed by `decdn pool open` +# run once from the treasury wallet (docs/operator.md upstream). The daemon checks +# at boot that the treasury keystore's address owns it on-chain and exits if not. +sponsord_pool_id: "" + +# The RPC endpoint (SPONSORD_RPC_URL). It may embed a provider API key, so it has +# TWO homes, exactly like decdn_node's decdn_rpc_url: leave it "" and provision a +# 0600 /etc/sponsord/secret.env on the host (files/secret.env.example) so the +# secret never touches the control machine, or set it in the git-ignored +# host_vars//secret.yml and the role authors that file for you. +sponsord_rpc_url: "" # SENSITIVE + +# --- Listener ------------------------------------------------------------------- +# One listener carries the authenticated API and the unauthenticated /healthz and +# /metrics. It stays on IPv4 loopback (AGENTS.md hard rule 2): the onramp that calls +# it runs on the same host, and the role asserts the address is 127.x. +sponsord_bind_address: 127.0.0.1 +sponsord_port: 8090 + +# --- Issuance + keeper limits ----------------------------------------------------- +# "" omits the variable so the binary's own default applies (shown in brackets). +# These are economic decisions for the operator, not facts this repo carries. +sponsord_max_spending_cap_micro_usdc: "" # [5000000 = $5] largest cap per capability +sponsord_max_ttl_secs: "" # [172800 = 48h] longest capability TTL +sponsord_pool_low_water_micro_usdc: "" # [20000000 = $20] top up below this +sponsord_pool_refill_micro_usdc: "" # [100000000 = $100] each top-up adds this +sponsord_pool_watch_interval_secs: "" # [3600] how often the keeper checks the pool + +# --- Secrets on the host -------------------------------------------------------- +# Root-owned under sponsord_etc: the secret files 0600, sponsord.env 0644 (it holds +# no secret). systemd reads the API token, treasury keystore and password as root +# and hands them to the DynamicUser service as credentials (LoadCredential=); the +# unit copies the keystore 0600 into its private runtime directory, because +# upstream rejects the 0440 credential file newer systemd (255) writes. secret.env reaches +# the daemon as an EnvironmentFile. +sponsord_etc: /etc/sponsord +sponsord_env_file: /etc/sponsord/sponsord.env # non-secret settings (0644, role-owned) +sponsord_secret_env_file: /etc/sponsord/secret.env # SPONSORD_RPC_URL (0600) +sponsord_api_token_file: /etc/sponsord/api-token # bearer token (0600) +sponsord_treasury_keystore_file: /etc/sponsord/treasury-keystore.json # operator-provisioned (0600) +sponsord_treasury_password_file: /etc/sponsord/treasury-password # operator-provisioned (0600) +# " " record of who last wrote secret.env (the role from inventory, +# or the operator on the host), as decdn_node's decdn_env_checksum_file. Written as +# soon as the file is final, so a failed run cannot leave it stale. Restarts are +# tracked by sponsord_inputs_checksum_file, not by this. +sponsord_secret_env_checksum_file: /etc/sponsord/.secret.env.sha256 +# Confirm that rendering secret.env from inventory may replace a file the role did +# not write: one whose record is missing, says `host`, or no longer matches. +sponsord_secret_env_overwrite_host_file: false +# sha256 record of every restart input (credentials, both env files, unit, binary) +# as of the last converge that saw the daemon healthy. A rotated credential, an +# out-of-band edit, or a restart lost to a failed run is caught on the next converge. +sponsord_inputs_checksum_file: /etc/sponsord/.inputs.sha256 + +# Generate the API token on the host when sponsord_api_token_file is absent (32 +# random bytes, hex). An existing token is never replaced: the onramp holds a copy. +# false = the operator provisions it (>= 32 bytes, one line). +sponsord_generate_api_token: true + +sponsord_bin: /usr/local/bin/sponsord + +# systemd TimeoutStopSec. On SIGTERM the daemon stops serving, then waits for a +# pool top-up it has already sent to settle; there is no internal timeout, so this +# is the cap before SIGKILL. +sponsord_stop_timeout_sec: 120 + +# --- Readiness -------------------------------------------------------------------- +# The daemon binds only after it has decrypted the keystore and confirmed on-chain +# that the treasury owns sponsord_pool_id, so /healthz answering 200 is the role's +# config gate (sponsord has no `config validate`). Unlike decdn_node's advisory +# probe, a timeout here FAILS the deploy. Window = retries x delay (~60s). +sponsord_readiness_retries: 30 +sponsord_readiness_delay: 2 diff --git a/ansible/roles/sponsord/files/secret.env.example b/ansible/roles/sponsord/files/secret.env.example new file mode 100644 index 0000000..bac6cf0 --- /dev/null +++ b/ansible/roles/sponsord/files/secret.env.example @@ -0,0 +1,20 @@ +# /etc/sponsord/secret.env — sponsord's secret environment file. +# +# Provision this ON THE TARGET HOST, as root, and leave sponsord_rpc_url empty in +# inventory: the role then never reads this file, it only enforces 0600 root and +# checks that a SPONSORD_RPC_URL line exists. systemd reads it before starting the +# service. +# +# umask 077 # never briefly world-readable +# sudo mkdir -p /etc/sponsord +# sudo tee /etc/sponsord/secret.env >/dev/null # then paste the line, Ctrl-D +# sudo chmod 600 /etc/sponsord/secret.env # sudo may apply its own umask +# +# After editing it, re-run the playbook (the role notices and restarts sponsord) +# or `sudo systemctl restart sponsord`. +# +# FORMAT: systemd's EnvironmentFile parser, not a shell. One KEY=value per line, no +# `export`, no expansion. A line it cannot parse is SKIPPED with only a journal +# warning, so single-quote a value containing a space, quote, backslash or `$`. + +SPONSORD_RPC_URL='https://arb-sepolia.example-provider.invalid/v2/' diff --git a/ansible/roles/sponsord/files/sponsord-release-KEYS.asc b/ansible/roles/sponsord/files/sponsord-release-KEYS.asc new file mode 100644 index 0000000..047854b --- /dev/null +++ b/ansible/roles/sponsord/files/sponsord-release-KEYS.asc @@ -0,0 +1,81 @@ +Public keys trusted to sign sponsord releases. Import with `gpg --import KEYS`. +The same maintainer keys sign decdn's releases (decdn/decdn's KEYS). + +A release tag, the SHA256SUMS manifest, and each image's SBOM and digest file +are each signed by one maintainer's personal key. A good signature from ANY +key in this file is authentic — see SECURITY.md. Adding or removing a +maintainer is a change to this file. + + Ant Somers + DA75 1570 6F18 73D2 74D8 A369 9E11 A9FF D62D AADB + +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mDMEanigZRYJKwYBBAHaRw8BAQdAXB6xg4XVGmve59MN5EUHPlEVhZuSO2V0TYQa +j0l7ASu0GkFudCBTb21lcnMgPGFudEBkZWNkbi5vcmc+iJkEExYKAEEWIQTadRVw +bxhz0nTYo2meEan/1i2q2wUCanigZQIbAwUJBaOagAULCQgHAgIiAgYVCgkICwIE +FgIDAQIeBwIXgAAKCRCeEan/1i2q2z4VAP9R8ha+nmRqgWnZh7N8VInlO71pXbeW +8GIYaZxp1FdgBQD+L+U2XomAtXAK7Yh52DOUfhQhq2ME2AXutkGeA2UEoQm4OARq +eKBlEgorBgEEAZdVAQUBAQdA0M8mYVvDIMRDYQHurFQdYhmCKjjxuLFNyyEtb82H +/VEDAQgHiHgEGBYKACAWIQTadRVwbxhz0nTYo2meEan/1i2q2wUCanigZQIbDAAK +CRCeEan/1i2q2yLLAQC/TKs0jU6nQu7oot+sAJI754t0fktoAks1dfb8k8rKOgEA +2JAAMJACUKFQArfPQrekJvVL2ZMINf8AdU7FXPt2Jwc= +=ulJ+ +-----END PGP PUBLIC KEY BLOCK----- + + Alper Gundogdu + E27B 9A2D 2519 1E8E C90B F8AE 57E2 823C 16CC D376 + +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBGqz2LQBEACyX15adtYJ0hV7kisaybQWy0HVQFWDJ+kQqRPYXtACEfDbkMZL +uaupsHoXYqTykAjK4tchWZFJNppUvO8zXIM+c4BPwZxW004k5Zh5CtkaLRCqco9m +xkrnsbp0zb9QWPMlWKJzFk3j8D2mARgdYZLNG6fonmaeVIyD2F2VTBeNUZ01KAhX +49Yd+7IHPyCiV73TOIoqgZ3U3EFz+UVmOa8QSuUce4OelUkJcgVLOUxu2LQVLm3A +qK1wtxLJSi5mr526lCbkaIoQZUrg2Z8NN8LbYIycYZkZS8hgGl4hr88uYTSSx0/H +JEuBCqvdylPlNEoDe/ZGdbBC0pmFNCyWn2bIM6FJWdEcAJtitDaIUocDSEcSewHZ +rxkBSq0HMh7T8nsaRksl+NvH1mIphDFmjFk4EvxFHZztHi1ncucBiEnpNaM8AciV +xNUsGQFNB/7MQbvj8KWTfA0gq0ijEkr48/vxF0tMiffTC+z9ThtnHyltbJnqMiAm +p12x5jtX9Cv8Wp4dq3utHJt/86jL31Q1X+7g3gEPFH+wa+QCG/vT4LL+zk/XXWEd +R3JBVbnWQ5yIddkJ47oHoJVxSVUbXLnvE/dZSxh/opMpvs5edmMyqxdOXu/L/VwF +2wJHV6WMMMgxOEmxviMRBWpFyolxNAfkcapowJYW9zCgGw2pRjbbRfj4rQARAQAB +tCBBbHBlciBHdW5kb2dkdSA8YWxwZXJAZGVjZG4ub3JnPokCbQQTAQgAVxYhBOJ7 +mi0lGR6OyQv4rlfigjwWzNN2BQJqs9i0GxSAAAAAAAQADm1hbnUyLDIuNSsxLjEy +LDAsMwIbAwULCQgHAgIiAgYVCgkICwIEFgIDAQIeBwIXgAAKCRBX4oI8FszTdo7e +D/9JI0fYDXPwmz4jF1dK7I9WVtze1ZzWBLsydUV2omYrrzB8/DN1uWWsInMIBB37 +qRYwoUvY3ZSM8kSiNNtmFYITw0W4/a15+bWQYzYbTKdS1SDx/7eQghKdWzzDko0u +Y/txF2GbE3CgsVVsOiFmvWj8Bet9vP2uTtYF8YCBdq1PDbX8mobSd7GCUQA3i4Wv +t01SSqDFEoHSKbg+MLMVFrxuWIfeF1UFuSb77fU0fh6XC8d8pec5WJy7De6Xds7H +XaEbxaLAtWEulnKzZxOOciLf/Ii7obOfREfAhqaN679VqtMYPjU/rhHTUa7dtxQr +459joc3FnjVeI39lMiNdlRfqEzkPYViTQ/k9yLnogV/oHnsKsowUdWDlnrHOX71j +5L5F/CUbtvedV/fqe+cqcETdoK5+4BMepbBM7axGx3cmpWoAYiUCdlJtJ7J7cxe3 +q1d56qvH4o88rYVzwkVJaGMNtk0wraUURBKR9L1RgWY7zgjuBozEy51gBViRS1S1 +/Is7AG98TMjMODcQ7QBLYlP+dc5rncvz9aTFY/PwKOyHoV9kKRXfGurps/Eqy+jv +obf7pn729wmBoAzcTZd8AS2HqLWRAJLINe52s+lSMQtpChgmIELglK4K/9BsYXAX +nw3151Nw2xno1GeIN6x5gd2kp4t66y8EHmGidIdYZGvGsLkCDQRqs9i0ARAAwv89 +dMeXmgQYsYGm00pQTa4Cq5soaO8YkLWSFNznW5MPeLCOXa/gMq+73m1M1MMZQlrn +udSTbm7jpGQTJ6dXfT0l3DdjghBEJ/wb54DqVj7ft23ai5sOF6WO5EhRz5c2niBZ +5qE3qi0XAN/OFOOrvxQhpKr7NOupE2uOFggVmOdd4zssQuaB4kE7+UZSNhFoE2pD +OLKEV1qWj/pYBQ9NnL9W8luPcKbyZzNtc4kIYcXdNdNFKHZFezIDWhuxyLcLFyJm +MP7A1HXaP8h+wrU1DfrM/AmnXGksQWMQw1cejuzKbsWc+G2qnIASR6p+oquofvGj +wh5FSC6LKXgqqsS78lPK34DinK4ZEsv7tVzbKItOMaHS8I5mFmhR0b7eu2PYV7F7 +DHcd/eAgoY2cF36dEoG6c+lfnwkC9kyoTBY2GLJJFZPVu0dKoc8f0DHwjZEXpTId +veoKVWopo9LUUvHLT0EvIft4bcp/k87wPbs3u91PIkEgIK+8L7q1XJ9vq/7X4kJO +PAKTWJ8rMmvIhaKsC/TAeWEkvToDXGslv1VPYVE7IANRBACTu52246WB3IeUqllz +LdA6FKF9zkpYGFS5BLZRlLKU5fIm1ZDuB88d/MXcv+dECUK5KMRlGlF2wJEnydqW +gnFa9dHBMG25A13ZV/f2l6+oIOAZoUlhm5Hgw/kAEQEAAYkCUgQYAQgAPBYhBOJ7 +mi0lGR6OyQv4rlfigjwWzNN2BQJqs9i0GxSAAAAAAAQADm1hbnUyLDIuNSsxLjEy +LDAsMwIbDAAKCRBX4oI8FszTduMKD/9nTgAd6driyrvKlHv+xXkvq0/L0Jc4xi6j +cW0K1mfQTJrEn8swV/kvmCYWioSUGrg8eVioW5eyjQaG4E/paZSw72ybDwyzPvQh +Rhv14P+ZseebjXb9s3Woj1sCZ3ICZCkdHfsMbVzZUc+XJWPvPd3RSNnK8LbTbpwl +B316lAauEIatWOKNEHJiV89H2WbZJ8+emk2Q7GcLoMEeY7v5PdAZzvmsM0p0MjLo +Fc9HZVBDEe9pzKCw8E2VvvBreOlJcQe9d0XojEhoRN2ozV8RaX3R4GclCAolRxvz +xqvJNndDHEKdOBe6zfYWpCQdM2i4xYOqFtTIn6NKjhMonpSG9ZG5tlmeCG+daTpg +B0auw+G9ztRbs2QnC7oC0M1EEnFbKqEViywEaSfhEOp/GtdIBo28FwGF9scfLa5M +U13Wz16lSoCkdkMFpnEydmv/q5sZU7qZ9CMkNzXrtD/fNz8Khajd7G1ldh4SlAGw +bc/wi9USJO0a3QLXpJIrHdSezZ2TnOvdWU13MUblKQiHol7sAWFFLcz5gAu3+8JN +z8dNMGcLB3hl5gavPynzORCzJxfkSBtB9Ix5sODe74otk3hH2fzkRQj2nSbm+G+Q +qbh5wnX7qLwH80wKd+N6qqCuZ8UCq4d+2qarPvY61Ig/7r1bsA/y78k9JE/lREUB +ru+pjA/vJw== +=i6Bj +-----END PGP PUBLIC KEY BLOCK----- diff --git a/ansible/roles/sponsord/handlers/main.yml b/ansible/roles/sponsord/handlers/main.yml new file mode 100644 index 0000000..bfa42bf --- /dev/null +++ b/ansible/roles/sponsord/handlers/main.yml @@ -0,0 +1,10 @@ +--- +# Reloads systemd itself rather than relying on a separate "Reload systemd" +# handler running first: a same-named handler loaded later (baseline's +# devsec.hardening.os_hardening defines one) shadows the role's own and would run +# AFTER this restart, starting sponsord from systemd's cached copy of the unit. +- name: Restart sponsord + ansible.builtin.systemd: + name: sponsord + state: restarted + daemon_reload: true diff --git a/ansible/roles/sponsord/meta/main.yml b/ansible/roles/sponsord/meta/main.yml new file mode 100644 index 0000000..fe1875b --- /dev/null +++ b/ansible/roles/sponsord/meta/main.yml @@ -0,0 +1,16 @@ +--- +galaxy_info: + role_name: sponsord + author: deCDN Contributors + description: >- + deCDN onboarding sponsor (sponsord) — release-tarball or manual local-binary + install, hardened DynamicUser systemd unit with credentials, loopback API. + license: MIT + min_ansible_version: "2.15" + galaxy_tags: [decdn, sponsord, systemd, web3, debian] + platforms: + - name: Debian + versions: [bookworm, trixie] + - name: Ubuntu + versions: [noble, resolute] +dependencies: [] diff --git a/ansible/roles/sponsord/tasks/install.yml b/ansible/roles/sponsord/tasks/install.yml new file mode 100644 index 0000000..752cc1a --- /dev/null +++ b/ansible/roles/sponsord/tasks/install.yml @@ -0,0 +1,254 @@ +--- +# Install the sponsord binary. Two methods, selected by sponsord_install_method: +# "release": download the pinned sponsord-v GitHub Release tarball, +# verify it against the GPG-signed SHA256SUMS manifest, then extract. +# Idempotent and upgrade-aware via a version stamp. +# "manual" (default today): copy a locally-built binary from the control machine. +# The verification steps mirror roles/decdn_node/tasks/install.yml, which carries +# the full rationale for each (status-parsed gpg, exact manifest entry, private +# staging dir kept on failure). + +- name: Install from the pinned GitHub Release tarball + when: sponsord_install_method == "release" + block: + - name: Name the release archive + ansible.builtin.set_fact: + # The archive holds ONE flat binary at its root (upstream release.yml). + sponsord_release_archive: "sponsord-{{ sponsord_version }}-{{ sponsord_target }}.tar.gz" + sponsord_release_url: "{{ sponsord_release_base }}/sponsord-v{{ sponsord_version }}" + + - name: Read the installed-version stamp + ansible.builtin.slurp: + src: /usr/local/lib/sponsord/installed-version + register: sponsord_stamp + failed_when: false + changed_when: false + + - name: Decide whether (re)install is needed + ansible.builtin.set_fact: + sponsord_need_install: >- + {{ (sponsord_stamp.content is not defined) + or ((sponsord_stamp.content | b64decode | trim) != sponsord_version) }} + + - name: Report that the release download is not simulated in check mode + ansible.builtin.debug: + msg: >- + check mode: skipping the download + signature verification of sponsord + {{ sponsord_version }}. The config and unit below still diff normally. + when: + - sponsord_need_install + - ansible_check_mode + + - name: Install sponsord {{ sponsord_version }} + when: + - sponsord_need_install + - not ansible_check_mode + block: + - name: Create a private download staging directory + ansible.builtin.tempfile: + state: directory + prefix: sponsord-install- + register: sponsord_stage + changed_when: false + + - name: Restrict the staging directory to root + ansible.builtin.file: + path: "{{ sponsord_stage.path }}" + state: directory + owner: root + group: root + mode: "0700" + + - name: WARN that release signature verification is DISABLED + ansible.builtin.debug: + msg: |- + sponsord_verify_release_signature is false. SHA256SUMS is fetched + UNAUTHENTICATED from {{ sponsord_release_url }}, so the checksum check + proves only that the download was not corrupted — NOT that the binary + came from deCDN. This binary holds the treasury key. + when: not sponsord_verify_release_signature + + - name: Ensure gnupg is available for signature verification + ansible.builtin.package: + name: gnupg + state: present + when: sponsord_verify_release_signature + + - name: Download the release tarball + ansible.builtin.get_url: + url: "{{ sponsord_release_url }}/{{ sponsord_release_archive }}" + dest: "{{ sponsord_stage.path }}/{{ sponsord_release_archive }}" + owner: root + group: root + mode: "0600" + + - name: Download the SHA256SUMS manifest and signature + ansible.builtin.get_url: + url: "{{ sponsord_release_url }}/{{ item }}" + dest: "{{ sponsord_stage.path }}/{{ item }}" + owner: root + group: root + mode: "0600" + loop: "{{ ['SHA256SUMS', 'SHA256SUMS.asc'] if sponsord_verify_release_signature else ['SHA256SUMS'] }}" + + - name: Create the keyring staging directory + ansible.builtin.file: + path: "{{ sponsord_stage.path }}/keyring" + state: directory + owner: root + group: root + mode: "0700" + when: sponsord_verify_release_signature + + - name: Stage the release signing keyring + ansible.builtin.copy: + src: "{{ sponsord_release_keyring }}" + dest: "{{ sponsord_stage.path }}/keyring/KEYS.asc" + owner: root + group: root + mode: "0600" + when: sponsord_verify_release_signature + + # Isolated GNUPGHOME; VALIDSIG required, an expired/revoked key refused, + # --no-autostart so no gpg-agent outlives the task (see decdn_node). + - name: Verify the SHA256SUMS signature against the release keyring + ansible.builtin.shell: + chdir: "{{ sponsord_stage.path }}" + executable: /bin/bash + cmd: | + set -euo pipefail + export GNUPGHOME="{{ sponsord_stage.path }}/gnupg" + mkdir -p "$GNUPGHOME" + chmod 700 "$GNUPGHOME" + gpg --batch --quiet --no-autostart --import keyring/KEYS.asc + gpg --batch --no-autostart --status-fd 3 --verify SHA256SUMS.asc SHA256SUMS \ + 3>gpg-status.txt + if grep -qE '^\[GNUPG:\] (EXPKEYSIG|REVKEYSIG|EXPSIG)' gpg-status.txt; then + echo "SHA256SUMS carries a good signature from an EXPIRED or REVOKED" >&2 + echo "release key. Refusing to install. Re-sync" >&2 + echo "roles/sponsord/files/sponsord-release-KEYS.asc from upstream KEYS," >&2 + echo "or verify this release out of band." >&2 + exit 1 + fi + grep -q '^\[GNUPG:\] VALIDSIG ' gpg-status.txt + register: sponsord_sig_check + changed_when: false + failed_when: sponsord_sig_check.rc != 0 + when: sponsord_verify_release_signature + + - name: Verify the tarball against SHA256SUMS + ansible.builtin.shell: + chdir: "{{ sponsord_stage.path }}" + executable: /bin/bash + cmd: | + set -euo pipefail + f={{ sponsord_release_archive | quote }} + entry=$(awk -v f="$f" 'substr($0, index($0, " ") + 2) == f' SHA256SUMS) + n=$(printf '%s\n' "$entry" | grep -c . || true) + if [ "$n" -ne 1 ]; then + echo "SHA256SUMS has $n entries for $f (expected exactly 1)." >&2 + exit 1 + fi + printf '%s\n' "$entry" | sha256sum --check --strict - + register: sponsord_sum_check + changed_when: false + failed_when: sponsord_sum_check.rc != 0 + + - name: Extract the binary into /usr/local/bin + ansible.builtin.unarchive: + src: "{{ sponsord_stage.path }}/{{ sponsord_release_archive }}" + dest: "{{ sponsord_bin | dirname }}" + remote_src: true + owner: root + group: root + mode: "0755" + notify: Restart sponsord + + always: + - name: Report the retained staging directory after a failure + ansible.builtin.debug: + msg: >- + Verification failed; leaving {{ sponsord_stage.path }} in place for + inspection (gpg-status.txt, SHA256SUMS, SHA256SUMS.asc, the archive). + Remove it by hand once done. + when: + - sponsord_stage.path is defined + - sponsord_sig_check.failed | default(false) or sponsord_sum_check.failed | default(false) + + - name: Remove the download staging directory + ansible.builtin.file: + path: "{{ sponsord_stage.path }}" + state: absent + when: + - sponsord_stage.path is defined + - not (sponsord_sig_check.failed | default(false) or sponsord_sum_check.failed | default(false)) + +- name: Install a locally-built binary from the control machine + when: sponsord_install_method == "manual" + block: + - name: Copy the sponsord binary from the control machine + ansible.builtin.copy: + src: "{{ _sponsord_bin_src }}" + dest: "{{ sponsord_bin }}" + owner: root + group: root + mode: "0755" + notify: Restart sponsord + + # A leftover stamp would make a later release deploy of the SAME version skip + # the download, keeping this manual binary. + - name: Clear the release version stamp in manual mode + ansible.builtin.file: + path: /usr/local/lib/sponsord/installed-version + state: absent + +# Runs every converge, so a binary that drifted from the pin is caught. With +# sponsord_version empty (the usual manual case) it degrades to a liveness check. +# An exact match on clap's `sponsord ` line, so a 0.1.0 pin does not +# accept 10.1.0 or 0.1.0-rc.1. +- name: Run the installed sponsord --version (version backstop) + ansible.builtin.command: "{{ sponsord_bin }} --version" + register: sponsord_installed_version + changed_when: false + failed_when: false + # The binary only exists once the copy/extract above really ran. + when: not ansible_check_mode + +- name: Require the installed sponsord to run and match the pinned version + ansible.builtin.assert: + that: + - sponsord_installed_version.rc == 0 + - >- + sponsord_version | length == 0 + or (sponsord_installed_version.stdout | trim) == 'sponsord ' ~ sponsord_version + fail_msg: >- + {{ sponsord_bin }} --version exited {{ sponsord_installed_version.rc }} with + "{{ sponsord_installed_version.stdout | trim }}"{{ + (', expected "sponsord ' ~ sponsord_version ~ '"') if sponsord_version | length > 0 else '' }}. + The binary on the host is broken or is not the pinned release. In release mode, + remove /usr/local/lib/sponsord/installed-version to force a fresh download. + when: not ansible_check_mode + +# Written only once the version check passed, so a tarball that installed a wrong +# or broken binary is fetched again next run instead of being trusted via the stamp. +- name: Record the installed release version + when: + - sponsord_install_method == "release" + - sponsord_need_install | default(false) + - not ansible_check_mode + block: + - name: Ensure the stamp directory exists + ansible.builtin.file: + path: /usr/local/lib/sponsord + state: directory + owner: root + group: root + mode: "0755" + + - name: Record the installed version + ansible.builtin.copy: + dest: /usr/local/lib/sponsord/installed-version + content: "{{ sponsord_version }}\n" + owner: root + group: root + mode: "0644" diff --git a/ansible/roles/sponsord/tasks/main.yml b/ansible/roles/sponsord/tasks/main.yml new file mode 100644 index 0000000..13f8a83 --- /dev/null +++ b/ansible/roles/sponsord/tasks/main.yml @@ -0,0 +1,767 @@ +--- +# Provision sponsord: validate, place secrets, install, configure, start, gate on +# /healthz. Every pure-input assert runs before the first host mutation, and every +# gate (host files included) before the binary is installed. The pool itself +# (`decdn pool open` from the treasury wallet, then funding that wallet) is an +# operator action — see roles/sponsord/README.md. + +# --- Install method + target ---------------------------------------------------- +- name: Require a valid install method + ansible.builtin.assert: + that: + - sponsord_install_method in ["release", "manual"] + fail_msg: >- + sponsord_install_method must be "release" (pinned GitHub Release tarball) or + "manual" (copy a locally-built binary from the control machine). + +# Upstream decdn/sponsord has cut no sponsord-v* tag yet; this names the cause +# instead of surfacing a bare 404 from get_url. +- name: Require a pinned release version + ansible.builtin.assert: + that: + - sponsord_version | length > 0 + fail_msg: >- + sponsord_install_method "release" requires sponsord_version (e.g. "0.1.0") and + a matching sponsord-v GitHub Release. Until upstream cuts one, build + the binary from a decdn/sponsord checkout and use sponsord_install_method + "manual". + when: sponsord_install_method == "release" + +- name: Validate sponsord_verify_release_signature is a real boolean + ansible.builtin.assert: + that: + - sponsord_verify_release_signature is boolean + fail_msg: >- + sponsord_verify_release_signature must be a boolean (true/false). Leave it true + unless you install from a mirror that strips SHA256SUMS.asc. + when: sponsord_install_method == "release" + +# Explicit sponsord_manual_bin_src never consults the triple, so it skips this. +- name: Resolve the sponsord target triple + when: >- + sponsord_install_method == "release" + or (sponsord_release_target_dir | length > 0 and sponsord_manual_bin_src | length == 0) + block: + - name: Require a supported target triple + ansible.builtin.assert: + that: + - sponsord_target in _sponsord_arch_targets.values() + fail_msg: >- + sponsord_target resolved to "{{ sponsord_target }}", which is not one of the + Linux targets upstream releases ({{ _sponsord_arch_targets.values() | join(', ') }}). + It is derived from the host's gathered architecture + ("{{ ansible_facts.get('architecture', 'not gathered') }}"): gather facts + or set sponsord_target explicitly. + + - name: Require the target triple to match the host architecture + ansible.builtin.assert: + that: + - _sponsord_arch_targets[ansible_facts['architecture']] == sponsord_target + fail_msg: >- + sponsord_target is {{ sponsord_target }}, but this host is + {{ ansible_facts['architecture'] }} and needs + {{ _sponsord_arch_targets[ansible_facts['architecture']] }}. Drop the explicit + sponsord_target (it is derived from the host) or fix it for this host. + when: ansible_facts.get('architecture', '') in _sponsord_arch_targets + +# Resolve + validate the manual binary on the CONTROL MACHINE before copying it. +# An explicit sponsord_manual_bin_src is copied verbatim (a test stub is legitimately +# not an ELF); sponsord_release_target_dir is searched (plus the cross output dir +# target//release) and the result must be an ELF for the host, which +# catches a Mach-O from a host-native `cargo build` on macOS at deploy time. +- name: Resolve and validate the manual binary source + when: sponsord_install_method == "manual" + block: + - name: Require a manual binary source + ansible.builtin.assert: + that: + - sponsord_manual_bin_src | length > 0 or sponsord_release_target_dir | length > 0 + fail_msg: >- + sponsord_install_method "manual" needs sponsord_manual_bin_src (a + control-machine path to the sponsord binary) or sponsord_release_target_dir + (the control-machine target/release dir of a decdn/sponsord checkout). + + # Internal facts, never a set_fact on sponsord_manual_bin_src itself: an + # include_role `vars:` value would outrank it, and a derived path would leak + # into every later include on the same host. A trailing slash is stripped so + # `dirname` lands on target/, not target/release. + - name: Decide whether target-dir resolution applies + ansible.builtin.set_fact: + _sponsord_use_dir: "{{ sponsord_manual_bin_src | length == 0 }}" + _sponsord_target_dir: "{{ sponsord_release_target_dir | regex_replace('/+$', '') }}" + _sponsord_bin_src: "{{ sponsord_manual_bin_src }}" + + - name: Stat the binary in each candidate directory + ansible.builtin.stat: + path: "{{ item }}/sponsord" + register: _sponsord_bin_stat + loop: + - "{{ _sponsord_target_dir }}" + - "{{ (_sponsord_target_dir | dirname) ~ '/' ~ sponsord_target ~ '/release' }}" + loop_control: + label: "{{ item }}" + delegate_to: localhost + become: false + when: _sponsord_use_dir | bool + + - name: Select the first candidate directory that holds the binary + ansible.builtin.set_fact: + _sponsord_resolved_dir: >- + {{ (_sponsord_bin_stat.results | selectattr('stat.exists') | map(attribute='item') + | first) | default('', true) }} + when: _sponsord_use_dir | bool + + - name: Fail if no candidate directory held the binary + ansible.builtin.assert: + that: + - _sponsord_resolved_dir | length > 0 + fail_msg: >- + No sponsord binary under sponsord_release_target_dir + ({{ sponsord_release_target_dir }}) or its cross output dir + ({{ (_sponsord_target_dir | dirname) ~ '/' ~ sponsord_target ~ '/release' }}). + Build it in a decdn/sponsord checkout — `cargo build --release -p sponsord`, + or cross-compiling for the host, + `cross build --release -p sponsord --target {{ sponsord_target }}`. + when: _sponsord_use_dir | bool + + - name: Derive the effective binary source path + ansible.builtin.set_fact: + _sponsord_bin_src: "{{ _sponsord_resolved_dir ~ '/sponsord' }}" + when: _sponsord_use_dir | bool + + - name: Identify the on-disk format of the resolved binary + ansible.builtin.command: + argv: [file, -b, --, "{{ _sponsord_bin_src }}"] + register: _sponsord_bin_file + changed_when: false + check_mode: false + delegate_to: localhost + become: false + when: _sponsord_use_dir | bool + + - name: Require the binary to be an ELF for the host's architecture + ansible.builtin.assert: + that: + - "'ELF' in _sponsord_bin_file.stdout" + - _sponsord_arch_file_tokens[sponsord_target] in _sponsord_bin_file.stdout + fail_msg: >- + {{ _sponsord_bin_src }} is "{{ _sponsord_bin_file.stdout }}" — not an ELF + {{ _sponsord_arch_file_tokens[sponsord_target] }} binary for {{ sponsord_target }}. + Cross-compile for the host: `cross build --release -p sponsord --target + {{ sponsord_target }}`. + when: _sponsord_use_dir | bool + +# --- Chain ------------------------------------------------------------------------ +- name: Require a known sponsord_network + ansible.builtin.assert: + that: + - sponsord_network == "" or sponsord_network in _sponsord_network_profiles + fail_msg: >- + sponsord_network "{{ sponsord_network }}" is not a known network. Known: + {{ _sponsord_network_profiles.keys() | join(', ') }}. Leave it "" to set + sponsord_chain_id and sponsord_payment_pool_address yourself. + +# A chain_id from inventory that points at another chain would silently pair that +# chain with this profile's PaymentPool address. +- name: Require chain_id to match the network profile + ansible.builtin.assert: + that: + - sponsord_chain_id | int == _sponsord_profile.chain_id | int + fail_msg: >- + sponsord_chain_id is {{ sponsord_chain_id }}, but sponsord_network + "{{ sponsord_network }}" is chain {{ _sponsord_profile.chain_id }}. Drop + sponsord_chain_id from inventory (the profile supplies it) or clear + sponsord_network. + when: sponsord_network | length > 0 + +- name: Report a PaymentPool address overridden in inventory + ansible.builtin.debug: + msg: >- + sponsord_network "{{ sponsord_network }}" is in use, but inventory overrides + sponsord_payment_pool_address. The inventory value wins over the profile; make + sure that is deliberate (a stale hand-copied address is the usual cause). + when: + - sponsord_network | length > 0 + - (sponsord_payment_pool_address | lower) != (_sponsord_profile.payment_pool_address | lower) + +- name: Require chain, PaymentPool and pool id + ansible.builtin.assert: + that: + - sponsord_chain_id | string is match('^[0-9]+$') + - sponsord_chain_id | int > 0 + - sponsord_payment_pool_address is match('^0x[0-9a-fA-F]{40}$') + - sponsord_payment_pool_address != '0x' ~ '0' * 40 + - sponsord_pool_id is match('^0x[0-9a-fA-F]{64}$') + - sponsord_pool_id != '0x' ~ '0' * 64 + fail_msg: >- + A positive sponsord_chain_id, sponsord_payment_pool_address (0x + 40 hex, not + zero) and sponsord_pool_id (0x + 64 hex, not zero) are required. Set + sponsord_network (e.g. "arbitrum-sepolia") to take chain_id and the PaymentPool + address from the role's mirror of upstream's deployment manifest — never + hand-copy them — and sponsord_pool_id to the id `decdn pool open` printed when + the treasury wallet opened its pool. + +# --- Listener + limits -------------------------------------------------------------- +# AGENTS.md hard rule 2. The API is bearer-authenticated but /metrics is not, and +# the onramp that calls it runs on the same host. IPv4 loopback only: the value is +# parsed as a SocketAddr, so "localhost" would not start, and Alloy scrapes it as +# host:port. +- name: Require the listener to stay on loopback + ansible.builtin.assert: + that: + # 127.x.y.z with every octet 0-255. + - sponsord_bind_address is match('^127(\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])){3}$') + - sponsord_port | string is match('^[0-9]+$') + - sponsord_port | int >= 1 + - sponsord_port | int <= 65535 + fail_msg: >- + sponsord_bind_address must be an IPv4 loopback address (got + "{{ sponsord_bind_address }}") and sponsord_port a TCP port (got + "{{ sponsord_port }}"). /metrics has no auth and the API holds the treasury's + signing power; exposing it is an architecture change, not a config tweak. + Front it with a reverse proxy that terminates auth + TLS (AGENTS.md hard rule 2). + +- name: Validate the issuance and keeper limits + ansible.builtin.assert: + that: + - item.value | string is match('^[0-9]+$') + - item.key == 'sponsord_pool_low_water_micro_usdc' or item.value | int > 0 + fail_msg: >- + {{ item.key }} must be "" (upstream default) or a non-negative integer + ({{ 'zero allowed' if item.key == 'sponsord_pool_low_water_micro_usdc' else 'greater than zero' }}), + got "{{ item.value }}". + loop: "{{ _sponsord_limits | dict2items | rejectattr('value', 'equalto', '') | list }}" + loop_control: + label: "{{ item.key }}" + vars: + _sponsord_limits: + sponsord_max_spending_cap_micro_usdc: "{{ sponsord_max_spending_cap_micro_usdc }}" + sponsord_max_ttl_secs: "{{ sponsord_max_ttl_secs }}" + sponsord_pool_low_water_micro_usdc: "{{ sponsord_pool_low_water_micro_usdc }}" + sponsord_pool_refill_micro_usdc: "{{ sponsord_pool_refill_micro_usdc }}" + sponsord_pool_watch_interval_secs: "{{ sponsord_pool_watch_interval_secs }}" + +- name: Validate the role's boolean knobs + ansible.builtin.assert: + that: + - sponsord_generate_api_token is boolean + - sponsord_secret_env_overwrite_host_file is boolean + fail_msg: >- + sponsord_generate_api_token and sponsord_secret_env_overwrite_host_file must be + booleans (true/false), not quoted strings. + +# --- RPC URL: inventory or host-provisioned secret.env ------------------------------ +# The same two authoring paths, provenance record and guards as decdn_node's +# decdn_rpc_url (#39) — see the long comments in roles/decdn_node/tasks/main.yml. +# sponsord_rpc_url set -> the role authors secret.env (OVERWRITES the host's) +# sponsord_rpc_url empty -> the operator wrote secret.env on the host; the role +# never reads it back, it only locks it down and greps +# The record " " tells a file the role wrote from one the operator +# owns, and a changed hash means an out-of-band edit. +- name: Validate the inventory RPC URL is one-line EnvironmentFile data + ansible.builtin.assert: + that: + - sponsord_rpc_url | string is match('^[^\x00-\x1f\x7f]*\Z') + fail_msg: >- + sponsord_rpc_url must contain no ASCII control characters; the role renders it + as one EnvironmentFile assignment. + no_log: true + when: sponsord_rpc_url | length > 0 + +- name: Check for a host-provisioned secret environment file + ansible.builtin.stat: + path: "{{ sponsord_secret_env_file }}" + follow: false + get_checksum: true + checksum_algorithm: sha256 + register: sponsord_secret_env_pre_stat + +- name: Read the secret.env checksum this role last recorded + ansible.builtin.slurp: + src: "{{ sponsord_secret_env_checksum_file }}" + register: sponsord_secret_env_recorded + # A missing record is normal (first converge); any other read error is not, and + # treating it as "no record" would silently disable the provenance guards. + failed_when: >- + sponsord_secret_env_recorded.failed + and 'not found' not in (sponsord_secret_env_recorded.msg | default('')) + changed_when: false + +- name: Resolve secret.env provenance + ansible.builtin.set_fact: + sponsord_secret_env_record_source: "{{ (_rec.split() | length == 2) | ternary(_rec.split()[0], '') }}" + sponsord_secret_env_recorded_sum: "{{ (_rec.split() | length == 2) | ternary(_rec.split()[1], '') }}" + vars: + _rec: "{{ sponsord_secret_env_recorded.content | default('') | b64decode | trim }}" + +- name: Require an RPC endpoint — inventory value or a host-provisioned secret.env + ansible.builtin.assert: + that: + - sponsord_rpc_url | length > 0 or sponsord_secret_env_pre_stat.stat.exists + fail_msg: |- + No RPC endpoint: sponsord_rpc_url is empty and {{ sponsord_secret_env_file }} + does not exist on this host. The URL may embed a provider API key, so the + preferred posture is to provision it on the host. Run this on the TARGET: + umask 077 + sudo mkdir -p {{ sponsord_etc }} + echo 'SPONSORD_RPC_URL=https://…' | sudo tee {{ sponsord_secret_env_file }} >/dev/null + sudo chmod 600 {{ sponsord_secret_env_file }} + Or set sponsord_rpc_url in the git-ignored host_vars//secret.yml and the + role will author the file for you. + +# Not a symlink or directory: the lock-down below chowns/chmods this path as root. +- name: Require secret.env to be a regular file + ansible.builtin.assert: + that: + - sponsord_secret_env_pre_stat.stat.isreg | default(false) + fail_msg: >- + {{ sponsord_secret_env_file }} exists but is not a regular file + (symlink/directory/device). Replace it with a real file. + when: sponsord_secret_env_pre_stat.stat.exists + +# An empty sponsord_rpc_url also happens when secret.yml is simply missing (a fresh +# clone, an undecrypted vault). Adopting the role's own stale file as +# operator-provisioned would deploy green on the old endpoint. +- name: Refuse to adopt a role-authored secret.env as host-provisioned + ansible.builtin.assert: + that: + - sponsord_secret_env_record_source != 'inventory' + or sponsord_secret_env_recorded_sum != (sponsord_secret_env_pre_stat.stat.checksum | default('')) + fail_msg: >- + sponsord_rpc_url is empty, but {{ sponsord_secret_env_file }} is byte-for-byte + the file THIS ROLE last wrote from inventory — the value has gone missing from + the control machine (an absent secret.yml, an undecrypted vault) rather than + being handed to the host. Restore sponsord_rpc_url, or hand the file over + explicitly by discarding the role's provenance record: + sudo rm {{ sponsord_secret_env_checksum_file }} + when: sponsord_rpc_url | length == 0 + +# Unlike decdn_node's guard, NO record is not an exemption: decdn_node allows it for +# hosts that predate its record, but every host this role ever wrote secret.env on +# has one (it is written straight after the file). So an existing file without a +# matching `inventory` record is the operator's, including on the first converge. +- name: Require confirmation to overwrite a secret.env this role did not write + ansible.builtin.assert: + that: + - sponsord_secret_env_overwrite_host_file | bool + fail_msg: >- + sponsord_rpc_url is set, but {{ sponsord_secret_env_file }} on this host is not + a file this role authored (record + "{{ sponsord_secret_env_record_source | default('none', true) }}", or its + checksum no longer matches). Rendering it from inventory would discard + whatever the host's copy holds. Clear sponsord_rpc_url to keep the host's + copy, or set sponsord_secret_env_overwrite_host_file: true. + when: + - sponsord_rpc_url | length > 0 + - sponsord_secret_env_pre_stat.stat.exists + - >- + sponsord_secret_env_record_source != 'inventory' + or sponsord_secret_env_recorded_sum != (sponsord_secret_env_pre_stat.stat.checksum | default('')) + +# --- Treasury wallet (operator-provisioned) ------------------------------------------ +# follow: false — a symlink is refused, because the lock-down below would otherwise +# chmod its target as root. (The daemon itself reads a 0600 copy the unit makes at +# start, never this path.) +- name: Check the treasury keystore and password files + ansible.builtin.stat: + path: "{{ item }}" + follow: false + get_checksum: false + register: sponsord_treasury_stat + loop: + - "{{ sponsord_treasury_keystore_file }}" + - "{{ sponsord_treasury_password_file }}" + +- name: Require the treasury keystore and password as regular files + ansible.builtin.assert: + that: + - item.stat.isreg | default(false) + fail_msg: |- + {{ item.item }} is {{ 'missing' if not item.stat.exists else 'not a regular file (symlink/directory?)' }}. + sponsord signs with the TREASURY wallet — the one that owns sponsord_pool_id + on-chain — and the role never creates it. Create it once with the decdn CLI + (`decdn key-gen`), open and fund the pool from it (`decdn pool open`), then + copy the keystore JSON and its password onto this host, as root: + umask 077 + sudo mkdir -p {{ sponsord_etc }} + sudo install -m 0600 treasury-keystore.json {{ sponsord_treasury_keystore_file }} + sudo install -m 0600 treasury-password {{ sponsord_treasury_password_file }} + Keep only a few top-ups' worth of USDC plus gas in that wallet: it is a hot key. + loop: "{{ sponsord_treasury_stat.results }}" + loop_control: + label: "{{ item.item }}" + +- name: Check for a provisioned API token + ansible.builtin.stat: + path: "{{ sponsord_api_token_file }}" + follow: false + register: sponsord_api_token_pre_stat + +- name: Require an API token file when generation is off + ansible.builtin.assert: + that: + - sponsord_api_token_pre_stat.stat.isreg | default(false) + fail_msg: >- + sponsord_generate_api_token is false and {{ sponsord_api_token_file }} is not + a regular file. Provision a token of at least 32 bytes on the host + (`umask 077; openssl rand -hex 32 | sudo tee {{ sponsord_api_token_file }}`), + or set sponsord_generate_api_token: true. + when: not sponsord_generate_api_token + +- name: Require the API token path to be a regular file when present + ansible.builtin.assert: + that: + - sponsord_api_token_pre_stat.stat.isreg | default(false) + fail_msg: "{{ sponsord_api_token_file }} exists but is not a regular file (symlink/directory?)." + when: sponsord_api_token_pre_stat.stat.exists + +# --- Secrets ------------------------------------------------------------------------ +- name: Create the sponsord config directory + ansible.builtin.file: + path: "{{ sponsord_etc }}" + state: directory + owner: root + group: root + mode: "0755" + +# Generated on the host (never on the control machine) and never replaced: the +# onramp holds a copy. umask first so the file is never briefly readable. +- name: Generate the API token on the host + ansible.builtin.shell: + executable: /bin/bash + cmd: | + set -euo pipefail + umask 077 + od -An -N32 -tx1 /dev/urandom | tr -d ' \n' > {{ sponsord_api_token_file | quote }} + creates: "{{ sponsord_api_token_file }}" + when: sponsord_generate_api_token + +- name: Lock down the credential files (ownership + mode only) + ansible.builtin.file: + path: "{{ item }}" + owner: root + group: root + mode: "0600" + follow: false + loop: + - "{{ sponsord_api_token_file }}" + - "{{ sponsord_treasury_keystore_file }}" + - "{{ sponsord_treasury_password_file }}" + # Under --check a generated token is only simulated, so it may not exist yet. + when: not (ansible_check_mode and item == sponsord_api_token_file and not sponsord_api_token_pre_stat.stat.exists) + +# Mirrors upstream's Secret::resolve: strip ONE trailing "\n" or "\r\n", then the +# rest is the bearer token verbatim. It must be at least 32 bytes (upstream refuses +# less at start) and one line (a CR/LF inside cannot travel in an Authorization +# header, yet upstream would start on it). Prints the length or "multiline" only, +# so the token never leaves the host. +- name: Measure the API token + ansible.builtin.shell: + executable: /bin/bash + cmd: | + set -euo pipefail + export LC_ALL=C + tok=$(cat {{ sponsord_api_token_file | quote }}; printf x) + tok=${tok%x} + case $tok in + *$'\r\n') tok=${tok%$'\r\n'} ;; + *$'\n') tok=${tok%$'\n'} ;; + esac + case $tok in + *[$'\r\n']*) echo multiline ;; + *) printf '%s' "$tok" | wc -c ;; + esac + register: sponsord_api_token_len + changed_when: false + check_mode: false + when: not (ansible_check_mode and not sponsord_api_token_pre_stat.stat.exists) + +- name: Require a single-line API token of at least 32 bytes + ansible.builtin.assert: + that: + - sponsord_api_token_len.stdout != 'multiline' + - sponsord_api_token_len.stdout | int >= 32 + fail_msg: >- + {{ sponsord_api_token_file }} holds + {{ 'a token with a line break inside' if sponsord_api_token_len.stdout == 'multiline' + else (sponsord_api_token_len.stdout | trim) ~ ' bytes' }}; sponsord needs one + line of at least 32 bytes (one trailing newline is fine). Replace it + (`umask 077; openssl rand -hex 32 | sudo tee {{ sponsord_api_token_file }}`) and + give the onramp the same value. + when: sponsord_api_token_len.stdout is defined + +- name: Lock down a host-provisioned secret.env (ownership + mode only) + ansible.builtin.file: + path: "{{ sponsord_secret_env_file }}" + owner: root + group: root + mode: "0600" + when: sponsord_rpc_url | length == 0 + +# grep -q keeps the secret out of stdout and argv; never slurp it. +- name: Confirm a host-provisioned secret.env carries a non-empty SPONSORD_RPC_URL + ansible.builtin.command: + cmd: grep -qE '^[[:space:]]*SPONSORD_RPC_URL=[^[:space:]]' {{ sponsord_secret_env_file | quote }} + register: sponsord_secret_env_grep + changed_when: false + check_mode: false + failed_when: false + when: sponsord_rpc_url | length == 0 + +- name: Report a host-provisioned secret.env with no usable SPONSORD_RPC_URL + ansible.builtin.fail: + msg: >- + {{ sponsord_secret_env_file }} carries no `SPONSORD_RPC_URL=` line with a + non-whitespace value (grep exited {{ sponsord_secret_env_grep.rc | default('-') }} + {{ sponsord_secret_env_grep.stderr | default('', true) }}), + so the daemon would start with no chain endpoint. Add one — single-quote a value + containing a space, quote, backslash or `$`, because systemd skips a line it + cannot parse: + echo 'SPONSORD_RPC_URL=https://your-endpoint.example/rpc' | sudo tee -a + {{ sponsord_secret_env_file }} >/dev/null + when: + - sponsord_rpc_url | length == 0 + - sponsord_secret_env_grep.rc | default(1) != 0 + +# secret.env is an EnvironmentFile, and those override the unit's Environment= and +# every other setting: a pasted upstream sponsord.env.example would quietly replace +# the validated bind address, pool and contract (a public listener, or a green +# deploy against a stale PaymentPool). Only SPONSORD_RPC_URL belongs here. The +# check prints key NAMES only, so no value leaves the host. +- name: List any keys besides SPONSORD_RPC_URL in a host-provisioned secret.env + ansible.builtin.shell: + executable: /bin/bash + cmd: | + set -o pipefail + { grep -oE '^[[:space:]]*(export[[:space:]]+)?[A-Za-z_][A-Za-z0-9_]*[[:space:]]*=' \ + {{ sponsord_secret_env_file | quote }} || true; } \ + | sed -E 's/^[[:space:]]*(export[[:space:]]+)?//; s/[[:space:]]*=$//' \ + | { grep -vx SPONSORD_RPC_URL || true; } + register: sponsord_secret_env_extra_keys + changed_when: false + check_mode: false + when: sponsord_rpc_url | length == 0 + +- name: Refuse keys besides SPONSORD_RPC_URL in a host-provisioned secret.env + ansible.builtin.assert: + that: + - sponsord_secret_env_extra_keys.stdout | length == 0 + fail_msg: >- + {{ sponsord_secret_env_file }} sets {{ sponsord_secret_env_extra_keys.stdout_lines | join(', ') }}. + It may carry SPONSORD_RPC_URL only: an EnvironmentFile overrides every other + setting of the unit, so these would silently replace what this role validated + (bind address, chain, PaymentPool, pool id, credential paths). Remove them from + the file and set the matching sponsord_* variables in inventory instead. + when: sponsord_rpc_url | length == 0 + +# After every gate, so a deploy that stops at one has notified no restart for a unit +# that may not exist yet. A run that fails LATER loses its pending restart (Ansible +# runs no handlers for a failed host); the inputs record below catches that. +- name: Install the sponsord binary + ansible.builtin.import_tasks: install.yml + +- name: Write secret.env (inventory-supplied) + ansible.builtin.copy: + dest: "{{ sponsord_secret_env_file }}" + owner: root + group: root + mode: "0600" + # to_json emits the double-quoted, C-escaped form systemd's EnvironmentFile + # parser expects (see decdn_node's env-file task for the full rationale). + content: | + SPONSORD_RPC_URL={{ sponsord_rpc_url | string | to_json(ensure_ascii=false) }} + no_log: true + notify: Restart sponsord + when: sponsord_rpc_url | length > 0 + +# Provenance only ("who wrote secret.env"), so it is written as soon as the file is +# final, not after the health gate: a run that wrote the file and then failed must +# still be recorded as having written it, or the next run would refuse to overwrite +# the role's own file, or adopt it as the operator's. Restarts are tracked +# separately by the restart-inputs record below. +- name: Re-stat secret.env now that it is final + ansible.builtin.stat: + path: "{{ sponsord_secret_env_file }}" + get_checksum: true + checksum_algorithm: sha256 + register: sponsord_secret_env_post_stat + +- name: Record who wrote secret.env, and its checksum + ansible.builtin.copy: + dest: "{{ sponsord_secret_env_checksum_file }}" + owner: root + group: root + mode: "0600" + content: > + {{ (sponsord_rpc_url | length > 0) | ternary('inventory', 'host') }} + {{ sponsord_secret_env_post_stat.stat.checksum }} + # In check mode the inventory write is only simulated, so there may be nothing to hash. + when: (sponsord_secret_env_post_stat.stat.checksum | default('')) | length > 0 + +# --- Config + unit ------------------------------------------------------------------ +- name: Template the non-secret sponsord.env + ansible.builtin.template: + src: sponsord.env.j2 + dest: "{{ sponsord_env_file }}" + owner: root + group: root + mode: "0644" + notify: Restart sponsord + +- name: Install the sponsord systemd unit + ansible.builtin.template: + src: sponsord.service.j2 + dest: /etc/systemd/system/sponsord.service + owner: root + group: root + mode: "0644" + notify: Restart sponsord + +# --- Restart inputs ----------------------------------------------------------------- +# Everything the daemon starts from: the credentials, both env files, the unit and +# the binary. Their hashes are recorded only once the daemon is healthy (end of this +# file), so whatever left the running daemon on stale inputs is caught on the next +# converge — an operator rotating a credential or editing a host-provisioned +# secret.env out of band, or an earlier run that changed a file and then failed +# before its restart handler ran. An ABSENT record counts only +# when the daemon is already running (below). +- name: Hash the restart inputs + ansible.builtin.stat: + path: "{{ item }}" + get_checksum: true + checksum_algorithm: sha256 + register: sponsord_inputs_stat + loop: + - "{{ sponsord_api_token_file }}" + - "{{ sponsord_treasury_keystore_file }}" + - "{{ sponsord_treasury_password_file }}" + - "{{ sponsord_secret_env_file }}" + - "{{ sponsord_env_file }}" + - /etc/systemd/system/sponsord.service + - "{{ sponsord_bin }}" + +- name: Read the restart-input checksums this role last recorded + ansible.builtin.slurp: + src: "{{ sponsord_inputs_checksum_file }}" + register: sponsord_inputs_recorded + failed_when: sponsord_inputs_recorded.failed and 'not found' not in (sponsord_inputs_recorded.msg | default('')) + changed_when: false + +# With no record, a daemon that is ALREADY running was never confirmed healthy on +# these inputs (an earlier run timed out at the gate, then the daemon came up on +# whatever it had then), so it cannot be trusted to be on them now. On a truly +# fresh host the unit is not active and nothing is owed. Read-only. +- name: Read whether sponsord is already running + ansible.builtin.systemd_service: + name: sponsord + register: sponsord_is_active + +- name: Summarise the restart-input checksums + ansible.builtin.set_fact: + sponsord_inputs_sums: >- + {{ sponsord_inputs_stat.results | map(attribute='stat') | map('default', {}, true) + | map(attribute='checksum', default='') | join(' ') }} + sponsord_inputs_recorded_sums: "{{ sponsord_inputs_recorded.content | default('') | b64decode | trim }}" + +- name: Note restart inputs that changed since the daemon last started healthy + ansible.builtin.debug: + msg: >- + {{ 'A credential, env file, the unit or the binary differs from what sponsord was + last confirmed healthy on (rotated or edited on the host, or changed by a run + that failed before restarting)' if sponsord_inputs_recorded_sums | length > 0 + else 'sponsord is running but was never confirmed healthy on its current inputs' }} + — restarting sponsord. + changed_when: true + notify: Restart sponsord + when: >- + (sponsord_inputs_recorded_sums | length > 0 and sponsord_inputs_sums != sponsord_inputs_recorded_sums) + or (sponsord_inputs_recorded_sums | length == 0 + and sponsord_is_active.status.ActiveState | default('') == 'active') + +- name: Apply pending sponsord changes + ansible.builtin.meta: flush_handlers + +- name: Enable and start sponsord + ansible.builtin.systemd: + name: sponsord + enabled: true + state: started + daemon_reload: true + +# --- Readiness (fatal) ------------------------------------------------------------ +# The listener binds only after the keystore decrypts and the on-chain owner check +# passes, so this is the deploy's config gate. Retry while nothing answers (-1); +# any answer other than 200 is a fault and stops the retries at once. The journal +# is NOT echoed here: a provider error can quote the RPC URL, and this output lands +# on the control machine. +# KICS flags plain HTTP here; the target is the loopback listener asserted above. +- name: Wait for sponsord /healthz + ansible.builtin.uri: + url: "http://{{ sponsord_bind_address }}:{{ sponsord_port }}/healthz" + status_code: 200 + return_content: true + # A listener that accepts and then hangs must not stretch the window ~30x. + timeout: 5 + register: sponsord_health + retries: "{{ sponsord_readiness_retries }}" + delay: "{{ sponsord_readiness_delay }}" + until: sponsord_health.status | default(-1) != -1 + changed_when: false + failed_when: false + when: not ansible_check_mode + +- name: Report a sponsord that did not become healthy + ansible.builtin.fail: + msg: |- + sponsord did not answer 200 on http://{{ sponsord_bind_address }}:{{ sponsord_port }}/healthz + within ~{{ (sponsord_readiness_retries | int) * (sponsord_readiness_delay | int) }}s + (last status {{ sponsord_health.status | default('none') }}). It binds only after + decrypting the treasury keystore and confirming on-chain that the treasury owns + sponsord_pool_id, so the usual causes are a wrong password, a pool id owned by a + different wallet, the wrong chain/PaymentPool address, or an unreachable RPC + endpoint. Read the reason on the host: + sudo journalctl -u sponsord -e + when: + - not ansible_check_mode + - sponsord_health.status | default(-1) != 200 or sponsord_health.json.ok | default(false) is not true + +# /healthz is a constant answer, so the 200 alone does not prove it came from THIS +# unit: something else holding the port would pass while sponsord crash-loops on +# EADDRINUSE. The listener must belong to the running unit's main process. +- name: Read the sponsord unit state + ansible.builtin.systemd_service: + name: sponsord + register: sponsord_unit_state + when: not ansible_check_mode + +- name: Read the owner of the sponsord listener + ansible.builtin.command: + argv: [ss, -Hltnp, "sport = :{{ sponsord_port }}"] + register: sponsord_listener + changed_when: false + when: not ansible_check_mode + +- name: Require the listener to belong to the running sponsord unit + ansible.builtin.assert: + that: + - sponsord_unit_state.status.ActiveState == 'active' + - sponsord_unit_state.status.SubState == 'running' + - ('pid=' ~ sponsord_unit_state.status.MainPID ~ ',') in sponsord_listener.stdout + fail_msg: >- + Something answered /healthz on {{ sponsord_bind_address }}:{{ sponsord_port }}, + but it is not the sponsord unit (state + {{ sponsord_unit_state.status.ActiveState }}/{{ sponsord_unit_state.status.SubState }}, + MainPID {{ sponsord_unit_state.status.MainPID }}; listeners: + {{ sponsord_listener.stdout | default('none', true) }}). Another service holds + the port, so sponsord cannot bind it: free the port or set sponsord_port. + when: not ansible_check_mode + +# --- Records (the daemon is now running on these inputs) ----------------------------- +- name: Record the restart-input checksums + ansible.builtin.copy: + dest: "{{ sponsord_inputs_checksum_file }}" + owner: root + group: root + mode: "0600" + content: "{{ sponsord_inputs_sums }}\n" + when: not ansible_check_mode diff --git a/ansible/roles/sponsord/templates/sponsord.env.j2 b/ansible/roles/sponsord/templates/sponsord.env.j2 new file mode 100644 index 0000000..3f8a848 --- /dev/null +++ b/ansible/roles/sponsord/templates/sponsord.env.j2 @@ -0,0 +1,16 @@ +# MANAGED BY the sponsord role — do not edit by hand. NO secrets: the RPC URL is in +# {{ sponsord_secret_env_file }}; the API token, treasury keystore and password +# reach the daemon as systemd credentials (see sponsord.service). +SPONSORD_BIND={{ sponsord_bind_address }}:{{ sponsord_port }} +SPONSORD_CHAIN_ID={{ sponsord_chain_id }} +SPONSORD_PAYMENT_POOL_ADDR={{ sponsord_payment_pool_address }} +SPONSORD_POOL_ID={{ sponsord_pool_id }} +{% for name, value in [ + ('SPONSORD_MAX_SPENDING_CAP_MICRO_USDC', sponsord_max_spending_cap_micro_usdc), + ('SPONSORD_MAX_TTL_SECS', sponsord_max_ttl_secs), + ('SPONSORD_POOL_LOW_WATER_MICRO_USDC', sponsord_pool_low_water_micro_usdc), + ('SPONSORD_POOL_REFILL_MICRO_USDC', sponsord_pool_refill_micro_usdc), + ('SPONSORD_POOL_WATCH_INTERVAL_SECS', sponsord_pool_watch_interval_secs), + ] if value | string | length > 0 %} +{{ name }}={{ value }} +{% endfor %} diff --git a/ansible/roles/sponsord/templates/sponsord.service.j2 b/ansible/roles/sponsord/templates/sponsord.service.j2 new file mode 100644 index 0000000..5406505 --- /dev/null +++ b/ansible/roles/sponsord/templates/sponsord.service.j2 @@ -0,0 +1,80 @@ +# MANAGED BY the sponsord role — do not edit by hand. +# +# sponsord signs capabilities with the treasury wallet and tops its PaymentPool +# pool up. It listens on loopback only ({{ sponsord_bind_address }}:{{ sponsord_port }}: +# the bearer-authenticated API plus unauthenticated /healthz and /metrics) and +# keeps no state. The API token, treasury keystore and password never sit in the +# environment: systemd reads the root-only files under {{ sponsord_etc }} and hands +# them to this unit as credentials, readable only by its dynamic user. The RPC URL +# does reach it as an environment variable, from the 0600 secret.env. +[Unit] +Description=sponsord: capability signing and pool top-ups +Documentation=https://github.com/decdn/sponsord +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +ExecStart={{ sponsord_bin }} + +# The RPC URL (may embed an API key; 0600 root), then the non-secret settings. +# A later file wins on a repeated key, so the role's validated settings win over +# anything stray in secret.env (the role also refuses any key there but +# SPONSORD_RPC_URL). Both override the Environment= lines below. +EnvironmentFile={{ sponsord_secret_env_file }} +EnvironmentFile={{ sponsord_env_file }} +# Plain-text logs into journald, without ANSI colour codes. +Environment=NO_COLOR=1 + +# Credentials land at $CREDENTIALS_DIRECTORY (%d), readable only by this unit. +LoadCredential=api-token:{{ sponsord_api_token_file }} +LoadCredential=treasury-password:{{ sponsord_treasury_password_file }} +LoadCredential=treasury-keystore.json:{{ sponsord_treasury_keystore_file }} +Environment=SPONSORD_API_TOKEN_FILE=%d/api-token +Environment=SPONSORD_TREASURY_PASSWORD_FILE=%d/treasury-password +# The keystore cannot be read from %d directly: newer systemd (255) writes credentials +# 0440, and upstream refuses a keystore with any group/other bit. So it is copied +# at 0600 into this unit's private 0700 runtime directory (tmpfs, removed when the +# unit stops). The file is the ENCRYPTED keystore; the password stays in %d. +RuntimeDirectory=sponsord +RuntimeDirectoryMode=0700 +ExecStartPre=/usr/bin/install -m 0600 %d/treasury-keystore.json %t/sponsord/treasury-keystore.json +Environment=SPONSORD_TREASURY_KEYSTORE=%t/sponsord/treasury-keystore.json + +# On SIGTERM the daemon stops serving, then waits for a top-up it already sent; +# TimeoutStopSec is the cap before SIGKILL. +KillSignal=SIGTERM +TimeoutStopSec={{ sponsord_stop_timeout_sec }} +Restart=on-failure +RestartSec=5 + +# --- Hardening ------------------------------------------------------------- +# Stateless: no StateDirectory, nothing on disk is writable. +DynamicUser=true +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +PrivateDevices=true +ProtectKernelTunables=true +ProtectKernelModules=true +ProtectKernelLogs=true +ProtectControlGroups=true +ProtectClock=true +ProtectHostname=true +CapabilityBoundingSet= +# AF_INET/AF_INET6 for the listener and the RPC endpoint; AF_UNIX + AF_NETLINK +# for the libc resolver. +RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK +RestrictNamespaces=true +RestrictRealtime=true +RestrictSUIDSGID=true +LockPersonality=true +MemoryDenyWriteExecute=true +SystemCallArchitectures=native +SystemCallFilter=@system-service +SystemCallErrorNumber=EPERM +UMask=0077 + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/sponsord/vars/main/arch.yml b/ansible/roles/sponsord/vars/main/arch.yml new file mode 100644 index 0000000..db87007 --- /dev/null +++ b/ansible/roles/sponsord/vars/main/arch.yml @@ -0,0 +1,16 @@ +--- +# Host architecture (ansible_facts['architecture']) -> the upstream release target +# triple. These are the two Linux targets decdn/sponsord's release builds (LINUX in +# .github/scripts/release_plan.py); +# the defaults derive sponsord_target from this map and tasks/main.yml asserts the +# result is one of its values. Role vars, not defaults: the supported set is a fact +# about upstream's release matrix, not an operator knob. +_sponsord_arch_targets: + x86_64: x86_64-unknown-linux-gnu + aarch64: aarch64-unknown-linux-gnu + +# sponsord_target -> the architecture token `file -b` prints for a matching ELF, +# used by the manual dir-mode binary check. +_sponsord_arch_file_tokens: + x86_64-unknown-linux-gnu: x86-64 + aarch64-unknown-linux-gnu: aarch64 diff --git a/ansible/roles/sponsord/vars/main/networks.yml b/ansible/roles/sponsord/vars/main/networks.yml new file mode 100644 index 0000000..e1324fe --- /dev/null +++ b/ansible/roles/sponsord/vars/main/networks.yml @@ -0,0 +1,14 @@ +--- +# GENERATED by scripts/sync-network-profiles.py -- do not hand-edit. +# Source: decdn/decdn @ 5ca3e7f1c5ac84890fb5b666b29b6f384e3797ec +# crates/cli/src/known_chains.rs (KNOWN_CHAINS, KnownChain::addresses) +# + contracts/deployments/.json +# +# The subset of roles/decdn_node/vars/main/networks.yml sponsord needs. The role's +# chain defaults read this map lazily when sponsord_network is set, so an explicit +# inventory value still wins. Protocol facts: upstream's deployment manifests are +# the source, this file only mirrors them. +_sponsord_network_profiles: + arbitrum-sepolia: + chain_id: 421614 + payment_pool_address: "0x351E5544471e11638b89DF8E3A3299c28E00cEc3" diff --git a/ansible/tests/alloy-config/render.yml b/ansible/tests/alloy-config/render.yml index dbb6960..0ed9fbb 100644 --- a/ansible/tests/alloy-config/render.yml +++ b/ansible/tests/alloy-config/render.yml @@ -275,6 +275,47 @@ grafana_alloy_bin_effective: /usr/bin/alloy grafana_alloy_logs_drop_priority_regex: 'info|debug|trace' + # A host that runs only sponsord (playbooks/group_vars/all.yml derives these + # toggles from sponsord_hosts): no decdn-node scrape or identity relabeller, a + # sponsord scrape on a non-default port with its own identity. + - name: Render the sponsord-only configuration + ansible.builtin.template: + src: "{{ playbook_dir }}/../../roles/grafana_alloy/templates/{{ item.src }}" + dest: "{{ _ga_render_dir }}/sponsordonly.{{ item.ext }}" + mode: "0644" + loop: *ga_render_pair + loop_control: + label: "sponsordonly.{{ item.ext }}" + vars: + _ga_instance_id: sponsord-1 + _ga_journal_groups: [systemd-journal, adm] + grafana_alloy_bin_effective: /usr/bin/alloy + grafana_alloy_deployment_environment: production + grafana_alloy_region: US + grafana_alloy_node_enabled: false + grafana_alloy_sponsord_enabled: true + sponsord_port: 18090 + + # A host in both decdn_nodes and sponsord_hosts: both scrapes, both identities. + # With an info-dropping guardrail, so validate.sh's level harness proves both + # daemons are judged by the level they logged, not journald's. + - name: Render the co-located node + sponsord configuration + ansible.builtin.template: + src: "{{ playbook_dir }}/../../roles/grafana_alloy/templates/{{ item.src }}" + dest: "{{ _ga_render_dir }}/colocated.{{ item.ext }}" + mode: "0644" + loop: *ga_render_pair + loop_control: + label: "colocated.{{ item.ext }}" + vars: + _ga_instance_id: decdn-node-1 + _ga_journal_groups: [systemd-journal, adm] + grafana_alloy_bin_effective: /usr/bin/alloy + grafana_alloy_deployment_environment: production + grafana_alloy_region: US + grafana_alloy_sponsord_enabled: true + grafana_alloy_logs_drop_priority_regex: 'info|debug|trace' + - name: Export the pinned release for the validation script ansible.builtin.copy: dest: "{{ _ga_render_dir }}/pin.env" diff --git a/ansible/tests/alloy-config/validate.sh b/ansible/tests/alloy-config/validate.sh index a9722af..508c0d1 100755 --- a/ansible/tests/alloy-config/validate.sh +++ b/ansible/tests/alloy-config/validate.sh @@ -236,6 +236,24 @@ assert_lacks hostonly.service 'SupplementaryGroups' "journal groups while logs a assert_lacks legacy.service 'SupplementaryGroups' "journal groups while logs are disabled" echo "ok: machine-monitoring render matrix" +# --- Gate 1b': which daemons are scraped follows the per-host toggles ----------- +assert_lacks defaults.alloy 'prometheus.scrape "sponsord"' "a sponsord scrape while it is disabled" +assert_lacks defaults.alloy 'sponsord\\.service' "a sponsord log rule while it is disabled" +assert_lacks sponsordonly.alloy 'prometheus.scrape "decdn_node"' "the node scrape on a host without a node" +assert_lacks sponsordonly.alloy 'prometheus.relabel "decdn_identity"' "the node identity relabeller on a host without a node" +assert_has sponsordonly.alloy 'prometheus.scrape "sponsord"' "the sponsord scrape" +assert_has sponsordonly.alloy '"__address__" = "127.0.0.1:18090"' "the sponsord listener from sponsord_port" +assert_has sponsordonly.alloy '"job" = "sponsord"' "the sponsord job label" +assert_has sponsordonly.alloy 'forward_to = [prometheus.relabel.sponsord_identity.receiver]' "the sponsord scrape feeding its own identity" +assert_has sponsordonly.alloy 'replacement = "sponsord"' "the sponsord metric service_name" +assert_has sponsordonly.alloy 'regex = "sponsord\\.service"' "the sponsord log-stream unit match" +assert_has sponsordonly.alloy 'replacement = "sponsord"' "the sponsord log-stream service_name" +assert_has sponsordonly.alloy '(decdn-node|sponsord|alloy|ssh|sshd)' "sponsord in the systemd collector's unit list" +assert_has colocated.alloy 'prometheus.scrape "decdn_node"' "the node scrape on a co-located host" +assert_has colocated.alloy 'prometheus.scrape "sponsord"' "the sponsord scrape on a co-located host" +assert_has colocated.alloy '"__address__" = "127.0.0.1:8090"' "the default sponsord listener" +echo "ok: per-daemon scrape toggles" + # --- Gate 1c: the daemon's level comes from its JSON body ---------------------- # Loading proves syntax, not behaviour. Run the RENDERED journal_rules and # loki.process "daemon_level" blocks in the real Alloy, fed sample lines through @@ -259,6 +277,15 @@ samples_json() { # nanosecond timestamp {"streams":[ {"stream":{"journal__systemd_unit":"other.service","journal_priority_keyword":"debug"},"values":[ ["$1","m-other-debug"]]}, + {"stream":{"journal__systemd_unit":"sponsord.service","journal_priority_keyword":"info"},"values":[ + ["$1","2026-10-05T07:59:10.000001Z TRACE sponsord: s-trace"], + ["$1","2026-10-05T07:59:10.000002Z DEBUG sponsord_core::keeper: s-debug"], + ["$1","2026-10-05T07:59:10.000003Z INFO sponsord: s-info"], + ["$1","2026-10-05T07:59:10.000004Z WARN sponsord_core::keeper: s-warn"], + ["$1","2026-10-05T07:59:10.000005Z ERROR sponsord: s-error"], + ["$1","Error: s-raw read PaymentPool.usdc()"], + ["$1","Caused by: s-cause connection refused"], + ["$1","s-plain an unlevelled line"]]}, {"stream":{"journal__systemd_unit":"other.service","journal_priority_keyword":"info"},"values":[ ["$1","{\"level\":\"ERROR\",\"fields\":{\"message\":\"m-other-info\"}}"]]}, {"stream":{"journal__systemd_unit":"other.service","journal_priority_keyword":"error"},"values":[ @@ -356,8 +383,11 @@ dropped() { # marker ! grep -qF "$1" "$harness_log" || harness_fail "$1 should have been dropped" } -# Defaults (guardrail "debug|trace"). -run_level_harness defaults.alloy 9 +# Defaults (guardrail "debug|trace"), sponsord not enabled: its plain-text lines +# are not re-levelled and stand at journald's info. +run_level_harness defaults.alloy 17 +kept s-trace info sponsord.service +kept s-error info sponsord.service kept m-warn-uc warning decdn-node.service kept m-warn-lc warning decdn-node.service kept m-error error decdn-node.service @@ -376,6 +406,7 @@ echo "ok: daemon log level follows the JSON body (defaults)" # must be judged by its JSON level — errors and warnings survive, and lines with # no JSON level are never dropped for a priority they did not choose. run_level_harness priorityinfo.alloy 7 +dropped s-error # sponsord not enabled: not exempt either kept m-warn-uc warning decdn-node.service kept m-warn-lc warning decdn-node.service kept m-error error decdn-node.service @@ -390,6 +421,34 @@ dropped m-other-info dropped m-other-debug echo "ok: daemon log level follows the JSON body (info in the guardrail)" +# sponsord enabled: its level is parsed from the plain-text line. +run_level_harness sponsordonly.alloy 15 +kept s-info info sponsord.service +kept s-warn warning sponsord.service +kept s-error error sponsord.service +kept s-raw error sponsord.service # anyhow's exit error: an error +kept s-cause error sponsord.service # ... and its cause chain +kept s-plain info sponsord.service # any other unlevelled line: journald's stands +dropped s-trace +dropped s-debug +kept m-error error decdn-node.service # the node's JSON stage is unaffected +echo "ok: sponsord log level follows its text line (defaults)" + +# Co-located, guardrail "info|debug|trace": both daemons exempt from the +# journald-priority drop and judged by the level they logged. +run_level_harness colocated.alloy 12 +kept s-warn warning sponsord.service +kept s-error error sponsord.service +kept s-raw error sponsord.service +kept s-plain info sponsord.service # exempt: no level of its own to judge by +kept m-error error decdn-node.service +kept m-text info decdn-node.service +dropped s-info +dropped s-trace +dropped m-info +dropped m-other-info +echo "ok: both daemons judged by their own level (info in the guardrail)" + # --- Gate 2: every ExecStart flag exists ------------------------------------- # `alloy run` ignores nothing: an unknown flag exits non-zero, i.e. a systemd # crash-loop on the target host the moment the unit starts. diff --git a/docs/requirements.md b/docs/requirements.md index 034c1e5..47682b9 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -27,7 +27,7 @@ and they run the same roles. | | Supported | How it is tested | |---|---|---| -| OS | Debian 12 (bookworm), Debian 13 (trixie), Ubuntu 24.04 (noble), Ubuntu 26.04 (resolute) | molecule converges `decdn_node` on all four in systemd containers; `grafana_alloy`'s install path on Debian 12 only (its disabled path on all four); `baseline` on real hosts | +| OS | Debian 12 (bookworm), Debian 13 (trixie), Ubuntu 24.04 (noble), Ubuntu 26.04 (resolute) | molecule converges `decdn_node` on all four in systemd containers; `grafana_alloy`'s install path on Debian 12 only (its disabled path on all four); `sponsord` on Debian 12 and Ubuntu 24.04; `baseline` on real hosts | | Architecture | x86_64, aarch64 | upstream builds both; the Ansible role derives the target from the host | | Ansible (control machine) | ansible-core ≥ 2.15 | CI runs the current release | | cloud-init | the provider image's own; the bootstrap installs its pinned ansible-core on the host | the user-data is booted with the distro's cloud-init in Debian 12 and Ubuntu 26.04 containers, to a running node | diff --git a/scripts/sync-network-profiles.py b/scripts/sync-network-profiles.py index 6c6232e..a9a8138 100755 --- a/scripts/sync-network-profiles.py +++ b/scripts/sync-network-profiles.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Regenerate the decdn_node role's network profiles from upstream deployment manifests. +"""Regenerate the roles' network profiles from upstream deployment manifests. scripts/sync-network-profiles.py [--ref origin/main] [--check] @@ -8,6 +8,10 @@ would bake into node.toml. Operators then set `decdn_network: ` instead of hand-copying eight addresses that change wholesale on every upstream redeploy. +Also writes ansible/roles/sponsord/vars/main/networks.yml, the subset sponsord needs +(chain_id + the PaymentPool address), from the same parsed data, so the two roles can +never disagree about a chain. --check covers both files. + Everything is read from git at --ref (default origin/main), never from the working tree, so a checkout sitting on a feature branch still yields main's deployment. Two upstream files are the source, and both the chain list AND the manifest-key mapping @@ -35,6 +39,7 @@ REPO = Path(__file__).resolve().parent.parent OUT = REPO / "ansible/roles/decdn_node/vars/main/networks.yml" +SPONSORD_OUT = REPO / "ansible/roles/sponsord/vars/main/networks.yml" # Upstream ChainAddresses field -> decdn_node variable suffix (decdn_). # Order is the output order. @@ -49,6 +54,10 @@ "usdc": "usdc_address", } +# The FIELDS values sponsord's profile keeps (its role variables are +# sponsord_). Order is the output order. +SPONSORD_FIELDS = ["payment_pool_address"] + ADDRESS = re.compile(r"^0x[0-9a-fA-F]{40}$") REGISTRY = re.compile(r"KNOWN_CHAINS\s*:\s*&\[\s*KnownChain\s*\]\s*=\s*&\[(?P.*?)\];", re.S) ENTRY = re.compile(r"KnownChain\s*\{(?P.*?)\}", re.S) @@ -108,25 +117,14 @@ def parse_mapping(source: str) -> dict[str, tuple[str, str]]: return mapping -def render(upstream: Path, ref: str) -> str: +def load(upstream: Path, ref: str) -> tuple[str, list[dict]]: + """(commit, [{name, chain_id, ..., deploy_block}]) from upstream at ref.""" commit = git(upstream, "rev-parse", "--verify", f"{ref}^{{commit}}").strip() source = git(upstream, "show", f"{ref}:crates/cli/src/known_chains.rs") chains = parse_registry(source) mapping = parse_mapping(source) - lines = [ - "---", - "# GENERATED by scripts/sync-network-profiles.py -- do not hand-edit.", - f"# Source: decdn/decdn @ {commit}", - "# crates/cli/src/known_chains.rs (KNOWN_CHAINS, KnownChain::addresses)", - "# + contracts/deployments/.json", - "#", - "# The addresses `decdn config init --chain ` bakes into node.toml. The role's", - "# address defaults read this map lazily when decdn_network is set, so an explicit", - "# inventory value still wins. Protocol facts: upstream's deployment manifests are", - "# the source, this file only mirrors them.", - "_decdn_network_profiles:", - ] + profiles = [] for name, chain_id in chains: raw = git(upstream, "show", f"{ref}:contracts/deployments/{chain_id}.json") try: @@ -135,14 +133,56 @@ def render(upstream: Path, ref: str) -> str: die(f"{chain_id}.json is not valid JSON: {e}") if manifest.get("chainId") != chain_id: die(f"{chain_id}.json chainId {manifest.get('chainId')} != registry {chain_id} for {name}") - lines += [f" {name}:", f" chain_id: {chain_id}"] + profile = {"name": name, "chain_id": chain_id} for field, var in FIELDS.items(): section, key = mapping[field] value = manifest.get(section, {}).get(key) if not isinstance(value, str) or not ADDRESS.match(value): die(f"{name}: {section}.{key} is {value!r} in {chain_id}.json, not a 0x address") - lines.append(f' {var}: "{value}"') - lines.append(f" deploy_block: {int(manifest['deployBlock'])}") + profile[var] = value + profile["deploy_block"] = int(manifest["deployBlock"]) + profiles.append(profile) + return commit, profiles + + +def render(commit: str, profiles: list[dict]) -> str: + lines = [ + "---", + "# GENERATED by scripts/sync-network-profiles.py -- do not hand-edit.", + f"# Source: decdn/decdn @ {commit}", + "# crates/cli/src/known_chains.rs (KNOWN_CHAINS, KnownChain::addresses)", + "# + contracts/deployments/.json", + "#", + "# The addresses `decdn config init --chain ` bakes into node.toml. The role's", + "# address defaults read this map lazily when decdn_network is set, so an explicit", + "# inventory value still wins. Protocol facts: upstream's deployment manifests are", + "# the source, this file only mirrors them.", + "_decdn_network_profiles:", + ] + for p in profiles: + lines += [f" {p['name']}:", f" chain_id: {p['chain_id']}"] + lines += [f' {var}: "{p[var]}"' for var in FIELDS.values()] + lines.append(f" deploy_block: {p['deploy_block']}") + return "\n".join(lines) + "\n" + + +def render_sponsord(commit: str, profiles: list[dict]) -> str: + lines = [ + "---", + "# GENERATED by scripts/sync-network-profiles.py -- do not hand-edit.", + f"# Source: decdn/decdn @ {commit}", + "# crates/cli/src/known_chains.rs (KNOWN_CHAINS, KnownChain::addresses)", + "# + contracts/deployments/.json", + "#", + "# The subset of roles/decdn_node/vars/main/networks.yml sponsord needs. The role's", + "# chain defaults read this map lazily when sponsord_network is set, so an explicit", + "# inventory value still wins. Protocol facts: upstream's deployment manifests are", + "# the source, this file only mirrors them.", + "_sponsord_network_profiles:", + ] + for p in profiles: + lines += [f" {p['name']}:", f" chain_id: {p['chain_id']}"] + lines += [f' {var}: "{p[var]}"' for var in SPONSORD_FIELDS] return "\n".join(lines) + "\n" @@ -157,17 +197,23 @@ def main() -> None: ap.add_argument("--check", action="store_true", help="exit 1 if the committed file is stale") args = ap.parse_args() - text = render(args.upstream, args.ref) + commit, profiles = load(args.upstream, args.ref) + outputs = [(OUT, render(commit, profiles)), (SPONSORD_OUT, render_sponsord(commit, profiles))] if args.check: - current = OUT.read_text() if OUT.exists() else "" - if strip_commit(current) != strip_commit(text): - print(f"{OUT.relative_to(REPO)} is stale against {args.ref}; re-run without --check", file=sys.stderr) + stale = False + for out, text in outputs: + current = out.read_text() if out.exists() else "" + if strip_commit(current) != strip_commit(text): + print(f"{out.relative_to(REPO)} is stale against {args.ref}; re-run without --check", file=sys.stderr) + stale = True + else: + print(f"{out.relative_to(REPO)} is current against {args.ref}") + if stale: sys.exit(1) - print(f"{OUT.relative_to(REPO)} is current against {args.ref}") return - OUT.write_text(text) - print(f"wrote {OUT.relative_to(REPO)}") - + for out, text in outputs: + out.write_text(text) + print(f"wrote {out.relative_to(REPO)}") if __name__ == "__main__": main()