diff --git a/AGENTS.md b/AGENTS.md index 11c2d92..7398d6d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -140,10 +140,19 @@ make security # = security-ansible + security-helm (KICS over the render cd ansible make deps # vendor pinned Galaxy collections into ./collections make check / deploy # deCDN node (site.yml): dry-run / provision - # fleet-wide by default; LIMIT= scopes, ANSIBLE_ARGS='…' passes through + # fleet-wide by default; LIMIT= scopes, ANSIBLE_ARGS='…' passes through; + # INVENTORY= targets a private fleet overlay (default inventory/hosts.yml) make build / galaxy-check # stage + build the decdn.node collection, then validate it ``` +**Inventory is private; the firewall hole is not.** This repo is public, so +`ansible/inventory/hosts.yml` is git-ignored and a real fleet lives in a private overlay +(template: `ansible/inventory/fleet.example/`, launch sequence in +`ansible/docs/launch-runbook.md`). Inventory-adjacent group_vars do not load for an +overlay, so anything every node needs regardless of inventory (today only the udp/4433 +`baseline_extra_inbound` hole) lives in `ansible/playbooks/group_vars/decdn_nodes.yml`. +Don't move it back under `inventory/`. + **Galaxy collection (`decdn.node`).** The three roles (`baseline` + `decdn_node` + `grafana_alloy`) ship as a distributable collection. The overlay lives in `ansible/galaxy/` and is staged into a clean diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..fd46ed8 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,39 @@ +# Security + +## Reporting a vulnerability + +Email `security@decdn.org`. Please do not open a public issue for security +reports. That includes a secret or real host address you find committed here. + +## Scope + +This repo holds the deployment tooling for a deCDN node: the Ansible roles, the +`decdn.node` Galaxy collection, and the Helm chart. Reports about the node daemon or +protocol belong to [decdn/decdn](https://github.com/decdn/decdn), but the same address +reaches both. + +The security model these tools follow is described in [README.md § Security +model](README.md#security-model): nothing secret committed, loopback-only backends, +default-deny inbound, DevSec host hardening. + +## Release verification + +In `release` install mode, and while `decdn_verify_release_signature` keeps its +default of `true`, the `decdn_node` role checks every downloaded tarball against the +release's GPG-signed `SHA256SUMS`, using the maintainer key +vendored at `ansible/roles/decdn_node/files/decdn-release-KEYS.asc`. That key is a +copy of upstream's `KEYS`: + +``` +Ant Somers +Fingerprint: DA75 1570 6F18 73D2 74D8 A369 9E11 A9FF D62D AADB +``` + +The same fingerprint is published in +[decdn/decdn's SECURITY.md](https://github.com/decdn/decdn/blob/main/SECURITY.md). +Check a new copy of the vendored key against it before trusting that copy, with +`gpg --show-keys --with-fingerprint `. + +Setting `decdn_verify_release_signature: false` (meant for an air-gapped mirror that +strips signatures) drops that guarantee, and the role prints a warning when it's off. +`manual` mode verifies nothing: it installs whatever binaries you point it at. diff --git a/ansible/.gitignore b/ansible/.gitignore index 62ced30..fb6807b 100644 --- a/ansible/.gitignore +++ b/ansible/.gitignore @@ -5,9 +5,19 @@ collections/ # copy named secret.yaml or secret.json (Ansible loads .yml/.yaml/.json all the same) # can't leak; the committed secret.*.example template is re-included. Everything else in # host_vars — main.yml (contract addresses, region, cache origin, …) — is committable -# template config. hosts.yml is committable too (your fork's inventory is yours to track). +# template config. inventory/host_vars/*/secret.* !inventory/host_vars/*/secret.*.example +# Same rule for inventory templates nested one level down (inventory/fleet.example/). +inventory/*/host_vars/*/secret.* +!inventory/*/host_vars/*/secret.*.example + +# The real inventory. This repo is PUBLIC, so real host IPs must never land here +# (Launch/release-checklist.md §5). Copy hosts.yml.example to hosts.yml for a local +# run, or keep a fleet's inventory in a private overlay and pass INVENTORY= +# (see README.md "Private fleet inventory"). The .example templates stay tracked. +inventory/hosts.yml +inventory/hosts.yaml # Ansible scratch *.retry diff --git a/ansible/Makefile b/ansible/Makefile index fcbd44d..be7522c 100644 --- a/ansible/Makefile +++ b/ansible/Makefile @@ -33,6 +33,20 @@ LIMIT ?= ANSIBLE_ARGS ?= limit_arg = $(if $(LIMIT),--limit '$(LIMIT)') +# Which inventory check/deploy run against. The default is the git-ignored +# inventory/hosts.yml (copy hosts.yml.example). A real fleet keeps its inventory in a +# private overlay outside this public repo and points at it here: +# make deploy INVENTORY=../../decdn-fleet/hosts.yml LIMIT= +# The overlay's own group_vars/ and host_vars/ must sit beside that hosts.yml, where +# Ansible loads them automatically. See ansible/README.md "Private fleet inventory". +INVENTORY ?= inventory/hosts.yml +# A missing or unparseable inventory fails the run. Ansible's default only warns, +# skips the `hosts: decdn_nodes` play and exits 0, so a mistyped INVENTORY= would +# report a successful deploy that did nothing. This applies to check/deploy only: +# ansible.cfg can't set it globally, because ansible-lint's own syntax checks run +# with no inventory present. +strict_inventory = ANSIBLE_INVENTORY_UNPARSED_FAILED=True + # Fail loud on the two ways a run can look scoped but silently be fleet-wide. Scoped to # check/deploy via MAKECMDGOALS so an unrelated `make lint` is never blocked. NOTE: this # does not catch an argument that is well-formed but selects nothing — `--tags` with a @@ -47,6 +61,12 @@ ifneq ($(filter check deploy,$(MAKECMDGOALS)),) ifeq ($(origin ANSIBLE_ARGS),environment) $(error ANSIBLE_ARGS is set in your environment and would silently alter this run. Unset it or pass it on the command line) endif + ifeq ($(origin INVENTORY),environment) + $(error INVENTORY is set in your environment ($(INVENTORY)) and would silently retarget this run. Unset it or pass it on the command line: make $(scoped_goal) INVENTORY=) + endif + ifeq ($(strip $(INVENTORY)),) + $(error INVENTORY expanded empty (unset shell variable?) - refusing to guess which fleet to target) + endif ifeq ($(origin LIMIT),command line) ifeq ($(strip $(LIMIT)),) $(error LIMIT was supplied but expanded empty (unset shell variable?) - refusing to run fleet-wide by accident. Omit LIMIT entirely to mean "all hosts") @@ -54,13 +74,13 @@ ifneq ($(filter check deploy,$(MAKECMDGOALS)),) endif endif -# Dry run against the real inventory — shows diffs, mutates nothing. +# Dry run against $(INVENTORY) — shows diffs, mutates nothing. check: - ansible-playbook playbooks/site.yml --check --diff $(limit_arg) $(ANSIBLE_ARGS) + $(strict_inventory) ansible-playbook -i '$(INVENTORY)' playbooks/site.yml --check --diff $(limit_arg) $(ANSIBLE_ARGS) -# Provision the deCDN node host(s) in inventory/hosts.yml. +# Provision the deCDN node host(s) in $(INVENTORY) (default inventory/hosts.yml). deploy: - ansible-playbook playbooks/site.yml $(limit_arg) $(ANSIBLE_ARGS) + $(strict_inventory) ansible-playbook -i '$(INVENTORY)' playbooks/site.yml $(limit_arg) $(ANSIBLE_ARGS) # --- Tests ------------------------------------------------------------------- # Containerised converge + idempotence + verify of the decdn_node role against a diff --git a/ansible/README.md b/ansible/README.md index e3a7a6c..5b3b9ba 100644 --- a/ansible/README.md +++ b/ansible/README.md @@ -79,8 +79,33 @@ when the two would collide: an inventory value that would discard a host-side ed empty `decdn_rpc_url` that turns out to mean "`secret.yml` went missing" rather than "the host owns this file". -Your `hosts.yml` is no longer force-ignored — commit it in your fork if you want, or keep -it local. +`inventory/hosts.yml` is git-ignored. This repo is public, and real host IPs never belong in +it. + +### Private fleet inventory + +To track a real fleet, keep its inventory in a **private overlay** outside this repo and +point the deploy targets at it: + +```bash +cp -r inventory/fleet.example ../../decdn-fleet # into a PRIVATE repo/dir, then fill it in +make check INVENTORY=../../decdn-fleet/hosts.yml LIMIT= +make deploy INVENTORY=../../decdn-fleet/hosts.yml LIMIT= +``` + +Ansible loads `group_vars/` and `host_vars/` from beside whichever `hosts.yml` you pass, so +the overlay carries its own copies. The template's own `.gitignore` comes along too, so the +private repo never tracks a `host_vars/*/secret.yml`. The public `inventory/group_vars/` and +`inventory/host_vars/` are **not** loaded for it. Settings every node needs regardless of +inventory, currently only the udp/4433 QUIC firewall hole, live in +`playbooks/group_vars/decdn_nodes.yml`, so an overlay can't drop them. Override that per node +in `host_vars` if you have to. Playbook group_vars beat inventory group_vars. + +[`inventory/fleet.example/`](inventory/fleet.example/hosts.yml) is the launch-fleet +template. Hosts are grouped by role (`decdn_seed` holds the catalogue in an fs origin, +`decdn_edge` pulls through) and by billing (`decdn_metered` gets an egress budget, +`decdn_unmetered` does not). The group_vars size the cache for large model blobs. The +launch sequence is in [`docs/launch-runbook.md`](docs/launch-runbook.md). By default baseline **deploys you as yourself**: the runner (your control-machine `$USER` + its autodetected `~/.ssh` key, `id_ed25519` > `ecdsa` > `rsa`) is prepended as the head of diff --git a/ansible/docs/launch-runbook.md b/ansible/docs/launch-runbook.md new file mode 100644 index 0000000..1ea5732 --- /dev/null +++ b/ansible/docs/launch-runbook.md @@ -0,0 +1,110 @@ +# Launch fleet runbook + +The ops sequence for putting the launch fleet up with this project. The plan it +serves, with targets, owners and dates, is `Launch/fleet-plan.md` in the private +`decdn/internal` repo. This file records only the mechanics. Protocol and economic +facts (bond sizing, fees) come from the deCDN ADRs; nothing here restates them. + +## 0. Inventory + +1. Copy [`inventory/fleet.example/`](../inventory/fleet.example/hosts.yml) into a + **private** repo or dir (see [README § Private fleet inventory](../README.md#private-fleet-inventory)). + Never commit real IPs to this public repo. +2. For each host set `ansible_host` and `decdn_region`, then put it in exactly one + **role** group (`decdn_seed` / `decdn_edge`) and one **billing** group + (`decdn_metered` / `decdn_unmetered`). Hero and wow-factor traffic belongs on + unmetered nodes. +3. In `group_vars/decdn_nodes.yml`, fill the MUST-EDIT lines: binary source and + contract addresses copied verbatim from `decdn/contracts/deployments/421614.json`. + Size `decdn_cache_size_mb` to each host's disk. +4. Provision each node's secret on the host, the preferred way (`0600 /etc/decdn/decdn.env`, + [roles/decdn_node/README.md § Secrets](../roles/decdn_node/README.md#secrets)). + Provision the keystore too, or set `decdn_node_generate_keystore: true`. +5. For every metered node, set a **billing alert** in the provider console (Hetzner / + OVHcloud). `decdn_load_shed_egress_budget_mbps` caps throughput, and the alert makes + the spend visible. + +## 1. Deploy + +```bash +cd ansible +make deps +INV=../../decdn-fleet/hosts.yml # your private overlay +make check INVENTORY=$INV LIMIT= ANSIBLE_ARGS='-u root' # first converge: bootstrap user +make deploy INVENTORY=$INV LIMIT= ANSIBLE_ARGS='-u root' +make deploy INVENTORY=$INV # every run after bootstrap, fleet-wide +``` + +The role runs `decdn config validate` against the installed binary after templating, +so a key the binary doesn't know fails the deploy instead of crash-looping the node. + +## 2. Stake and register each node (manual) + +On-chain onboarding ([ADR 019](https://github.com/decdn/decdn/blob/main/adr/019-node-onboarding.md) +Phase 2) is an operator step, not automated here. Fund the node's eth address first +(`key-gen` printed it). Then run `decdn setup` on the node. It reads the rendered +config plus `DECDN_RPC_URL` from `decdn.env`, and `systemd-run` loads that file with +the same parser the node's unit uses, so the RPC key never lands in argv: + +```bash +# Preview first: pre-flight checks and the bond it would post, no transactions. +sudo systemd-run --pty --wait --collect -p User=decdn \ + -p EnvironmentFile=/etc/decdn/decdn.env \ + /usr/local/bin/decdn --config /etc/decdn/node.toml setup \ + --mbps --region \ + --multiaddr /ip4//udp/4433/quic-v1 \ + --keystore-password-file /etc/decdn/keystore.password --dry-run +# Then the same command without --dry-run. It shows the operator terms and the bond to confirm. +``` + +- `--region` is the on-chain `regionHint` ([ADR 030](https://github.com/decdn/decdn/blob/main/adr/030-node-region-self-attestation.md)). + It must match the host's `decdn_region`, since the stats map and takedown scope read it. +- The bond is read from the on-chain `bondRequired(mbps)` curve + ([ADR 026](https://github.com/decdn/decdn/blob/main/adr/026-tokenomics.md)). Declare + `--mbps` for what the node can really serve; there's no flat minimum to quote. +- The flags, the `node bond` / `node register` primitives and the exit path are in + [roles/decdn_node/README.md § On-chain onboarding](../roles/decdn_node/README.md#on-chain-onboarding). + +A node counts toward launch metrics only once it is registered and serving. + +## 3. Seed the catalogue (seed nodes) + +On each `decdn_seed` host, run `Launch/seed-model.sh` (from `decdn/internal`) as the +service user, writing into the fs origin that group_vars configure: + +```bash +sudo -u decdn OUT_DIR=/var/lib/decdn/origin ./seed-model.sh +``` + +It needs the `hf` CLI and disk for **both** the origin copy and the cache, on the same +`/var/lib/decdn` volume. With the template values that's about 352 GiB of origin (the +current whole-repo set, measured 2026-09-22), plus `decdn_cache_size_mb` (400 GiB), plus +`decdn_disk_headroom_mb` (8 GiB): **about 760 GiB per seed**. `hf download` also stages +a full copy before import, so allow for the largest model (~177 GiB, Mixtral) on top +while seeding, or lower the seeds' `decdn_cache_size_mb` in `group_vars/decdn_seed.yml`. +Record every BLAKE3 hash it prints: the bundle manifests plus the large blobs. + +## 4. Pin and fill the edges + +1. Add the recorded hashes (lowercase 64-hex, no `b3:` prefix) to + `decdn_pinned_hashes` in the overlay's `group_vars/decdn_nodes.yml`, then + `make deploy INVENTORY=$INV`. +2. A pin only stops eviction. Each edge still fetches a blob on its first miss, and + it's launch-ready once the blob is resident on enough nodes per region. + +## 5. Verify: the fleet part of the go/no-go + +- From a cold machine near each region, pull the **largest** blob (currently the + 13.5 GiB Mistral v0.3 `consolidated.safetensors`) by hash. A `BlobTooLarge` error + means that node's `decdn_max_blob_size_mb` is too small. +- `curl -s 127.0.0.1:9090/metrics` on a node (over SSH, since metrics are loopback) + shows cache hits and misses. With `decdn_grafana_cloud_enabled`, watch egress and + errors in Grafana Cloud. +- Every node shows up registered, with its region, on the explorer / stats map. + +## 6. When `v0.1.0` is tagged + +Switch the overlay to the signed tarballs (`decdn_node_install_method: release`, +`decdn_node_version: "0.1.0"`) and redeploy. The role verifies them against the +release's GPG-signed `SHA256SUMS`, using the vendored maintainer key. Re-check the config +schema coupling (`molecule/schema`, `make lint-helm`) against the tagged `types.rs`. diff --git a/ansible/inventory/fleet.example/.gitignore b/ansible/inventory/fleet.example/.gitignore new file mode 100644 index 0000000..de2033b --- /dev/null +++ b/ansible/inventory/fleet.example/.gitignore @@ -0,0 +1,4 @@ +# Travels with `cp -r inventory/fleet.example `, so the private +# repo never tracks a per-node secret either (decdn_rpc_url may embed an API key). +host_vars/*/secret.* +!host_vars/*/secret.*.example diff --git a/ansible/inventory/fleet.example/group_vars/decdn_edge.yml b/ansible/inventory/fleet.example/group_vars/decdn_edge.yml new file mode 100644 index 0000000..9932f44 --- /dev/null +++ b/ansible/inventory/fleet.example/group_vars/decdn_edge.yml @@ -0,0 +1,4 @@ +--- +# Edge nodes have no origin of their own. With no decdn_cache_origin_kind set, the +# role defaults node-to-node pull-through ON, so an edge fills a cache miss from +# a seed or peer that holds the blob. diff --git a/ansible/inventory/fleet.example/group_vars/decdn_metered.yml b/ansible/inventory/fleet.example/group_vars/decdn_metered.yml new file mode 100644 index 0000000..c9cf405 --- /dev/null +++ b/ansible/inventory/fleet.example/group_vars/decdn_metered.yml @@ -0,0 +1,9 @@ +--- +# Metered nodes bill per TB of egress, so a model that trends on r/LocalLLaMA +# must not run up the bill (Launch/fleet-plan.md). Cap their egress. +# This is OPERATOR COST CONFIG, not a protocol fact. Size it from the provider's +# per-TB price and the monthly spend you accept, e.g. 250 Mbps sustained is about +# 81 TB/month. +decdn_load_shed_egress_budget_mbps: 250 +# Also set a billing alert in the provider console (Hetzner / OVHcloud) for every +# metered node. The budget above caps throughput, the alert makes the spend visible. diff --git a/ansible/inventory/fleet.example/group_vars/decdn_nodes.yml b/ansible/inventory/fleet.example/group_vars/decdn_nodes.yml new file mode 100644 index 0000000..1796ef2 --- /dev/null +++ b/ansible/inventory/fleet.example/group_vars/decdn_nodes.yml @@ -0,0 +1,59 @@ +--- +# Settings shared by every launch node. Copied with the overlay; edit the private copy. + +# --- Binaries ----------------------------------------------------------------- +# Until upstream tags v0.1.0, install locally built binaries. Build them from a +# decdn/decdn checkout on the release commit: +# cargo build --release -p decdn-node -p decdn-cli +decdn_node_install_method: manual +decdn_release_target_dir: "" # MUST-EDIT - control-machine path to decdn's target/release +# Once the v0.1.0 release exists (Launch/release-checklist.md §1), switch to the +# GPG-verified tarballs and drop the two lines above: +# decdn_node_install_method: release +# decdn_node_version: "0.1.0" + +# --- Contracts (Arbitrum Sepolia, 421614) ---------------------------------------- +# Protocol facts. Copy them VERBATIM from the upstream deployment manifest +# decdn/contracts/deployments/421614.json. The public repo's +# inventory/host_vars/decdn-node-1/main.yml carries the current copy with its sync +# note. Never guess. Upstream redeploys change every address at once. +decdn_chain_id: 421614 +decdn_payment_pool_address: "" # MUST-EDIT +decdn_capacity_bond_address: "" # MUST-EDIT +decdn_slash_judge_address: "" # MUST-EDIT +decdn_content_blacklist_address: "" # MUST-EDIT +decdn_slash_appeal_address: "" +decdn_origin_assignment_address: "" +decdn_publisher_registry_address: "" +decdn_usdc_address: "" + +# --- Cache sizing for the launch model catalogue --------------------------------- +# `decdn origin import` stores every file as ONE whole-file blob (--optimize only +# adds chunk hints to the manifest), and a node rejects any blob over +# max_blob_size_mb with BlobTooLarge on pull-through. The role default (1024) is far +# too small for model weights. Measured from the Hugging Face file trees on +# 2026-09-22 for the seed-model.sh set (whole repos, as that script downloads them): +# largest single file: 13.5 GiB Mistral-7B-Instruct-v0.3 consolidated.safetensors +# 12.8 GiB gpt-oss-20b original/model.safetensors +# 11.3 GiB Mixtral-8x7B consolidated.02.pt +# 9.6 GiB SDXL unet (fp32) +# whole set: ~352 GiB (Mixtral alone ~177 GiB) +# Re-measure if the seed list or its --exclude patterns change. +decdn_max_blob_size_mb: 16384 # 16 GiB: largest file 13.5 GiB, plus headroom +# Must hold the catalogue you expect a node to serve, plus room for LRU churn, and +# must be >= max_blob_size_mb. The daemon also keeps decdn_disk_headroom_mb (default +# 8 GiB) free on the volume. Size it to each host's disk: 400 GiB covers the whole +# set above. Override per host in host_vars if disks differ. +decdn_cache_size_mb: 409600 # 400 GiB + +# --- Pinning the catalogue -------------------------------------------------------- +# Eviction-exempt pins. Fill these after seeding: the lowercase 64-hex BLAKE3 hashes +# `decdn origin import` prints for each model's bundle manifest AND its large blobs. +# A pin only stops eviction. The node still has to fetch the blob once before it +# holds it. +decdn_pinned_hashes: [] + +# --- Observability ---------------------------------------------------------------- +# Recommended for launch: live metrics + journald per node (egress, errors). +# Provision the token first (roles/grafana_alloy/README.md). +# decdn_grafana_cloud_enabled: true diff --git a/ansible/inventory/fleet.example/group_vars/decdn_seed.yml b/ansible/inventory/fleet.example/group_vars/decdn_seed.yml new file mode 100644 index 0000000..7eeed0c --- /dev/null +++ b/ansible/inventory/fleet.example/group_vars/decdn_seed.yml @@ -0,0 +1,11 @@ +--- +# Seed nodes hold the model catalogue in a local filesystem origin, which is the +# directory Launch/seed-model.sh writes into: +# sudo -u decdn OUT_DIR=/var/lib/decdn/origin ./seed-model.sh +# /var/lib/decdn is the unit's StateDirectory, so the hardened daemon can read it +# under ProtectSystem=strict. The role creates the directory on first converge. +decdn_cache_origin_kind: fs +decdn_cache_origin_path: /var/lib/decdn/origin +# Seeds need disk for the origin copy (~352 GiB for the full set) AND the cache +# (decdn_cache_size_mb) on the same volume, about 760 GiB with the template values. +# See docs/launch-runbook.md §3. Lower decdn_cache_size_mb here for smaller seed disks. diff --git a/ansible/inventory/fleet.example/group_vars/decdn_unmetered.yml b/ansible/inventory/fleet.example/group_vars/decdn_unmetered.yml new file mode 100644 index 0000000..fc0a3ba --- /dev/null +++ b/ansible/inventory/fleet.example/group_vars/decdn_unmetered.yml @@ -0,0 +1,4 @@ +--- +# Unmetered nodes: flat-rate bandwidth, capped by port speed rather than billed per +# TB. They take the hero and wow-factor traffic, so leave egress uncapped +# (decdn_load_shed_egress_budget_mbps unset, the daemon default of no budget). diff --git a/ansible/inventory/fleet.example/host_vars/fsn1-a/secret.yml.example b/ansible/inventory/fleet.example/host_vars/fsn1-a/secret.yml.example new file mode 100644 index 0000000..63729e0 --- /dev/null +++ b/ansible/inventory/fleet.example/host_vars/fsn1-a/secret.yml.example @@ -0,0 +1,6 @@ +--- +# OPTIONAL, and only in the PRIVATE overlay. The preferred posture is to provision a +# 0600 /etc/decdn/decdn.env on the host and leave this file absent, so the RPC URL +# never touches the control machine. See roles/decdn_node/README.md § Secrets. +# To use it, copy to secret.yml beside this file and keep it git-ignored. +decdn_rpc_url: "" # SENSITIVE; may embed an API key diff --git a/ansible/inventory/fleet.example/hosts.yml b/ansible/inventory/fleet.example/hosts.yml new file mode 100644 index 0000000..87f8a4c --- /dev/null +++ b/ansible/inventory/fleet.example/hosts.yml @@ -0,0 +1,68 @@ +--- +# TEMPLATE for a private fleet overlay: the launch fleet from +# decdn/internal Launch/fleet-plan.md (10–20 nodes, Hetzner + OVHcloud, >= 4–5 +# countries on 3 continents). Copy this whole directory OUT of this public repo: +# cp -r ansible/inventory/fleet.example ../../decdn-fleet # a PRIVATE repo/dir +# make deploy INVENTORY=../../decdn-fleet/hosts.yml LIMIT= +# Ansible loads group_vars/ and host_vars/ from beside the hosts.yml you pass, so the +# overlay carries its own. The QUIC firewall hole is NOT in here: it lives in +# playbooks/group_vars/decdn_nodes.yml, which applies to every inventory. +# +# Host names and addresses below are placeholders. Real IPs go in the private copy only. +# Bootstrap, ansible_user and the sudo-rs note: see ../hosts.yml.example. +# +# Every host belongs to decdn_nodes, the group site.yml targets, and to one group +# from each of two axes: +# role: decdn_seed - holds the catalogue in a local fs origin (seed-model.sh) +# decdn_edge - no origin; fills its cache from seeds/peers on demand +# billing: decdn_unmetered - flat-rate bandwidth: hero/wow-factor traffic goes here +# decdn_metered - pay-per-TB egress: map coverage, egress-capped +# Record the billing choice here, before provisioning (fleet-plan.md). The group is +# the record. +# +# decdn_region is REQUIRED per host (ISO 3166-1 alpha-2 of the physical location). +# It drives the stats map and the ADR 030 / ADR 011 regional scope. The role +# refuses to deploy without it. +decdn_nodes: + children: + decdn_seed: + hosts: + fsn1-a: # Hetzner Falkenstein + ansible_host: REPLACE_WITH_IP_OR_DNS + decdn_region: DE + ash-a: # Hetzner Ashburn + ansible_host: REPLACE_WITH_IP_OR_DNS + decdn_region: US + decdn_edge: + hosts: + hel1-a: # Hetzner Helsinki + ansible_host: REPLACE_WITH_IP_OR_DNS + decdn_region: FI + hil-a: # Hetzner Hillsboro + ansible_host: REPLACE_WITH_IP_OR_DNS + decdn_region: US + gra-a: # OVHcloud Gravelines + ansible_host: REPLACE_WITH_IP_OR_DNS + decdn_region: FR + waw-a: # OVHcloud Warsaw + ansible_host: REPLACE_WITH_IP_OR_DNS + decdn_region: PL + bhs-a: # OVHcloud Beauharnois + ansible_host: REPLACE_WITH_IP_OR_DNS + decdn_region: CA + sgp-a: # OVHcloud Singapore + ansible_host: REPLACE_WITH_IP_OR_DNS + decdn_region: SG + # Billing axis: the same hosts, listed again by name only. + decdn_unmetered: + hosts: + fsn1-a: + ash-a: + hel1-a: + hil-a: + decdn_metered: + hosts: + gra-a: + waw-a: + bhs-a: + sgp-a: diff --git a/ansible/inventory/group_vars/decdn_nodes.yml b/ansible/inventory/group_vars/decdn_nodes.yml index 05379d9..03f5d7b 100644 --- a/ansible/inventory/group_vars/decdn_nodes.yml +++ b/ansible/inventory/group_vars/decdn_nodes.yml @@ -4,12 +4,9 @@ # The sensitive rpc_url is provisioned as /etc/decdn/decdn.env on the host, or (if # you would rather the role authored that file) in a git-ignored secret.yml. -# Open the node's public QUIC listener in the baseline firewall. Metrics (9090) -# and admin RPC (9191) stay loopback and get NO hole. -baseline_extra_inbound: - - proto: udp - port: "{{ decdn_bind_port }}" - comment: "decdn QUIC" +# The node's public QUIC firewall hole (udp/4433) is NOT set here. It lives in +# playbooks/group_vars/decdn_nodes.yml so it applies to every inventory, including a +# private fleet overlay passed with INVENTORY=. # --- Grafana Cloud observability (opt-in) -------------------------------------- # One mirrored flag drives BOTH roles (grafana_alloy install/config AND the diff --git a/ansible/inventory/hosts.yml.example b/ansible/inventory/hosts.yml.example index 7f0fa0a..adcb89b 100644 --- a/ansible/inventory/hosts.yml.example +++ b/ansible/inventory/hosts.yml.example @@ -1,7 +1,8 @@ --- -# Copy to inventory/hosts.yml and fill in your hosts. This .example is the shipped -# template; your real hosts.yml is yours — commit it in your fork if you like, or keep -# it local. (Just never push real host IPs to the upstream template.) +# Copy to inventory/hosts.yml and fill in your hosts. inventory/hosts.yml is +# git-ignored: this repo is public, so real host IPs never belong in it. To track a +# real fleet, keep its inventory in a private overlay and pass INVENTORY= +# (template: inventory/fleet.example/, see README.md "Private fleet inventory"). # # No ansible_user by default: Ansible connects as your local $USER, which is the admin # account the baseline role creates (the auto-detected runner). A fresh box doesn't have diff --git a/ansible/playbooks/group_vars/decdn_nodes.yml b/ansible/playbooks/group_vars/decdn_nodes.yml new file mode 100644 index 0000000..68b95e8 --- /dev/null +++ b/ansible/playbooks/group_vars/decdn_nodes.yml @@ -0,0 +1,17 @@ +--- +# Playbook-adjacent group_vars for decdn_nodes: Ansible loads these for EVERY +# inventory passed with -i / INVENTORY=, unlike inventory/group_vars/, which only +# loads alongside inventory/hosts.yml. Only settings that every node needs no matter +# which inventory drives the run belong here. A private fleet overlay (see +# ansible/README.md "Private fleet inventory") must not be able to lose them. +# Operator knobs go in the inventory's own group_vars / host_vars. +# +# Precedence: playbook group_vars beat inventory group_vars, so to change this +# for a node, override it in that node's host_vars. + +# Open the node's public QUIC listener in the baseline firewall. Metrics (9090) +# and admin RPC (9191) stay loopback and get NO hole. +baseline_extra_inbound: + - proto: udp + port: "{{ decdn_bind_port }}" + comment: "decdn QUIC"