feat(ansible): prepare the launch fleet (private overlay, catalogue sizing, runbook)

AGENTS.md

@@ -140,10 +140,19 @@ make security         # = security-ansible + security-helm (KICS over the render
 cd ansible
 make deps             # vendor pinned Galaxy collections into ./collections
 make check / deploy   # deCDN node (site.yml): dry-run / provision
-                      # fleet-wide by default; LIMIT=<host> scopes, ANSIBLE_ARGS='…' passes through
+                      # fleet-wide by default; LIMIT=<host> scopes, ANSIBLE_ARGS='…' passes through;
+                      # INVENTORY=<path> targets a private fleet overlay (default inventory/hosts.yml)
 make build / galaxy-check       # stage + build the decdn.node collection, then validate it
 ```
 
+**Inventory is private; the firewall hole is not.** This repo is public, so
+`ansible/inventory/hosts.yml` is git-ignored and a real fleet lives in a private overlay
+(template: `ansible/inventory/fleet.example/`, launch sequence in
+`ansible/docs/launch-runbook.md`). Inventory-adjacent group_vars do not load for an
+overlay, so anything every node needs regardless of inventory (today only the udp/4433
+`baseline_extra_inbound` hole) lives in `ansible/playbooks/group_vars/decdn_nodes.yml`.
+Don't move it back under `inventory/`.
+
 **Galaxy collection (`decdn.node`).** The three roles (`baseline` + `decdn_node` +
 `grafana_alloy`) ship as a
 distributable collection. The overlay lives in `ansible/galaxy/` and is staged into a clean

SECURITY.md

@@ -0,0 +1,39 @@
+# Security
+
+## Reporting a vulnerability
+
+Email `security@decdn.org`. Please do not open a public issue for security
+reports. That includes a secret or real host address you find committed here.
+
+## Scope
+
+This repo holds the deployment tooling for a deCDN node: the Ansible roles, the
+`decdn.node` Galaxy collection, and the Helm chart. Reports about the node daemon or
+protocol belong to [decdn/decdn](https://github.com/decdn/decdn), but the same address
+reaches both.
+
+The security model these tools follow is described in [README.md § Security
+model](README.md#security-model): nothing secret committed, loopback-only backends,
+default-deny inbound, DevSec host hardening.
+
+## Release verification
+
+In `release` install mode, and while `decdn_verify_release_signature` keeps its
+default of `true`, the `decdn_node` role checks every downloaded tarball against the
+release's GPG-signed `SHA256SUMS`, using the maintainer key
+vendored at `ansible/roles/decdn_node/files/decdn-release-KEYS.asc`. That key is a
+copy of upstream's `KEYS`:
+
+```
+Ant Somers <ant@decdn.org>
+Fingerprint: DA75 1570 6F18 73D2 74D8  A369 9E11 A9FF D62D AADB
+```
+
+The same fingerprint is published in
+[decdn/decdn's SECURITY.md](https://github.com/decdn/decdn/blob/main/SECURITY.md).
+Check a new copy of the vendored key against it before trusting that copy, with
+`gpg --show-keys --with-fingerprint <file>`.
+
+Setting `decdn_verify_release_signature: false` (meant for an air-gapped mirror that
+strips signatures) drops that guarantee, and the role prints a warning when it's off.
+`manual` mode verifies nothing: it installs whatever binaries you point it at.

ansible/.gitignore

@@ -5,9 +5,19 @@ collections/
 # copy named secret.yaml or secret.json (Ansible loads .yml/.yaml/.json all the same)
 # can't leak; the committed secret.*.example template is re-included. Everything else in
 # host_vars — main.yml (contract addresses, region, cache origin, …) — is committable
-# template config. hosts.yml is committable too (your fork's inventory is yours to track).
+# template config.
 inventory/host_vars/*/secret.*
 !inventory/host_vars/*/secret.*.example
+# Same rule for inventory templates nested one level down (inventory/fleet.example/).
+inventory/*/host_vars/*/secret.*
+!inventory/*/host_vars/*/secret.*.example
+
+# The real inventory. This repo is PUBLIC, so real host IPs must never land here
+# (Launch/release-checklist.md §5). Copy hosts.yml.example to hosts.yml for a local
+# run, or keep a fleet's inventory in a private overlay and pass INVENTORY=<path>
+# (see README.md "Private fleet inventory"). The .example templates stay tracked.
+inventory/hosts.yml
+inventory/hosts.yaml
 
 # Ansible scratch
 *.retry

ansible/Makefile

@@ -33,6 +33,20 @@ LIMIT ?=
 ANSIBLE_ARGS ?=
 limit_arg = $(if $(LIMIT),--limit '$(LIMIT)')
 
+# Which inventory check/deploy run against. The default is the git-ignored
+# inventory/hosts.yml (copy hosts.yml.example). A real fleet keeps its inventory in a
+# private overlay outside this public repo and points at it here:
+#   make deploy INVENTORY=../../decdn-fleet/hosts.yml LIMIT=<host>
+# The overlay's own group_vars/ and host_vars/ must sit beside that hosts.yml, where
+# Ansible loads them automatically. See ansible/README.md "Private fleet inventory".
+INVENTORY ?= inventory/hosts.yml
+# A missing or unparseable inventory fails the run. Ansible's default only warns,
+# skips the `hosts: decdn_nodes` play and exits 0, so a mistyped INVENTORY= would
+# report a successful deploy that did nothing. This applies to check/deploy only:
+# ansible.cfg can't set it globally, because ansible-lint's own syntax checks run
+# with no inventory present.
+strict_inventory = ANSIBLE_INVENTORY_UNPARSED_FAILED=True
+
 # Fail loud on the two ways a run can look scoped but silently be fleet-wide. Scoped to
 # check/deploy via MAKECMDGOALS so an unrelated `make lint` is never blocked. NOTE: this
 # does not catch an argument that is well-formed but selects nothing — `--tags` with a
@@ -47,20 +61,26 @@ ifneq ($(filter check deploy,$(MAKECMDGOALS)),)
   ifeq ($(origin ANSIBLE_ARGS),environment)
     $(error ANSIBLE_ARGS is set in your environment and would silently alter this run. Unset it or pass it on the command line)
   endif
+  ifeq ($(origin INVENTORY),environment)
+    $(error INVENTORY is set in your environment ($(INVENTORY)) and would silently retarget this run. Unset it or pass it on the command line: make $(scoped_goal) INVENTORY=<path>)
+  endif
+  ifeq ($(strip $(INVENTORY)),)
+    $(error INVENTORY expanded empty (unset shell variable?) - refusing to guess which fleet to target)
+  endif
   ifeq ($(origin LIMIT),command line)
     ifeq ($(strip $(LIMIT)),)
       $(error LIMIT was supplied but expanded empty (unset shell variable?) - refusing to run fleet-wide by accident. Omit LIMIT entirely to mean "all hosts")
     endif
   endif
 endif
 
-# Dry run against the real inventory — shows diffs, mutates nothing.
+# Dry run against $(INVENTORY) — shows diffs, mutates nothing.
 check:
-	ansible-playbook playbooks/site.yml --check --diff $(limit_arg) $(ANSIBLE_ARGS)
+	$(strict_inventory) ansible-playbook -i '$(INVENTORY)' playbooks/site.yml --check --diff $(limit_arg) $(ANSIBLE_ARGS)
 
-# Provision the deCDN node host(s) in inventory/hosts.yml.
+# Provision the deCDN node host(s) in $(INVENTORY) (default inventory/hosts.yml).
 deploy:
-	ansible-playbook playbooks/site.yml $(limit_arg) $(ANSIBLE_ARGS)
+	$(strict_inventory) ansible-playbook -i '$(INVENTORY)' playbooks/site.yml $(limit_arg) $(ANSIBLE_ARGS)
 
 # --- Tests -------------------------------------------------------------------
 # Containerised converge + idempotence + verify of the decdn_node role against a

ansible/README.md

@@ -79,8 +79,33 @@ when the two would collide: an inventory value that would discard a host-side ed
 empty `decdn_rpc_url` that turns out to mean "`secret.yml` went missing" rather than "the host
 owns this file".
 
-Your `hosts.yml` is no longer force-ignored — commit it in your fork if you want, or keep
-it local.
+`inventory/hosts.yml` is git-ignored. This repo is public, and real host IPs never belong in
+it.
+
+### Private fleet inventory
+
+To track a real fleet, keep its inventory in a **private overlay** outside this repo and
+point the deploy targets at it:
+
+```bash
+cp -r inventory/fleet.example ../../decdn-fleet     # into a PRIVATE repo/dir, then fill it in
+make check  INVENTORY=../../decdn-fleet/hosts.yml LIMIT=<host>
+make deploy INVENTORY=../../decdn-fleet/hosts.yml LIMIT=<host>
+```
+
+Ansible loads `group_vars/` and `host_vars/` from beside whichever `hosts.yml` you pass, so
+the overlay carries its own copies. The template's own `.gitignore` comes along too, so the
+private repo never tracks a `host_vars/*/secret.yml`. The public `inventory/group_vars/` and
+`inventory/host_vars/` are **not** loaded for it. Settings every node needs regardless of
+inventory, currently only the udp/4433 QUIC firewall hole, live in
+`playbooks/group_vars/decdn_nodes.yml`, so an overlay can't drop them. Override that per node
+in `host_vars` if you have to. Playbook group_vars beat inventory group_vars.
+
+[`inventory/fleet.example/`](inventory/fleet.example/hosts.yml) is the launch-fleet
+template. Hosts are grouped by role (`decdn_seed` holds the catalogue in an fs origin,
+`decdn_edge` pulls through) and by billing (`decdn_metered` gets an egress budget,
+`decdn_unmetered` does not). The group_vars size the cache for large model blobs. The
+launch sequence is in [`docs/launch-runbook.md`](docs/launch-runbook.md).
 
 By default baseline **deploys you as yourself**: the runner (your control-machine `$USER` +
 its autodetected `~/.ssh` key, `id_ed25519` > `ecdsa` > `rsa`) is prepended as the head of

ansible/docs/launch-runbook.md

@@ -0,0 +1,110 @@
+# Launch fleet runbook
+
+The ops sequence for putting the launch fleet up with this project. The plan it
+serves, with targets, owners and dates, is `Launch/fleet-plan.md` in the private
+`decdn/internal` repo. This file records only the mechanics. Protocol and economic
+facts (bond sizing, fees) come from the deCDN ADRs; nothing here restates them.
+
+## 0. Inventory
+
+1. Copy [`inventory/fleet.example/`](../inventory/fleet.example/hosts.yml) into a
+   **private** repo or dir (see [README § Private fleet inventory](../README.md#private-fleet-inventory)).
+   Never commit real IPs to this public repo.
+2. For each host set `ansible_host` and `decdn_region`, then put it in exactly one
+   **role** group (`decdn_seed` / `decdn_edge`) and one **billing** group
+   (`decdn_metered` / `decdn_unmetered`). Hero and wow-factor traffic belongs on
+   unmetered nodes.
+3. In `group_vars/decdn_nodes.yml`, fill the MUST-EDIT lines: binary source and
+   contract addresses copied verbatim from `decdn/contracts/deployments/421614.json`.
+   Size `decdn_cache_size_mb` to each host's disk.
+4. Provision each node's secret on the host, the preferred way (`0600 /etc/decdn/decdn.env`,
+   [roles/decdn_node/README.md § Secrets](../roles/decdn_node/README.md#secrets)).
+   Provision the keystore too, or set `decdn_node_generate_keystore: true`.
+5. For every metered node, set a **billing alert** in the provider console (Hetzner /
+   OVHcloud). `decdn_load_shed_egress_budget_mbps` caps throughput, and the alert makes
+   the spend visible.
+
+## 1. Deploy
+
+```bash
+cd ansible
+make deps
+INV=../../decdn-fleet/hosts.yml                               # your private overlay
+make check  INVENTORY=$INV LIMIT=<host> ANSIBLE_ARGS='-u root'   # first converge: bootstrap user
+make deploy INVENTORY=$INV LIMIT=<host> ANSIBLE_ARGS='-u root'
+make deploy INVENTORY=$INV                                    # every run after bootstrap, fleet-wide
+```
+
+The role runs `decdn config validate` against the installed binary after templating,
+so a key the binary doesn't know fails the deploy instead of crash-looping the node.
+
+## 2. Stake and register each node (manual)
+
+On-chain onboarding ([ADR 019](https://github.com/decdn/decdn/blob/main/adr/019-node-onboarding.md)
+Phase 2) is an operator step, not automated here. Fund the node's eth address first
+(`key-gen` printed it). Then run `decdn setup` on the node. It reads the rendered
+config plus `DECDN_RPC_URL` from `decdn.env`, and `systemd-run` loads that file with
+the same parser the node's unit uses, so the RPC key never lands in argv:
+
+```bash
+# Preview first: pre-flight checks and the bond it would post, no transactions.
+sudo systemd-run --pty --wait --collect -p User=decdn \
+  -p EnvironmentFile=/etc/decdn/decdn.env \
+  /usr/local/bin/decdn --config /etc/decdn/node.toml setup \
+  --mbps <declared-capacity> --region <same code as decdn_region> \
+  --multiaddr /ip4/<public-ip>/udp/4433/quic-v1 \
+  --keystore-password-file /etc/decdn/keystore.password --dry-run
+# Then the same command without --dry-run. It shows the operator terms and the bond to confirm.
+```
+
+- `--region` is the on-chain `regionHint` ([ADR 030](https://github.com/decdn/decdn/blob/main/adr/030-node-region-self-attestation.md)).
+  It must match the host's `decdn_region`, since the stats map and takedown scope read it.
+- The bond is read from the on-chain `bondRequired(mbps)` curve
+  ([ADR 026](https://github.com/decdn/decdn/blob/main/adr/026-tokenomics.md)). Declare
+  `--mbps` for what the node can really serve; there's no flat minimum to quote.
+- The flags, the `node bond` / `node register` primitives and the exit path are in
+  [roles/decdn_node/README.md § On-chain onboarding](../roles/decdn_node/README.md#on-chain-onboarding).
+
+A node counts toward launch metrics only once it is registered and serving.
+
+## 3. Seed the catalogue (seed nodes)
+
+On each `decdn_seed` host, run `Launch/seed-model.sh` (from `decdn/internal`) as the
+service user, writing into the fs origin that group_vars configure:
+
+```bash
+sudo -u decdn OUT_DIR=/var/lib/decdn/origin ./seed-model.sh
+```
+
+It needs the `hf` CLI and disk for **both** the origin copy and the cache, on the same
+`/var/lib/decdn` volume. With the template values that's about 352 GiB of origin (the
+current whole-repo set, measured 2026-09-22), plus `decdn_cache_size_mb` (400 GiB), plus
+`decdn_disk_headroom_mb` (8 GiB): **about 760 GiB per seed**. `hf download` also stages
+a full copy before import, so allow for the largest model (~177 GiB, Mixtral) on top
+while seeding, or lower the seeds' `decdn_cache_size_mb` in `group_vars/decdn_seed.yml`.
+Record every BLAKE3 hash it prints: the bundle manifests plus the large blobs.
+
+## 4. Pin and fill the edges
+
+1. Add the recorded hashes (lowercase 64-hex, no `b3:` prefix) to
+   `decdn_pinned_hashes` in the overlay's `group_vars/decdn_nodes.yml`, then
+   `make deploy INVENTORY=$INV`.
+2. A pin only stops eviction. Each edge still fetches a blob on its first miss, and
+   it's launch-ready once the blob is resident on enough nodes per region.
+
+## 5. Verify: the fleet part of the go/no-go
+
+- From a cold machine near each region, pull the **largest** blob (currently the
+  13.5 GiB Mistral v0.3 `consolidated.safetensors`) by hash. A `BlobTooLarge` error
+  means that node's `decdn_max_blob_size_mb` is too small.
+- `curl -s 127.0.0.1:9090/metrics` on a node (over SSH, since metrics are loopback)
+  shows cache hits and misses. With `decdn_grafana_cloud_enabled`, watch egress and
+  errors in Grafana Cloud.
+- Every node shows up registered, with its region, on the explorer / stats map.
+
+## 6. When `v0.1.0` is tagged
+
+Switch the overlay to the signed tarballs (`decdn_node_install_method: release`,
+`decdn_node_version: "0.1.0"`) and redeploy. The role verifies them against the
+release's GPG-signed `SHA256SUMS`, using the vendored maintainer key. Re-check the config
+schema coupling (`molecule/schema`, `make lint-helm`) against the tagged `types.rs`.

ansible/inventory/fleet.example/.gitignore

@@ -0,0 +1,4 @@
+# Travels with `cp -r inventory/fleet.example <private-overlay>`, so the private
+# repo never tracks a per-node secret either (decdn_rpc_url may embed an API key).
+host_vars/*/secret.*
+!host_vars/*/secret.*.example

ansible/inventory/fleet.example/group_vars/decdn_edge.yml

@@ -0,0 +1,4 @@
+---
+# Edge nodes have no origin of their own. With no decdn_cache_origin_kind set, the
+# role defaults node-to-node pull-through ON, so an edge fills a cache miss from
+# a seed or peer that holds the blob.

ansible/inventory/fleet.example/group_vars/decdn_metered.yml

@@ -0,0 +1,9 @@
+---
+# Metered nodes bill per TB of egress, so a model that trends on r/LocalLLaMA
+# must not run up the bill (Launch/fleet-plan.md). Cap their egress.
+# This is OPERATOR COST CONFIG, not a protocol fact. Size it from the provider's
+# per-TB price and the monthly spend you accept, e.g. 250 Mbps sustained is about
+# 81 TB/month.
+decdn_load_shed_egress_budget_mbps: 250
+# Also set a billing alert in the provider console (Hetzner / OVHcloud) for every
+# metered node. The budget above caps throughput, the alert makes the spend visible.

ansible/inventory/fleet.example/group_vars/decdn_nodes.yml

@@ -0,0 +1,59 @@
+---
+# Settings shared by every launch node. Copied with the overlay; edit the private copy.
+
+# --- Binaries -----------------------------------------------------------------
+# Until upstream tags v0.1.0, install locally built binaries. Build them from a
+# decdn/decdn checkout on the release commit:
+#   cargo build --release -p decdn-node -p decdn-cli
+decdn_node_install_method: manual
+decdn_release_target_dir: ""        # MUST-EDIT - control-machine path to decdn's target/release
+# Once the v0.1.0 release exists (Launch/release-checklist.md §1), switch to the
+# GPG-verified tarballs and drop the two lines above:
+# decdn_node_install_method: release
+# decdn_node_version: "0.1.0"
+
+# --- Contracts (Arbitrum Sepolia, 421614) ----------------------------------------
+# Protocol facts. Copy them VERBATIM from the upstream deployment manifest
+# decdn/contracts/deployments/421614.json. The public repo's
+# inventory/host_vars/decdn-node-1/main.yml carries the current copy with its sync
+# note. Never guess. Upstream redeploys change every address at once.
+decdn_chain_id: 421614
+decdn_payment_pool_address: ""        # MUST-EDIT
+decdn_capacity_bond_address: ""       # MUST-EDIT
+decdn_slash_judge_address: ""         # MUST-EDIT
+decdn_content_blacklist_address: ""   # MUST-EDIT
+decdn_slash_appeal_address: ""
+decdn_origin_assignment_address: ""
+decdn_publisher_registry_address: ""
+decdn_usdc_address: ""
+
+# --- Cache sizing for the launch model catalogue ---------------------------------
+# `decdn origin import` stores every file as ONE whole-file blob (--optimize only
+# adds chunk hints to the manifest), and a node rejects any blob over
+# max_blob_size_mb with BlobTooLarge on pull-through. The role default (1024) is far
+# too small for model weights. Measured from the Hugging Face file trees on
+# 2026-09-22 for the seed-model.sh set (whole repos, as that script downloads them):
+#   largest single file: 13.5 GiB  Mistral-7B-Instruct-v0.3 consolidated.safetensors
+#                        12.8 GiB  gpt-oss-20b original/model.safetensors
+#                        11.3 GiB  Mixtral-8x7B consolidated.02.pt
+#                         9.6 GiB  SDXL unet (fp32)
+#   whole set: ~352 GiB (Mixtral alone ~177 GiB)
+# Re-measure if the seed list or its --exclude patterns change.
+decdn_max_blob_size_mb: 16384         # 16 GiB: largest file 13.5 GiB, plus headroom
+# Must hold the catalogue you expect a node to serve, plus room for LRU churn, and
+# must be >= max_blob_size_mb. The daemon also keeps decdn_disk_headroom_mb (default
+# 8 GiB) free on the volume. Size it to each host's disk: 400 GiB covers the whole
+# set above. Override per host in host_vars if disks differ.
+decdn_cache_size_mb: 409600           # 400 GiB
+
+# --- Pinning the catalogue --------------------------------------------------------
+# Eviction-exempt pins. Fill these after seeding: the lowercase 64-hex BLAKE3 hashes
+# `decdn origin import` prints for each model's bundle manifest AND its large blobs.
+# A pin only stops eviction. The node still has to fetch the blob once before it
+# holds it.
+decdn_pinned_hashes: []
+
+# --- Observability ----------------------------------------------------------------
+# Recommended for launch: live metrics + journald per node (egress, errors).
+# Provision the token first (roles/grafana_alloy/README.md).
+# decdn_grafana_cloud_enabled: true