From 6b6f669fa7ad2581491f9f93ec59c408bde440c3 Mon Sep 17 00:00:00 2001 From: Ant Somers Date: Thu, 24 Sep 2026 04:28:02 +0300 Subject: [PATCH 1/2] feat(cloud-init): user-data bootstrap for a no-Ansible single-VM node Add a fourth deploy path: paste cloud-init/user-data.yaml into a VPS provider's "create server" form, and a fresh Debian 12/13 or Ubuntu 24.04/26.04 host runs this repo's site.yml against localhost. It reuses the baseline and decdn_node roles unchanged. - Stage 1 (embedded in the user-data) clones the repo at a pinned ref, checks a full SHA after checkout, and execs cloud-init/bootstrap.sh. It reads bootstrap.env as literal KEY=value lines and never sources it. - Stage 2 installs ansible-core from a hash-locked requirements.txt (2.19 for Python 3.11, 2.21 for 3.12+) and the exact collection set in collections.lock.yml. Without /etc/decdn/decdn.env it runs baseline only and records "awaiting-secret"; once the operator writes the file over SSH, `sudo decdn-bootstrap` runs the full release-mode install with a host-generated wallet. - No secrets in user-data: `make lint-cloud-init` runs cloud-init schema plus invariants (no secret-looking keys or URL credentials, release mode, keystore generated on the host, localhost in decdn_nodes, a keyed admin account, shellcheck-clean stage 1, a lock that covers ansible/requirements.yml), with negative cases in tests/scripts-test.sh. - The molecule `cloud-init` scenario boots the real user-data through cloud-init on Debian 12 and Ubuntu 26.04 against a locally signed release mirror, from the secret gate to a running node. It is the suite's first release-mode coverage. Baseline is skipped in containers, as in the other scenarios. - Docs: cloud-init/README.md, the deploy-path chooser, README, AGENTS, CONTRIBUTING, SECURITY and the issue templates. Closes #72 Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/ISSUE_TEMPLATE/bug_report.yml | 5 +- .github/ISSUE_TEMPLATE/feature_request.yml | 1 + .github/dependabot.yml | 10 +- .github/workflows/ci.yml | 40 ++- .github/workflows/molecule.yml | 8 +- .pre-commit-config.yaml | 9 +- AGENTS.md | 36 ++- CONTRIBUTING.md | 19 +- Makefile | 15 +- README.md | 6 +- SECURITY.md | 2 +- ansible/molecule/cloud-init/converge.yml | 208 ++++++++++++ ansible/molecule/cloud-init/molecule.yml | 72 +++++ ansible/molecule/cloud-init/prepare.yml | 157 +++++++++ ansible/molecule/cloud-init/side_effect.yml | 35 ++ ansible/molecule/cloud-init/verify.yml | 100 ++++++ cloud-init/README.md | 206 ++++++++++++ cloud-init/bootstrap.sh | 128 ++++++++ cloud-init/collections.lock.yml | 33 ++ cloud-init/requirements.in | 11 + cloud-init/requirements.txt | 342 ++++++++++++++++++++ cloud-init/tests/lint.py | 210 ++++++++++++ cloud-init/user-data.yaml | 134 ++++++++ compose/README.md | 2 +- docs/requirements.md | 29 +- tests/scripts-test.sh | 40 ++- 26 files changed, 1810 insertions(+), 48 deletions(-) create mode 100644 ansible/molecule/cloud-init/converge.yml create mode 100644 ansible/molecule/cloud-init/molecule.yml create mode 100644 ansible/molecule/cloud-init/prepare.yml create mode 100644 ansible/molecule/cloud-init/side_effect.yml create mode 100644 ansible/molecule/cloud-init/verify.yml create mode 100644 cloud-init/README.md create mode 100755 cloud-init/bootstrap.sh create mode 100644 cloud-init/collections.lock.yml create mode 100644 cloud-init/requirements.in create mode 100644 cloud-init/requirements.txt create mode 100755 cloud-init/tests/lint.py create mode 100644 cloud-init/user-data.yaml diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 6d93b09..15a07e1 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -1,6 +1,6 @@ --- name: Bug report -description: A deploy path (Ansible, Helm, Compose) does something wrong. +description: A deploy path (Ansible, cloud-init, Helm, Compose) does something wrong. labels: [bug] body: - type: markdown @@ -16,6 +16,7 @@ body: options: - Ansible (this repo's playbooks) - Ansible (decdn.node Galaxy collection) + - cloud-init user-data - Helm chart - Docker Compose validations: @@ -24,7 +25,7 @@ body: id: install attributes: label: Install method - description: Ansible only. + description: Ansible only (cloud-init is always release). options: - release (signed tarball) - manual (locally built binaries) diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml index b776b25..ff4b313 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yml +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -21,6 +21,7 @@ body: multiple: true options: - Ansible + - cloud-init - Helm chart - Docker Compose - All / not specific diff --git a/.github/dependabot.yml b/.github/dependabot.yml index c4b1195..d6ff708 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -10,14 +10,20 @@ # ${DECDN_IMAGE_DIGEST}, which the operator pins to a signed # release digest; nothing to bump # helm the chart has no dependencies, and its image is operator-set -# pip the repo has no pip manifest (CI installs are unpinned) +# pip CI installs are unpinned. The one pip manifest, +# cloud-init/requirements.txt, pins ansible-core twice with Python +# markers (2.19 for 3.11, 2.21 for 3.12+) and is compiled with uv; +# an automated bump could move the 3.11 pin to a release that +# drops 3.11 # Ansible Galaxy (ansible/requirements.yml) is not a Dependabot ecosystem. # # NOT covered, bump manually (CONTRIBUTING.md § Supply-chain / pinning rules): # - the KICS and kubeconform image digests in the Makefile # - the molecule image digests in ansible/molecule/*/molecule.yml # - the four setup-helm `version:` inputs (two in ci.yml, two in release.yml) -# - the Galaxy collections in ansible/requirements.yml +# - the Galaxy collections in ansible/requirements.yml, and their exact pins in +# cloud-init/collections.lock.yml +# - ansible-core in cloud-init/requirements.in (then recompile requirements.txt) # - the local yamllint hook's `additional_dependencies` pin version: 2 updates: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aeeeeda..c575b00 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,8 +18,8 @@ concurrency: # malicious code — cf. the March 2026 KICS action compromise). The trailing # comment records the human-readable version; .github/dependabot.yml bumps them. jobs: - # Detect whether ansible/, the Helm chart or compose/ changed so heavy jobs skip - # unrelated PRs. + # Detect whether ansible/, the Helm chart, compose/ or cloud-init/ changed so heavy + # jobs skip unrelated PRs. changes: runs-on: ubuntu-latest timeout-minutes: 5 # ~5s in practice @@ -30,6 +30,7 @@ jobs: ansible: ${{ steps.filter.outputs.ansible }} helm: ${{ steps.filter.outputs.helm }} compose: ${{ steps.filter.outputs.compose }} + cloud-init: ${{ steps.filter.outputs.cloud-init }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4.0.3 @@ -52,6 +53,13 @@ jobs: - 'compose/**' - 'Makefile' - '.github/workflows/ci.yml' + # lint-cloud-init lives in the root Makefile and reads + # ansible/requirements.yml (the collection lock must cover it). + cloud-init: + - 'cloud-init/**' + - 'ansible/requirements.yml' + - 'Makefile' + - '.github/workflows/ci.yml' # Ansible style + best-practice + the production-profile SECURITY rules, # plus a syntax-check of every playbook. Runs only when ansible/ changed. @@ -200,6 +208,24 @@ jobs: # DECDN_CLI=... before bumping the decdn version. run: make lint-helm + # cloud-init path: `cloud-init schema` plus the invariants cloud-init/README.md + # promises (no secrets, release install, host-generated wallet, localhost in + # decdn_nodes, a keyed admin account, shellcheck-clean stage 1, a collection lock + # covering ansible/requirements.yml). The boot itself is the molecule `cloud-init` + # scenario (molecule.yml). Runs only when cloud-init/ or the root Makefile changed. + cloud-init: + needs: changes + if: needs.changes.outputs.cloud-init == 'true' + runs-on: ubuntu-latest + timeout-minutes: 10 # an apt install at most; seconds otherwise + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + # yq and shellcheck ship on ubuntu-latest; cloud-init may not. + - name: Install cloud-init + run: command -v cloud-init || { sudo apt-get update -qq && sudo apt-get install -y -qq cloud-init; } + - name: cloud-init invariants + run: make lint-cloud-init + # Docker Compose path: render compose/compose.yaml with its example env files and # assert the invariants its README promises (host network, no published ports, # digest-pinned image, read-only rootfs, no capabilities, graceful stop). Runs @@ -343,13 +369,17 @@ jobs: run: pre-commit run --all-files --show-diff-on-failure # The repo's own guard rails: ansible/Makefile's scoping guards (dry runs), the - # release gate, and lint-compose's negative cases (tests/scripts-test.sh). Always - # runs: seconds, and it covers files every path filter above would miss. + # release gate, and the lint-compose and lint-cloud-init negative cases + # (tests/scripts-test.sh). Always runs: seconds, and it covers files every path + # filter above would miss. scripts: runs-on: ubuntu-latest - timeout-minutes: 5 + timeout-minutes: 10 # an apt install at most; seconds otherwise steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + # Without cloud-init on PATH the harness SKIPS the lint-cloud-init cases. + - name: Install cloud-init + run: command -v cloud-init || { sudo apt-get update -qq && sudo apt-get install -y -qq cloud-init; } - name: Script and guard tests run: make test-scripts diff --git a/.github/workflows/molecule.yml b/.github/workflows/molecule.yml index b462892..9e5fe48 100644 --- a/.github/workflows/molecule.yml +++ b/.github/workflows/molecule.yml @@ -5,13 +5,15 @@ name: Molecule # Heavy (privileged systemd Docker container) — scoped to ansible/ changes and # blocking. Mark it a required status check in branch protection once proven. # This file is in `paths` alongside ansible/ so a change to the job itself (its cache -# wiring, JOBS, the timeout) is exercised by the PR that makes it. +# wiring, JOBS, the timeout) is exercised by the PR that makes it. cloud-init/ is in +# `paths` because the `cloud-init` scenario boots cloud-init/user-data.yaml and runs +# cloud-init/bootstrap.sh. on: pull_request: - paths: ['ansible/**', '.github/workflows/molecule.yml'] + paths: ['ansible/**', 'cloud-init/**', '.github/workflows/molecule.yml'] push: branches: [main] - paths: ['ansible/**', '.github/workflows/molecule.yml'] + paths: ['ansible/**', 'cloud-init/**', '.github/workflows/molecule.yml'] permissions: contents: read diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ddfd5fd..abaffe3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -58,17 +58,18 @@ repos: hooks: - id: markdownlint-cli2 - # ── YAML lint for the Ansible tree (reuses ansible/.yamllint) ────────────── - # Scoped to ansible/ so it stays in lockstep with `make -C ansible lint`. + # ── YAML lint for the Ansible tree and cloud-init/ (reuses ansible/.yamllint) ─ + # Scoped so it stays in lockstep with `make -C ansible lint`. cloud-init/ is the + # Ansible project's user-data and collection lock, so it takes the same rules. # *.yml.example templates don't match (they end in .example) and are skipped. - repo: local hooks: - id: yamllint-ansible - name: yamllint (ansible/) + name: yamllint (ansible/, cloud-init/) entry: yamllint -c ansible/.yamllint language: python additional_dependencies: ["yamllint==1.35.1"] - files: ^ansible/.*\.(ya?ml)$ + files: ^(ansible|cloud-init)/.*\.(ya?ml)$ exclude: ^ansible/(collections|\.ansible)/ # Opt-in: full ansible-lint locally (needs `make -C ansible deps` first). diff --git a/AGENTS.md b/AGENTS.md index 1da8253..c0227b3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -6,10 +6,11 @@ DevOps repo. ## What this repo is The official DevOps project for deploying a **deCDN node**: infrastructure, deployment, -and operational tooling, for node operators anywhere. There are three deploy paths: +and operational tooling, for node operators anywhere. There are four deploy paths: **Ansible** (`ansible/`, VMs/bare metal, the primary path, also the `decdn.node` Galaxy -collection), **Docker Compose** (`compose/`, a single Docker host) and a **Helm chart** -(`charts/decdn-node/`, Kubernetes). +collection), **cloud-init** (`cloud-init/`, one VM that runs the Ansible playbook on +itself, no control machine), **Docker Compose** (`compose/`, a single Docker host) and a +**Helm chart** (`charts/decdn-node/`, Kubernetes). This repo is **infrastructure only**. It is *not* a source of truth for protocol or economic claims — those trace to the deCDN ADRs. If something here states a protocol fact @@ -56,6 +57,7 @@ ansible/ # the deployment project (DevSec-hardened, lean roles) playbooks/ # site.yml (decdn node), backup.yml, decommission.yml roles/ # baseline, decdn_node, grafana_alloy inventory/ galaxy/ molecule/ # see ansible/README.md +cloud-init/ # user-data.yaml + on-host bootstrap.sh; pinned ansible-core/collections (see its README.md) compose/ # Docker Compose deploy path for a single host (see its README.md) charts/ decdn-node/ # Helm chart for the node on Kubernetes (see its README.md) @@ -123,6 +125,26 @@ addresses) the repo carries, and they carry their upstream commit. everything, so only the real pinned binary proves the rendered config loads. See `ansible/roles/grafana_alloy/README.md`. +- **`cloud-init/`** — the Ansible path with no control machine. `user-data.yaml` carries + only public material (the lint refuses secret-looking keys, credentials in URLs and + unknown `bootstrap.env` keys) and a stage-1 `decdn-bootstrap`. That script clones this + repo at a pinned ref (a full SHA is verified after checkout) and execs + `cloud-init/bootstrap.sh`, which: + - installs ansible-core from the hash-locked `requirements.txt` into a venv (two pins + split by Python marker: 2.19 for Debian 12's 3.11, 2.21 for 3.12 and later); + - installs the exact collections from `collections.lock.yml`; + - runs `site.yml` against localhost. The inventory must put localhost in + `decdn_nodes`, or the play matches nothing and the udp/4433 hole never loads. + + With no `/etc/decdn/decdn.env`, it runs `--tags baseline` only and records + `awaiting-secret`. The operator writes the file over SSH and re-runs `decdn-bootstrap` + for the full playbook (`release` install, host-generated wallet). The roles are used + unchanged, so a role change reaches this path without edits here. When + `ansible/requirements.yml` changes, re-sync the lock: `make lint-cloud-init` checks + it covers the requirements. The molecule `cloud-init` scenario boots the real + user-data through cloud-init (skipping `baseline`) against a locally signed release + mirror, and is the suite's only release-mode coverage. + - **`compose/`** — the same node under Docker Compose on one host: the upstream image, always by digest (`compose.yaml` builds `DECDN_IMAGE_REPO@DECDN_IMAGE_DIGEST`), the role's host layout (`/etc/decdn` read-only, `/var/lib/decdn`), host @@ -161,7 +183,8 @@ make molecule # every ansible/molecule/*/ scenario in parallel (Docker; make lint-helm # chart: lint + render tests + kubeconform + schema keys make lint-alloy # grafana_alloy config against the real pinned Alloy binary make lint-compose # compose/ invariants -make test-scripts # Makefile guards, release gate, lint-compose negatives +make lint-cloud-init # cloud-init/user-data.yaml: schema + invariants (no secrets, release mode, lock) +make test-scripts # Makefile guards, release gate, lint-compose/lint-cloud-init negatives make security # KICS over ansible/, the rendered chart and compose/ # Ansible — run from ansible/ @@ -189,6 +212,7 @@ repository variable is `true` (RELEASING.md). Log changes under `[Unreleased]` i `ansible/galaxy/CHANGELOG.md` and `charts/decdn-node/CHANGELOG.md`. **CI.** `ci.yml` is the blocking gate: `pre-commit`, `scripts` and `actionlint` on every PR, the -Ansible, chart and compose jobs path-filtered, KICS on any of them; `molecule.yml` runs the -molecule suite on `ansible/**`. `ansible-lint` is **not** a per-commit hook (it needs +Ansible, chart, compose and cloud-init jobs path-filtered, KICS on the first three (KICS has +no cloud-init platform); `molecule.yml` runs the molecule suite on `ansible/**` and +`cloud-init/**`. `ansible-lint` is **not** a per-commit hook (it needs collections vendored): run `make lint-ansible`. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f3f7be5..64c4ba7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -33,7 +33,8 @@ targets, so a local pass means a CI pass. Deploy targets live in | `make lint-helm` | chart: `helm lint --strict`, positive/negative render tests, kubeconform (digest-pinned image), the shared schema-key check (needs `helm`, `yq`, `python3` ≥ 3.11, Docker). Set `DECDN_CLI=` to also run the real `decdn config validate` (CI can't). | | `make lint-alloy` | renders `roles/grafana_alloy`'s templates and validates them with the **real** digest-pinned Alloy binary. The molecule stub exits 0 for everything, so this is the only gate that proves the config loads. `ALLOY_BIN=` skips the download. | | `make lint-compose` | renders `compose/compose.yaml` with its example env and asserts its security invariants | -| `make test-scripts` | `tests/scripts-test.sh`: the `ansible/Makefile` scoping guards (dry runs), the release gate, and `lint-compose`'s negative cases. `UPSTREAM=` adds the sync generators' exit codes. | +| `make lint-cloud-init` | `cloud-init schema` on `cloud-init/user-data.yaml`, then `cloud-init/tests/lint.py`: no secrets, `release` install with a host-generated wallet, localhost in `decdn_nodes`, a keyed admin account, a shellcheck-clean stage 1, and a collection lock that covers `ansible/requirements.yml` (needs `cloud-init`, `shellcheck`, `yq`). `CLOUD_INIT_FILE=` checks your own filled-in copy. | +| `make test-scripts` | `tests/scripts-test.sh`: the `ansible/Makefile` scoping guards (dry runs), the release gate, and the negative cases of `lint-compose` and `lint-cloud-init` (the latter skipped without `cloud-init` on PATH). `UPSTREAM=` adds the sync generators' exit codes. | | `make security` | KICS IaC scan of `ansible/`, the rendered chart and `compose/` (digest-pinned engine, fail on HIGH) | | `make galaxy-check` | build the `decdn.node` collection and run galaxy-importer's checks | @@ -66,8 +67,13 @@ reports the two differently. - on `charts/**` (or the shared schema files, the root `Makefile`, `ci.yml`): `helm` (`make lint-helm`); - on `compose/**` (or the root `Makefile`, `ci.yml`): `compose` (`make lint-compose`); - - on any of those: `kics` (`make security`). -- **`molecule.yml`**: `make molecule JOBS=3` on `ansible/**` changes. + - on `cloud-init/**` (or `ansible/requirements.yml`, the root `Makefile`, `ci.yml`): + `cloud-init` (`make lint-cloud-init`); + - on the Ansible, chart or compose paths: `kics` (`make security`). KICS has no + cloud-init platform. +- **`molecule.yml`**: `make molecule JOBS=3` on `ansible/**` or `cloud-init/**` changes. + The `cloud-init` scenario boots the real user-data, so it needs network access to apt, + PyPI and Galaxy. - **`release.yml`**: on `vX.Y.Z` tags; see [RELEASING.md](RELEASING.md). - **`upstream-drift.yml`**: weekly, non-blocking; see "Upstream mirrors" above. - **Every job is bounded** by `timeout-minutes`. The values are bounds sized off @@ -103,8 +109,11 @@ reports the two differently. `KUBECONFORM_IMAGE` digests in the `Makefile`; the molecule image digests in `ansible/molecule/*/molecule.yml` (all together, `docker buildx imagetools inspect`); the four `setup-helm` `version:` inputs (`ci.yml`'s `helm` and `kics` jobs, both - jobs in `release.yml`); the collection versions in `ansible/requirements.yml`; and the local - yamllint hook's `additional_dependencies` pin. + jobs in `release.yml`); the collection versions in `ansible/requirements.yml`, and + their exact pins in `cloud-init/collections.lock.yml` (the full transitive set, from + a `make deps` resolve); ansible-core in `cloud-init/requirements.in`, followed by a + recompile of the hash-locked `requirements.txt` (command in `cloud-init/README.md`); + and the local yamllint hook's `additional_dependencies` pin. - **Bump the `cache-epoch:` counter in `ansible/requirements.yml` to make CI re-resolve the collections.** Those are `>=` ranges, so a warm cache pins the resolved set — transitive collections like `community.crypto` included — until the diff --git a/Makefile b/Makefile index ba33f91..9be7175 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ # Convenience targets for the deCDN DevOps monorepo. # Run from the repo root. Ansible-specific work is delegated to ansible/Makefile. -.PHONY: help hooks lint lint-ansible lint-helm lint-alloy lint-compose test-scripts security security-ansible security-helm security-compose molecule molecule-serial galaxy-build galaxy-check +.PHONY: help hooks lint lint-ansible lint-helm lint-alloy lint-compose lint-cloud-init test-scripts security security-ansible security-helm security-compose molecule molecule-serial galaxy-build galaxy-check SHELL := /bin/bash # KICS runs straight from the engine image, pinned by digest. This target IS the @@ -91,6 +91,19 @@ lint-compose: ## render compose/ with its examples and check its security || { echo "$(COMPOSE_FILE) violates an invariant (see the lint-compose comment in Makefile)" >&2; exit 1; } @echo "compose invariants hold" +# The cloud-init user-data (cloud-init/README.md): `cloud-init schema` for its shape, then +# cloud-init/tests/lint.py for what a schema cannot see. That covers no secrets, release +# install with a host-generated wallet, localhost in decdn_nodes, a keyed admin account, +# a shellcheck-clean stage 1, and a collection lock that covers ansible/requirements.yml. +# CLOUD_INIT_FILE is overridable so tests/scripts-test.sh can feed it broken variants. +CLOUD_INIT_FILE ?= cloud-init/user-data.yaml +lint-cloud-init: ## schema-check cloud-init/user-data.yaml and its invariants (needs cloud-init, shellcheck, yq) + @cloud-init schema -c '$(CLOUD_INIT_FILE)' >/dev/null 2>&1 \ + || { cloud-init schema -c '$(CLOUD_INIT_FILE)' 2>&1 | grep -v WARNING >&2; \ + echo "lint-cloud-init: $(CLOUD_INIT_FILE) is not a valid cloud-config (see above)" >&2; exit 2; } + @cloud-init/tests/lint.py '$(CLOUD_INIT_FILE)' + @echo "cloud-init invariants hold" + # The guard rails nothing else exercises: ansible/Makefile's scoping guards, the # release gate, lint-compose's negative cases, and (with UPSTREAM=) # the upstream-mirror generators' exit codes. CI job `scripts`. diff --git a/README.md b/README.md index bc9c4ae..d07c5d5 100644 --- a/README.md +++ b/README.md @@ -9,11 +9,12 @@ [![Conventional Commits](https://img.shields.io/badge/Conventional%20Commits-1.0.0-yellow.svg)](https://www.conventionalcommits.org) The official **DevOps repo** for running a deCDN node: infrastructure, deployment and -day-2 tooling for operators anywhere. Three ways to deploy the same node: +day-2 tooling for operators anywhere. Four ways to deploy the same node: | Path | For | Start here | |------|-----|------------| | **Ansible** (`ansible/`) | VMs and bare metal, one node or a fleet. Hardens the host too (firewall, SSH, patching). Also published as the `decdn.node` Galaxy collection. | [`ansible/README.md`](ansible/README.md) | +| **cloud-init** (`cloud-init/`) | One VM, no control machine: paste the user-data into your provider's "create server" form. It runs the Ansible playbook on the host itself, hardening included. | [`cloud-init/README.md`](cloud-init/README.md) | | **Docker Compose** (`compose/`) | One host that already runs Docker. | [`compose/README.md`](compose/README.md) | | **Helm** (`charts/decdn-node/`) | Kubernetes, one release per node. | [`charts/decdn-node/README.md`](charts/decdn-node/README.md) | @@ -64,7 +65,7 @@ make deploy LIMIT=decdn-node-1 # every run after that ``` The full flow (bootstrap user, keystore, secrets, fleets in a private inventory) is in -[`ansible/README.md`](ansible/README.md). Compose and Helm have their own quickstarts. +[`ansible/README.md`](ansible/README.md). cloud-init, Compose and Helm have their own quickstarts. ## Security model @@ -97,6 +98,7 @@ This is the canonical statement; the per-path READMEs add only what is specific | Path | What it is | |------|------------| | [`ansible/`](ansible/README.md) | The Ansible project: `inventory/`, `playbooks/` (`site.yml`, `backup.yml`, `decommission.yml`), `roles/` (`baseline`, `decdn_node`, `grafana_alloy`), `galaxy/` (the `decdn.node` collection), `molecule/`. | +| [`cloud-init/`](cloud-init/README.md) | The cloud-init deploy path: `user-data.yaml`, the on-host `bootstrap.sh`, and the pinned ansible-core and collections it installs. | | [`compose/`](compose/README.md) | The Docker Compose deploy path. | | [`charts/decdn-node/`](charts/decdn-node/README.md) | The Helm chart, with vendored dashboards and alert rules in `files/monitoring/`. | | [`docs/`](docs/requirements.md) | Cross-path operator docs: requirements, lifecycle. | diff --git a/SECURITY.md b/SECURITY.md index 89e46ab..920c380 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -8,7 +8,7 @@ reports. That includes a secret or real host address you find committed here. ## Scope This repo holds the deployment tooling for a deCDN node: the Ansible roles, the -`decdn.node` Galaxy collection, the Docker Compose file, and the Helm chart. Reports about the node daemon or +`decdn.node` Galaxy collection, the cloud-init user-data, the Docker Compose file, and the Helm chart. Reports about the node daemon or protocol belong to [decdn/decdn](https://github.com/decdn/decdn), but the same address reaches both. diff --git a/ansible/molecule/cloud-init/converge.yml b/ansible/molecule/cloud-init/converge.yml new file mode 100644 index 0000000..32ddf04 --- /dev/null +++ b/ansible/molecule/cloud-init/converge.yml @@ -0,0 +1,208 @@ +--- +# First boot. The seed is cloud-init/user-data.yaml as committed, with each MUST-EDIT +# filled in the way an operator would and the three outside dependencies redirected +# (repo, release mirror, signing key). Anything else in the file is exercised as +# written. If the file's placeholders change, the "no CHANGE_ME left" check below fails +# rather than letting this test drift into booting something else. +# +# It then runs cloud-init's own boot stages against that seed, and asserts the host +# stopped where the README says it does: hardened (here: skipped, see molecule.yml), +# nothing of the node installed, state "awaiting-secret", and cloud-init reporting done +# without errors. +- name: Converge — first boot of the user-data + hosts: all + become: true + vars: + userdata_src: "{{ lookup('ansible.builtin.env', 'MOLECULE_PROJECT_DIRECTORY') }}/../cloud-init/user-data.yaml" + seed: /var/lib/cloud/seed/nocloud + # Stood up by prepare.yml. + mirror_base: http://127.0.0.1:8080/releases + mirror_key: /srv/mirror/KEYS.asc + tasks: + - name: Read the commit the source repo is at (recorded by prepare.yml) + ansible.builtin.slurp: + src: /srv/devops-src.sha + register: src_sha_file + + - name: Decode it + ansible.builtin.set_fact: + src_sha: "{{ src_sha_file.content | b64decode | trim }}" + + - name: Create the NoCloud seed directory + ansible.builtin.file: + path: "{{ seed }}" + state: directory + owner: root + group: root + mode: "0700" + + - name: Seed the committed user-data + ansible.builtin.copy: + src: "{{ userdata_src }}" + dest: "{{ seed }}/user-data" + owner: root + group: root + mode: "0600" + + - name: Fill in the MUST-EDIT values and point at the local repo and mirror + ansible.builtin.replace: + path: "{{ seed }}/user-data" + regexp: "{{ item.regexp }}" + replace: "{{ item.replace }}" + loop: + - regexp: '^(\s*)DEVOPS_REPO=https://github\.com/decdn/devops\.git$' + replace: '\1DEVOPS_REPO=file:///srv/devops-src' + - regexp: '^(\s*)DEVOPS_REF=CHANGE_ME$' + replace: '\1DEVOPS_REF={{ src_sha }}' + - regexp: '^(\s*)DECDN_BOOTSTRAP_ANSIBLE_ARGS=$' + replace: '\1DECDN_BOOTSTRAP_ANSIBLE_ARGS=--skip-tags baseline' + - regexp: '^(\s*)- name: CHANGE_ME\b.*$' + replace: '\1- name: molecule' + - regexp: '^(\s*)- "CHANGE_ME"(.*)$' + replace: '\1- "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMolecule0000000000000000000000000000000000 molecule"' + - regexp: '^(\s*)decdn_node_version: "CHANGE_ME".*$' + replace: '\1decdn_node_version: "0.0.0-molecule-stub"' + - regexp: '^(\s*)# decdn_node_release_base: .*$' + replace: '\1decdn_node_release_base: "{{ mirror_base }}"\n\1decdn_release_keyring: {{ mirror_key }}' + - regexp: '^(\s*)decdn_region: "CHANGE_ME".*$' + replace: '\1decdn_region: "DE"' + loop_control: + label: "{{ item.regexp }}" + + # Values only (`KEY=CHANGE_ME`, `key: CHANGE_ME`, `- "CHANGE_ME"`), not the header + # comment or stage 1's own `!= CHANGE_ME` guard. + - name: Check that no placeholder value is left (the file and this test still agree) + ansible.builtin.command: + cmd: >- + grep -nE '(=|: |- )"?CHANGE_ME' {{ seed }}/user-data + register: leftover + changed_when: false + failed_when: leftover.rc != 1 + + - name: Write the NoCloud meta-data + ansible.builtin.copy: + dest: "{{ seed }}/meta-data" + owner: root + group: root + mode: "0600" + content: | + instance-id: molecule-cloud-init + local-hostname: {{ inventory_hostname }} + + # Container-only settings, none of which touch what the user-data does: find the + # seed without probing clouds, leave the container's network and hostname alone, and + # skip the locale module (the image ships no locales, so update-locale fails; a + # cloud image has them). + - name: Configure cloud-init for a container + ansible.builtin.copy: + dest: /etc/cloud/cloud.cfg.d/99-molecule.cfg + owner: root + group: root + mode: "0644" + content: | + datasource_list: [NoCloud, None] + network: {config: disabled} + preserve_hostname: true + locale: false + + - name: Boot + block: + - name: Start from a clean cloud-init state + ansible.builtin.command: + cmd: cloud-init clean --logs + changed_when: true + + # The four stages a real boot runs, in order. The last one runs runcmd, which is + # stage 1 of the bootstrap. + - name: Run cloud-init's boot stages + ansible.builtin.command: + cmd: "cloud-init {{ item }}" + loop: + - init --local + - init + - modules --mode=config + - modules --mode=final + changed_when: true + + - name: Read cloud-init's result + ansible.builtin.slurp: + src: /run/cloud-init/result.json + register: ci_result + + # Not `cloud-init status`: cloud-init was installed after this container booted, + # so the systemd generator never enabled it and the status reads + # "disabled-by-generator" even though every stage above ran. The stages' own + # records are the ground truth. + - name: Read cloud-init's per-stage status + ansible.builtin.slurp: + src: /run/cloud-init/status.json + register: ci_status + + - name: Assert cloud-init finished without errors + vars: + result: "{{ (ci_result.content | b64decode | from_json).v1 }}" + stages: "{{ (ci_status.content | b64decode | from_json).v1 }}" + ansible.builtin.assert: + that: + - result.errors | length == 0 + - result.datasource is search('NoCloud') + - stages['modules-final'].finished is not none + - stages['modules-final'].errors | length == 0 + fail_msg: "cloud-init did not finish cleanly: {{ result }} {{ stages['modules-final'] }}" + rescue: + - name: Read the bootstrap's output + ansible.builtin.command: + cmd: tail -n 80 /var/log/cloud-init-output.log + register: ci_output + changed_when: false + failed_when: false + + - name: Fail with the bootstrap's output + ansible.builtin.fail: + msg: "{{ ci_output.stdout_lines | default([]) }}" + + # --- Stopped at the secret gate ------------------------------------------------- + - name: Read the bootstrap state + ansible.builtin.slurp: + src: /var/lib/decdn-bootstrap/state + register: bootstrap_state + + # Stage 1 leaves the checkout on a detached HEAD, so .git/HEAD holds the SHA itself. + - name: Read the checkout's commit + ansible.builtin.slurp: + src: /opt/decdn-devops/.git/HEAD + register: checkout_head + + - name: Stat what must NOT exist before the secret + ansible.builtin.stat: + path: "{{ item }}" + register: premature + loop: + - /usr/local/bin/decdn-node + - /etc/systemd/system/decdn-node.service + - /var/lib/decdn/keystore.json + + - name: Run the login hint + ansible.builtin.command: + cmd: sh /etc/profile.d/decdn-bootstrap.sh + register: login_hint + changed_when: false + + - name: Validate the user-data cloud-init actually consumed + ansible.builtin.command: + cmd: cloud-init schema --system + changed_when: false + + - name: Assert the host is waiting for its secret, and nothing more + ansible.builtin.assert: + that: + - bootstrap_state.content | b64decode | trim == 'awaiting-secret' + - checkout_head.content | b64decode | trim == src_sha + - premature.results | selectattr('stat.exists') | list | length == 0 + - login_hint.stdout is search('waiting for its RPC secret') + fail_msg: >- + Expected state awaiting-secret at {{ src_sha }} with no node installed + and a login hint. Got state {{ bootstrap_state.content | b64decode | trim }}, + checkout {{ checkout_head.content | b64decode | trim }}, present: + {{ premature.results | selectattr('stat.exists') | map(attribute='item') | list }}, + hint {{ login_hint.stdout | to_json }}. diff --git a/ansible/molecule/cloud-init/molecule.yml b/ansible/molecule/cloud-init/molecule.yml new file mode 100644 index 0000000..567bc8d --- /dev/null +++ b/ansible/molecule/cloud-init/molecule.yml @@ -0,0 +1,72 @@ +--- +# Boots the REAL cloud-init/user-data.yaml in a systemd container and follows the +# operator walkthrough in cloud-init/README.md end to end: +# prepare installs cloud-init. It stands up a local release mirror: the shared stub +# daemon packaged as release tarballs, with a SHA256SUMS signed by a +# throwaway GPG key, served on loopback. It also commits the working tree +# into a git repo in the container, for stage 1 to clone. +# converge seeds NoCloud with the user-data (placeholders filled, the repo, mirror +# and keyring pointed at the local ones) and runs cloud-init's boot stages. +# It then asserts that the host stopped cleanly at the secret gate. +# side_effect writes 0600 /etc/decdn/decdn.env over "SSH", as the operator would, and +# re-runs `decdn-bootstrap`. +# verify checks a release-mode install, a host-generated wallet, a running +# daemon, and the pinned collection set. +# +# Release mode runs here and nowhere else in the suite: the role's download, GPG +# signature check, SHA256SUMS check, extract and version stamp, against a mirror, +# because upstream has not cut a release yet. +# +# Coverage boundary: `baseline` is skipped through the user-data's own +# DECDN_BOOTSTRAP_ANSIBLE_ARGS (--skip-tags baseline). Its host hardening (nftables, +# DevSec sysctl/sshd, fail2ban) means nothing in a throwaway container, the same as in +# every other scenario (see ../default/molecule.yml). So this scenario proves the +# bootstrap plumbing and the node path, not the hardening. `cloud-init/README.md` says +# the same. +# +# Oldest and newest supported distributions only. They bracket the controller Pythons +# that cloud-init/requirements.txt pins two ansible-core versions for (3.11 gets 2.19, +# 3.14 gets 2.21). The image digests are the same pins as ../default and ../os-matrix; +# bump them together. +# +# The stages need network access: apt (the user-data's packages), PyPI (the pinned +# ansible-core) and Galaxy (the pinned collections). +driver: + name: docker +platforms: + - name: decdn-cloud-init-deb12 + image: geerlingguy/docker-debian12-ansible@sha256:a131e3b7a9a38dda40e98b49b39d6200ffab0ad853c714a513f5eea41bab9a25 + pre_build_image: true + command: /usr/lib/systemd/systemd + privileged: true + cgroupns_mode: host + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + - name: decdn-cloud-init-u2604 + image: geerlingguy/docker-ubuntu2604-ansible@sha256:a81f2af83a18b609eabdc926d56bc4813d3d7558d60dc891db6056f868fe6054 + pre_build_image: true + command: /usr/lib/systemd/systemd + privileged: true + cgroupns_mode: host + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw +provisioner: + name: ansible + env: + ANSIBLE_ROLES_PATH: "${MOLECULE_PROJECT_DIRECTORY}/roles" + ANSIBLE_COLLECTIONS_PATH: "${MOLECULE_PROJECT_DIRECTORY}/collections" + # See ../default/molecule.yml for why pipelining is on. + ANSIBLE_PIPELINING: "true" +verifier: + name: ansible +scenario: + test_sequence: + # Leading destroy: see ../default/molecule.yml. No idempotence step: converge boots + # cloud-init once, which is what a first boot does. + - destroy + - create + - prepare + - converge + - side_effect + - verify + - destroy diff --git a/ansible/molecule/cloud-init/prepare.yml b/ansible/molecule/cloud-init/prepare.yml new file mode 100644 index 0000000..5b6ac9b --- /dev/null +++ b/ansible/molecule/cloud-init/prepare.yml @@ -0,0 +1,157 @@ +--- +# Everything the user-data needs from the outside world, stood up inside the container: +# * cloud-init itself (a cloud image ships it; these images do not); +# * the repo, committed from the WORKING TREE, so the scenario tests the change under +# review rather than whatever is pushed. Stage 1 clones it from file:// and checks +# out that commit exactly as it would a pinned SHA on GitHub; +# * a release mirror: the shared stub daemon packaged the way upstream names its +# release tarballs, with a SHA256SUMS signed by a throwaway key and served on +# loopback. The role verifies it against that key (decdn_release_keyring) rather +# than the vendored upstream KEYS, which is the only difference from a real +# release. +- name: Prepare — pack the working tree on the controller + hosts: localhost + gather_facts: false + vars: + repo_root: "{{ lookup('ansible.builtin.env', 'MOLECULE_PROJECT_DIRECTORY') }}/.." + scratch: "{{ lookup('ansible.builtin.env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}" + tasks: + # Tracked and untracked-but-not-ignored files, so a new file under review is + # included, and a tracked file deleted in the working tree is skipped instead of + # failing tar. + - name: Pack the working tree on the controller + ansible.builtin.shell: + cmd: | + set -euo pipefail + git ls-files -z -co --exclude-standard \ + | while IFS= read -r -d '' f; do if [ -e "$f" ]; then printf '%s\0' "$f"; fi; done \ + | tar --null -T - -czf {{ (scratch ~ '/devops-src.tgz') | quote }} + chdir: "{{ repo_root }}" + executable: /bin/bash + changed_when: true + +- name: Prepare + hosts: all + become: true + vars: + scratch: "{{ lookup('ansible.builtin.env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}" + stub_bin: "{{ lookup('ansible.builtin.env', 'MOLECULE_PROJECT_DIRECTORY') }}/molecule/default/files/decdn-node-stub" + stub_version: "0.0.0-molecule-stub" + mirror: /srv/mirror + release_dir: "{{ mirror }}/www/releases/v{{ stub_version }}" + targets: {x86_64: x86_64-unknown-linux-gnu, aarch64: aarch64-unknown-linux-gnu} + target: "{{ targets[ansible_facts.architecture] }}" + tasks: + - name: Install cloud-init, git and gnupg + ansible.builtin.apt: + name: [cloud-init, git, gnupg, tar] + state: present + update_cache: true + + # --- The repo, as a git remote (packed by the play above) ---------------------- + - name: Create the source repo directory + ansible.builtin.file: + path: /srv/devops-src + state: directory + owner: root + group: root + mode: "0755" + + - name: Unpack the working tree into the container + ansible.builtin.unarchive: + src: "{{ scratch }}/devops-src.tgz" + dest: /srv/devops-src + + - name: Commit it, so stage 1 has a SHA to pin and verify + ansible.builtin.shell: + cmd: | + set -euo pipefail + git init -q + git add -A + git -c user.name=molecule -c user.email=molecule@molecule.invalid commit -qm "molecule snapshot" + git rev-parse HEAD > /srv/devops-src.sha + chdir: /srv/devops-src + executable: /bin/bash + creates: /srv/devops-src/.git + + # --- The release mirror ----------------------------------------------------------- + # Each level explicitly: the file module gives parents it creates the leaf's mode, + # and the server (a DynamicUser) must traverse down to the release directory. + - name: Create the mirror's served tree + ansible.builtin.file: + path: "{{ item }}" + state: directory + owner: root + group: root + mode: "0755" + loop: + - "{{ mirror }}" + - "{{ mirror }}/www" + - "{{ mirror }}/www/releases" + - "{{ release_dir }}" + + - name: Create the mirror's private directories + ansible.builtin.file: + path: "{{ item }}" + state: directory + owner: root + group: root + mode: "0700" + loop: + - "{{ mirror }}/gnupg" + - "{{ mirror }}/stage" + + # One flat binary per archive, named as upstream's release does (install.yml). + - name: Stage the stub as both release binaries + ansible.builtin.copy: + src: "{{ stub_bin }}" + dest: "{{ mirror }}/stage/{{ item }}" + owner: root + group: root + mode: "0755" + loop: [decdn-node, decdn] + + - name: Build, checksum and sign the release + ansible.builtin.shell: + cmd: | + set -euo pipefail + export GNUPGHOME={{ (mirror ~ '/gnupg') | quote }} + for bin in decdn-node decdn; do + tar -C {{ (mirror ~ '/stage') | quote }} -czf "$bin-{{ stub_version }}-{{ target }}.tar.gz" "$bin" + done + sha256sum -- *.tar.gz > SHA256SUMS + gpg --batch --pinentry-mode loopback --passphrase '' \ + --quick-gen-key 'molecule mirror ' ed25519 sign 1d + gpg --batch --armor --export > {{ (mirror ~ '/KEYS.asc') | quote }} + gpg --batch --armor --detach-sign --output SHA256SUMS.asc SHA256SUMS + chmod 0644 -- * + chdir: "{{ release_dir }}" + executable: /bin/bash + creates: "{{ release_dir }}/SHA256SUMS.asc" + + - name: Install the mirror's HTTP server unit + ansible.builtin.copy: + dest: /etc/systemd/system/molecule-mirror.service + owner: root + group: root + mode: "0644" + content: | + [Unit] + Description=Loopback release mirror for the cloud-init molecule scenario + [Service] + ExecStart=/usr/bin/python3 -m http.server 8080 --bind 127.0.0.1 --directory {{ mirror }}/www + DynamicUser=yes + + - name: Start the mirror + ansible.builtin.systemd_service: + name: molecule-mirror + state: started + daemon_reload: true + + - name: Wait for the mirror to serve the signed manifest + ansible.builtin.uri: + url: "http://127.0.0.1:8080/releases/v{{ stub_version }}/SHA256SUMS.asc" + register: mirror_up + until: mirror_up.status == 200 + retries: 10 + delay: 1 diff --git a/ansible/molecule/cloud-init/side_effect.yml b/ansible/molecule/cloud-init/side_effect.yml new file mode 100644 index 0000000..5eff77c --- /dev/null +++ b/ansible/molecule/cloud-init/side_effect.yml @@ -0,0 +1,35 @@ +--- +# The operator's part, as cloud-init/README.md tells it: SSH in, write the RPC endpoint to +# a 0600 file on the host, and re-run the bootstrap. The value is a placeholder; the stub +# daemon never dials it. +- name: Side effect — provision the secret and re-run the bootstrap + hosts: all + become: true + tasks: + - name: Create /etc/decdn + ansible.builtin.file: + path: /etc/decdn + state: directory + owner: root + group: root + mode: "0700" + + - name: Write the RPC endpoint on the host + ansible.builtin.copy: + dest: /etc/decdn/decdn.env + content: "DECDN_RPC_URL=https://rpc.example.invalid/\n" + owner: root + group: root + mode: "0600" + + - name: Re-run the bootstrap + ansible.builtin.command: + cmd: /usr/local/sbin/decdn-bootstrap + register: rerun + changed_when: true + failed_when: false + + - name: Fail with the bootstrap's output + ansible.builtin.fail: + msg: "{{ rerun.stdout_lines[-60:] + rerun.stderr_lines }}" + when: rerun.rc != 0 diff --git a/ansible/molecule/cloud-init/verify.yml b/ansible/molecule/cloud-init/verify.yml new file mode 100644 index 0000000..0eb9ef2 --- /dev/null +++ b/ansible/molecule/cloud-init/verify.yml @@ -0,0 +1,100 @@ +--- +# After the secret: the node came from the (mirrored, signed) release, generated its own +# wallet and is running. The toolchain is exactly what cloud-init/ pins. +- name: Verify + hosts: all + become: true + vars: + ci_dir: "{{ lookup('ansible.builtin.env', 'MOLECULE_PROJECT_DIRECTORY') }}/../cloud-init" + venv: /opt/decdn-bootstrap/venv + lock: "{{ (lookup('ansible.builtin.file', ci_dir ~ '/collections.lock.yml') | from_yaml).collections }}" + # requirements.in's two ansible-core pins, keyed by which side of 3.12 they serve. + core_pins: >- + {{ lookup('ansible.builtin.file', ci_dir ~ '/requirements.in') + | regex_findall('(?m)^ansible-core==(\S+) ; python_version (<|>=) "3\.12"') }} + tasks: + - name: Read the bootstrap state + ansible.builtin.slurp: + src: /var/lib/decdn-bootstrap/state + register: bootstrap_state + + - name: Read the installed-version stamp (release mode only writes it) + ansible.builtin.slurp: + src: /usr/local/lib/decdn/installed-version + register: stamp + + - name: Stat the host-generated wallet + ansible.builtin.stat: + path: "{{ item }}" + register: wallet + loop: + - /var/lib/decdn/keystore.json + - /var/lib/decdn/node.secret + - /etc/decdn/keystore.password + + - name: Gather service facts + ansible.builtin.service_facts: + + - name: Run the login hint + ansible.builtin.command: + cmd: sh /etc/profile.d/decdn-bootstrap.sh + register: login_hint + changed_when: false + + - name: Assert the node is installed from the release, keyed and running + ansible.builtin.assert: + that: + - bootstrap_state.content | b64decode | trim == 'complete' + - stamp.content | b64decode | trim == '0.0.0-molecule-stub' + - wallet.results | rejectattr('stat.exists') | list | length == 0 + - wallet.results | map(attribute='stat.mode') | unique == ['0600'] + - ansible_facts.services['decdn-node.service'].state == 'running' + - login_hint.stdout == '' + fail_msg: >- + state={{ bootstrap_state.content | b64decode | trim }} + stamp={{ stamp.content | b64decode | trim }} + wallet={{ wallet.results | map(attribute='stat') | list }} + service={{ ansible_facts.services['decdn-node.service'] | default('absent') }} + hint={{ login_hint.stdout | to_json }} + + # --- The pinned toolchain --------------------------------------------------------- + - name: Read the venv's Python and ansible-core versions + ansible.builtin.command: + cmd: >- + {{ venv }}/bin/python -c + "import sys, ansible.release as r; print('%d.%d' % sys.version_info[:2], r.__version__)" + register: toolchain + changed_when: false + + - name: List the installed collections + ansible.builtin.command: + cmd: >- + {{ venv }}/bin/ansible-galaxy collection list --format json + -p /opt/decdn-devops/ansible/collections + register: galaxy_list + changed_when: false + + - name: Assert the toolchain is exactly the pinned one + vars: + py: "{{ toolchain.stdout.split()[0] }}" + core: "{{ toolchain.stdout.split()[1] }}" + want_core: >- + {{ (core_pins | selectattr(1, 'equalto', '<') | first)[0] + if py is version('3.12', '<') else + (core_pins | selectattr(1, 'equalto', '>=') | first)[0] }} + # -p adds to the default search paths, so pick out the checkout's own tree. + tree: >- + {{ (galaxy_list.stdout | from_json)['/opt/decdn-devops/ansible/collections/ansible_collections'] + | default({}) | dict2items }} + installed: >- + {{ tree | map(attribute='key') | zip(tree | map(attribute='value.version')) + | map('join', '==') | sort }} + wanted: "{{ lock | map(attribute='name') | zip(lock | map(attribute='version')) | map('join', '==') | sort }}" + ansible.builtin.assert: + that: + - core_pins | length == 2 + - core == want_core + - installed == wanted + fail_msg: >- + Python {{ py }} got ansible-core {{ core }} (pinned: {{ want_core }}); + collections {{ installed }} vs collections.lock.yml {{ wanted }}. diff --git a/cloud-init/README.md b/cloud-init/README.md new file mode 100644 index 0000000..ca900b0 --- /dev/null +++ b/cloud-init/README.md @@ -0,0 +1,206 @@ +# deCDN node from cloud-init user-data + +A deCDN node with no machine of your own in the loop. Paste +[`user-data.yaml`](user-data.yaml) into your provider's "create server" form, and a +fresh Debian 12/13 or Ubuntu 24.04/26.04 VM (x86_64 or aarch64) sets itself up. It +hardens itself and stops to wait for its RPC endpoint. You then SSH in once to write +that endpoint. + +On the host it runs the [Ansible project](../ansible/README.md)'s `site.yml` against +localhost. That means the same `baseline` hardening (nftables default-deny with only +SSH and udp/4433 open, DevSec SSH/OS hardening, fail2ban, unattended upgrades) and the +same `decdn_node` role, with no second copy of either. Nearly every provider accepts +cloud-init: Hetzner, DigitalOcean, OVHcloud, Vultr, AWS, Scaleway and others. + +Pick this path for one VM when you don't want a control machine. For a fleet, or +repeated deploys from your workstation, use the [Ansible project](../ansible/README.md) +directly. [`docs/requirements.md`](../docs/requirements.md) compares the paths. + +> **Upstream has not published a release yet.** The node installs only from a +> GPG-verified release tarball (`release` mode). The `manual` mode copies binaries +> from a control machine, which this path does not have. Until a release exists, serve +> `v/{decdn-node,decdn}--.tar.gz`, `SHA256SUMS` and +> `SHA256SUMS.asc` from a mirror, and set `decdn_node_release_base` in the user-data +> to point at it. The signature is still checked against deCDN's release key. + +## What happens at boot + +1. cloud-init installs `git`, `python3-venv`, `ca-certificates` and `sudo`, then writes + three files: + - `/etc/decdn-bootstrap/bootstrap.env`: which revision of this repo to run. + - `/etc/decdn-bootstrap/inventory.yml`: your non-secret settings. + - `/usr/local/sbin/decdn-bootstrap`: stage 1 of the bootstrap. +2. **Stage 1** (`decdn-bootstrap`) clones this repo into `/opt/decdn-devops` at + `DEVOPS_REF`. If the ref is a full commit SHA, it checks that the checkout really is + at that commit. It then runs the checkout's [`bootstrap.sh`](bootstrap.sh). +3. **Stage 2** (`bootstrap.sh`) installs the pinned toolchain: + - ansible-core into `/opt/decdn-bootstrap/venv`, from + [`requirements.txt`](requirements.txt) with pip's hash checking on; + - the Galaxy collections at the exact versions in + [`collections.lock.yml`](collections.lock.yml). + + It then syntax-checks the playbook and checks that the inventory puts localhost in + `decdn_nodes`. +4. With no `/etc/decdn/decdn.env` yet, stage 2 runs only the `baseline` role and + records `awaiting-secret`. With the file present, it runs the whole playbook and + records `complete`. + +The state is in `/var/lib/decdn-bootstrap/state`: `running`, `awaiting-secret`, +`complete` or `failed`. A login hint (`/etc/profile.d/decdn-bootstrap.sh`) prints the +next step while the bootstrap is unfinished. The full log is in +`/var/log/cloud-init-output.log`. + +## Set up + +1. **Fill in the user-data.** Copy [`user-data.yaml`](user-data.yaml) and replace every + `CHANGE_ME`: + - `DEVOPS_REF`: a full 40-character commit SHA of this repo (recommended; it is + verified after checkout) or a release tag. Branch names are refused. + - `baseline_sudo_users`: your admin login and your SSH **public** key. Hardening + disables root and password logins. Some providers (Hetzner, DigitalOcean) inject + your key for `root` only, so without this entry you are locked out. + - `decdn_node_version`: the upstream release to install. + - `decdn_region`: the VM's ISO 3166-1 alpha-2 country code, e.g. `DE`. + + Optional: + - `ssh_allow_cidrs`, to accept SSH only from your addresses; + - `decdn_node_release_base`, for a mirror (see the note above); + - `decdn_network`, `arbitrum-sepolia` today. + + Any other role knob can go in the same `vars:` block + (`ansible/roles/*/defaults/main.yml`). Check the file before you paste it: + + ```bash + make lint-cloud-init CLOUD_INIT_FILE=path/to/your-user-data.yaml + ``` + + It fails on your edited copy only if an invariant breaks, for example a secret + added or `release` mode changed. + +2. **Create the VM** with the file as its user data. Examples: + - **Hetzner Cloud:** "Cloud config" field, or `hcloud server create --user-data-from-file`. + - **DigitalOcean:** "Advanced options → Add initialization scripts", or the + `user_data` of a `digitalocean_droplet` in Terraform + ([#48](https://github.com/decdn/devops/issues/48)). + - **AWS EC2:** "Advanced details → User data". + + Open **udp/4433** in the provider's own firewall, if it has one. The host's nftables + already allows it. + +3. **Wait for the first boot to finish.** This takes a few minutes: packages, pip, + Galaxy and the hardening run. + + ```bash + ssh @ cloud-init status --wait # status: done + ssh @ cat /var/lib/decdn-bootstrap/state # awaiting-secret + ``` + + `status: error` means the bootstrap failed. The reason is at the end of + `/var/log/cloud-init-output.log`. Fix it (usually a value in + `/etc/decdn-bootstrap/inventory.yml`), then run `sudo decdn-bootstrap`. + +4. **Write the RPC endpoint on the host.** The URL may embed a provider API key, so it + never goes in user-data: + + ```bash + umask 077 + sudo mkdir -p /etc/decdn + echo 'DECDN_RPC_URL=https://…' | sudo tee /etc/decdn/decdn.env >/dev/null + sudo chmod 600 /etc/decdn/decdn.env + ``` + + Other environment-borne secrets, such as S3 cache-origin credentials, go in the + same file ([`decdn.env.example`](../ansible/roles/decdn_node/files/decdn.env.example)). + +5. **Install and start the node:** + + ```bash + sudo decdn-bootstrap # ends with "decdn-bootstrap: complete" + ``` + + This runs the whole playbook: + - installs the verified release; + - generates the node's wallet on the host (`decdn_node_generate_keystore`); + - writes `node.toml`, and has the real binary validate it; + - starts `decdn-node`. + +6. **Stake and register on chain.** This is an operator step, driven by `decdn setup`. + Use the `decdn_chain` helper in + [`docs/lifecycle.md`](../docs/lifecycle.md#running-on-chain-commands): + + ```bash + decdn_chain setup --mbps 100 --region DE \ + --multiaddr /ip4//udp/4433/quic-v1 --dry-run + ``` + + The wallet's address is in `/var/lib/decdn/keystore.json`. Fund it before you run + the command without `--dry-run`. + +## Operate + +- **Re-run or change settings:** edit `/etc/decdn-bootstrap/inventory.yml`, then run + `sudo decdn-bootstrap`. Every run converges the whole host again, as `make deploy` + does. +- **Upgrade the deployment code:** set `DEVOPS_REF` in + `/etc/decdn-bootstrap/bootstrap.env` to the new SHA or tag, then run + `sudo decdn-bootstrap`. It fetches, verifies and re-installs the toolchain pinned at + that revision. Nothing pulls on a timer: the host only runs code you pinned. +- **Upgrade the node:** change `decdn_node_version` in the inventory and re-run. +- **Back up, migrate or decommission:** the host is an ordinary Ansible node. Add it to + an inventory on your workstation with the same variables and use `make backup`, + `make decommission` and the rest ([`docs/lifecycle.md`](../docs/lifecycle.md)). From + then on, manage it from one place: either `decdn-bootstrap` on the host or + `make deploy` from the workstation, never both. + +## Security notes + +- **User-data is not private.** Any local process can read it from the instance + metadata service, and the provider keeps it in its console and API. The file carries + only public material: an SSH public key, a version, a region, a repo URL. The RPC URL + is written over SSH, and the wallet is generated on the host. + `make lint-cloud-init` fails on: + - any secret-looking key (RPC URL, password, token, private key, keystore); + - a URL with embedded credentials; + - an unknown `bootstrap.env` key. +- **Everything is pinned.** + - This repo: by commit SHA (checked after checkout) or tag. + - ansible-core: by version and hash. + - The collections: by exact version. + - The node: by release version, installed only if `SHA256SUMS` carries a valid + signature from deCDN's release key (`decdn_verify_release_signature` cannot be + turned off in this file). + + Nothing is piped from `curl` into a shell. +- **No lockout.** Baseline refuses to harden SSH unless `baseline_sudo_users` names a + non-root account with a key. +- **The network posture is the role's.** It is nftables default-deny with only SSH and + the node's QUIC udp/4433 open. Metrics and the admin RPC stay on loopback. +- The files in `/etc/decdn-bootstrap/` are `root` `0600`. `decdn-bootstrap` refuses a + `bootstrap.env` that is not `root`-owned `0600`. It reads the file as literal + `KEY=value` lines and never sources it, and it rejects unknown keys. + +What CI proves: + +- `make lint-cloud-init` (CI job `cloud-init`) checks the schema and the invariants + above, and `make test-scripts` checks that it rejects broken variants. +- The molecule `cloud-init` scenario boots this file with cloud-init in Debian 12 and + Ubuntu 26.04 containers. It covers the secret gate, then `decdn.env`, then a running + node installed from a locally signed mirror. +- The scenario skips `baseline`, because host hardening means nothing in a container. + It is exercised on real hosts, as for the Ansible path. + +## Updating the pins + +- **ansible-core:** edit [`requirements.in`](requirements.in), then regenerate: + + ```bash + uv pip compile --universal --generate-hashes --python-version 3.11 \ + cloud-init/requirements.in -o cloud-init/requirements.txt + ``` + + Keep a pin whose controller Python range covers 3.11 (Debian 12) and one covering + 3.12 to 3.14 (Ubuntu 24.04, Debian 13, Ubuntu 26.04). +- **Collections:** run `make deps` in `ansible/`, then copy the resolved versions into + [`collections.lock.yml`](collections.lock.yml). `make lint-cloud-init` checks the + lock against `ansible/requirements.yml`. The molecule scenario checks that a node + ends up with exactly the locked set. diff --git a/cloud-init/bootstrap.sh b/cloud-init/bootstrap.sh new file mode 100755 index 0000000..3e8cd4d --- /dev/null +++ b/cloud-init/bootstrap.sh @@ -0,0 +1,128 @@ +#!/usr/bin/env bash +# Stage 2 of the cloud-init bootstrap (see README.md). Stage 1 is the small +# /usr/local/sbin/decdn-bootstrap that user-data.yaml writes. It clones this repo at the +# pinned ref, verifies the checkout, then execs this script from it. +# +# This script turns the host into a node by running ansible/playbooks/site.yml against +# localhost: +# 1. Install the pinned ansible-core into a venv, with hashes enforced +# (requirements.txt), and the pinned Galaxy collections (collections.lock.yml). +# 2. Syntax-check site.yml and check that the inventory puts localhost in decdn_nodes. +# Without that membership the play matches no host and exits 0, and the udp/4433 +# hole in playbooks/group_vars/decdn_nodes.yml never loads. +# 3. Pick the phase: +# - no /etc/decdn/decdn.env: run `baseline` only (SSH, firewall, patching, +# the admin account), then record "awaiting-secret". The node role would stop at +# its RPC gate anyway, so stopping here keeps cloud-init's status clean. +# - decdn.env present: run the whole playbook (install, keystore, service), then +# record "complete". +# +# Re-running it is how the operator continues after writing decdn.env, and how a host +# picks up a new pinned ref: `sudo decdn-bootstrap`. +set -Eeuo pipefail + +readonly CONF_DIR=/etc/decdn-bootstrap +readonly INVENTORY=$CONF_DIR/inventory.yml +readonly VENV=/opt/decdn-bootstrap/venv +readonly STATE_DIR=/var/lib/decdn-bootstrap +readonly STATE_FILE=$STATE_DIR/state +readonly LOGIN_HINT=/etc/profile.d/decdn-bootstrap.sh +# The decdn_node role's default decdn_env_file (roles/decdn_node/defaults/main.yml). +readonly ENV_FILE=/etc/decdn/decdn.env + +repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +readonly repo + +# World-readable on purpose: the login hint below runs as the admin account. +set_state() { + install -d -m 0755 "$STATE_DIR" + printf '%s\n' "$1" >"$STATE_FILE.tmp" + chmod 0644 "$STATE_FILE.tmp" + mv -f "$STATE_FILE.tmp" "$STATE_FILE" +} + +die() { + echo "decdn-bootstrap: $*" >&2 + [[ $EUID -ne 0 ]] || set_state failed + exit 1 +} + +trap 'set_state failed; echo "decdn-bootstrap: FAILED (see the output above; re-run: sudo decdn-bootstrap)" >&2' ERR + +[[ $EUID -eq 0 ]] || die "run as root (sudo decdn-bootstrap)" +[[ -f $INVENTORY ]] || die "$INVENTORY is missing (it is written by the cloud-init user-data)" +if grep -n 'CHANGE_ME' "$INVENTORY" >&2; then + die "$INVENTORY still has CHANGE_ME placeholders (the lines above); edit them, then re-run" +fi + +# Extra ansible-playbook arguments from bootstrap.env, for example +# `--skip-tags baseline` in the containerised CI test. Split on whitespace. +read -ra extra_args <<<"${DECDN_BOOTSTRAP_ANSIBLE_ARGS:-}" + +set_state running + +# A login hint for the admin account, written before anything can fail so that a failed +# first boot shows it too. It prints nothing once the node is complete. It lives in +# /etc/profile.d rather than a MOTD, because DevSec ssh_hardening disables the PAM motd. +cat >"$LOGIN_HINT" </dev/null)" in + running) + echo "deCDN: the bootstrap is running (cloud-init status --wait; log: /var/log/cloud-init-output.log)" ;; + awaiting-secret) + echo "deCDN: hardened, waiting for its RPC secret. Write 0600 $ENV_FILE, then run: sudo decdn-bootstrap" ;; + failed) + echo "deCDN: the last bootstrap run FAILED. Re-run it: sudo decdn-bootstrap" ;; +esac +EOF +chmod 0644 "$LOGIN_HINT" + +# --- 1. Pinned toolchain ------------------------------------------------------ +# A venv whose interpreter no longer runs (the distro's python3 changed under it after a +# release upgrade) is rebuilt instead of being patched. +if ! "$VENV/bin/python" -c '' 2>/dev/null; then + rm -rf "$VENV" + python3 -m venv "$VENV" +fi +# --only-binary: no compiler on the host, and nothing built from an sdist at boot. +"$VENV/bin/pip" install --quiet --disable-pip-version-check --no-input \ + --require-hashes --only-binary=:all: -r "$repo/cloud-init/requirements.txt" + +cd "$repo/ansible" # so ansible.cfg (roles_path, collections_path) applies +export PATH="$VENV/bin:$PATH" +# Fail on an inventory that does not parse, instead of warning and running against no +# hosts (ansible/Makefile sets the same). +export ANSIBLE_INVENTORY_UNPARSED_FAILED=True + +# --force: a re-run after a ref bump must replace a collection that the new lock pins at +# a different version. Without it, ansible-galaxy keeps any installed version. +ansible-galaxy collection install --force -p collections -r "$repo/cloud-init/collections.lock.yml" + +# --- 2. Checks before touching the host ---------------------------------------- +ansible-playbook -i "$INVENTORY" playbooks/site.yml --syntax-check +members=$(ansible -i "$INVENTORY" decdn_nodes --list-hosts) +grep -qE '^\s+localhost$' <<<"$members" \ + || die "$INVENTORY does not put localhost in the decdn_nodes group (site.yml would match nothing)" + +# --- 3. Converge ---------------------------------------------------------------- +if [[ -e $ENV_FILE ]]; then + ansible-playbook -i "$INVENTORY" playbooks/site.yml "${extra_args[@]}" + set_state complete +else + ansible-playbook -i "$INVENTORY" playbooks/site.yml --tags baseline "${extra_args[@]}" + set_state awaiting-secret +fi + +state=$(<"$STATE_FILE") +echo "decdn-bootstrap: $state" +if [[ $state == awaiting-secret ]]; then + cat </dev/null + sudo chmod 600 $ENV_FILE + sudo decdn-bootstrap +EOF +fi diff --git a/cloud-init/collections.lock.yml b/cloud-init/collections.lock.yml new file mode 100644 index 0000000..aaa3913 --- /dev/null +++ b/cloud-init/collections.lock.yml @@ -0,0 +1,33 @@ +--- +# Exact Galaxy collection versions for the cloud-init bootstrap (cloud-init/bootstrap.sh). +# +# ansible/requirements.yml lists `>=` ranges, and a control machine vendors whatever they +# resolve to (`make deps`). A node that bootstraps itself has no such control machine, so +# it installs these exact versions instead. The list covers the closure: the two +# collections the roles use directly, and every collection they depend on transitively. +# Pinning only the top two would leave the transitive ones to resolve to latest. +# +# To bump, run `make deps` and copy the resolved versions from +# ansible/collections/ansible_collections/*/*/MANIFEST.json. Leave out anything that +# neither of the top two needs (community.docker is only vendored for molecule). +# `make lint-cloud-init` checks that every collection in ansible/requirements.yml is +# pinned here within its range. The cloud-init molecule scenario checks that a node +# ends up with exactly this set. +collections: + # Direct: ansible/requirements.yml + - name: devsec.hardening + version: "10.6.0" + - name: ansible.posix + version: "2.2.2" + # Transitive, via devsec.hardening + - name: community.crypto + version: "3.3.0" + - name: community.general + version: "13.3.0" + - name: community.mysql + version: "5.0.2" + # Transitive, via community.general and community.mysql + - name: community.library_inventory_filtering_v1 + version: "1.1.5" + - name: ansible.mysql + version: "5.2.0" diff --git a/cloud-init/requirements.in b/cloud-init/requirements.in new file mode 100644 index 0000000..9951db8 --- /dev/null +++ b/cloud-init/requirements.in @@ -0,0 +1,11 @@ +# The ansible-core the cloud-init bootstrap installs on the node (cloud-init/bootstrap.sh). +# Regenerate requirements.txt after editing (see README.md, "Updating the pins"): +# uv pip compile --universal --generate-hashes --python-version 3.11 \ +# cloud-init/requirements.in -o cloud-init/requirements.txt +# +# Two pins, because no single ansible-core release supports every supported host's +# controller Python. The range is Debian 12's 3.11 through Ubuntu 26.04's 3.14. 2.19 is +# the last release that supports 3.11, and 2.20 and later support 3.12 to 3.14. Both +# stay at or above the roles' min_ansible_version (2.15). +ansible-core==2.19.13 ; python_version < "3.12" +ansible-core==2.21.4 ; python_version >= "3.12" diff --git a/cloud-init/requirements.txt b/cloud-init/requirements.txt new file mode 100644 index 0000000..9dcf2d6 --- /dev/null +++ b/cloud-init/requirements.txt @@ -0,0 +1,342 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile --universal --generate-hashes --python-version 3.11 cloud-init/requirements.in -o cloud-init/requirements.txt +ansible-core==2.19.13 ; python_full_version < '3.12' \ + --hash=sha256:3711fc5db7a265f93b34cf3f0db8b9708fb0eec12211a6be32ed6edd6c68044b \ + --hash=sha256:d4a256d0891b3cb31ddbc9ff8a7bf6fd359379007799c54390b66b8f94fed3d3 + # via -r cloud-init/requirements.in +ansible-core==2.21.4 ; python_full_version >= '3.12' \ + --hash=sha256:81a9329f4f12cfa5008dcab5d1bf23ae69b7effc08c0f00048ab2461147ae95a \ + --hash=sha256:ebe74d9c8fadcb41ad2151e031bf1e785e3098aaa015019b4e19a0a96f0dcc4f + # via -r cloud-init/requirements.in +cffi==2.1.1 ; platform_python_implementation != 'PyPy' \ + --hash=sha256:046bfc24911b37851ee1b51aab8bffe713d89c68c6a057b09484ce9fd5f69b4e \ + --hash=sha256:06c72bb76605a4b0cd0aad6930b69d4baf7dd5d806cfc409b824191099700e66 \ + --hash=sha256:0beceaabe56af686895136a2de78db54ecd8e4046b236b8fd6d6cb61389e9bf2 \ + --hash=sha256:154852545011f779917b11c78db2358d095da62a9a172b78ad0a583ee5adc0d0 \ + --hash=sha256:194cffa889098ced9976c3fc6340305e43f6303657d298da55366907c05c22d6 \ + --hash=sha256:19ee6127ee34de7d83ce3d371ebc5ed91addbdcc39f9ab15ce4eb35a4e534971 \ + --hash=sha256:1a18a57b58cfb21fc28d72e876acf10eaed67a1ed96226f92af4df681d571c4c \ + --hash=sha256:1aa5645c30469b09530c4ebca77ebf8f17618293c58f8549cb1a543a50236e7d \ + --hash=sha256:1dea0e4d7d4f11f619fe8c1d76caf49e24405b4b5743c0e3be16a500ecd930c9 \ + --hash=sha256:208f941bb9d18e768138677f0a6d2ce01f590df56043dda1df1535ac57c88517 \ + --hash=sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735 \ + --hash=sha256:246fa40ce8645a614ff682e0b70f37134e460eaf93a775e0cbe3cca585a67a80 \ + --hash=sha256:25792eac27877609e7bb06d42ff88278a6624fff2ba9bbb523c09616b117e80f \ + --hash=sha256:27350daa11d4f10c540e6e89dada4c54feb7256ad03e9a4dc075ebad7ba360d1 \ + --hash=sha256:28907ab9bfb6aa13184cfc17c6b8e1023c5ab6fd7076d8c20a35e59fe04f8f29 \ + --hash=sha256:2ae64be792b8966f2c69538199728b290e34726562896df1e5dc8ffd8d8188e8 \ + --hash=sha256:31348097ff5bbe827ccc41795d4dd099d9f0625e7def00ee653c137a490c2a6c \ + --hash=sha256:3143d81e29e1e20a9ce10901ec369012947876596f75a222235965f2b7ae832e \ + --hash=sha256:3222ba5d678f80a030e6afbcc33dc1ae5cb45facabb61cee2c7016b8432fde48 \ + --hash=sha256:3311ed60d36f83378794e1009ac6258bafbf81f7888b4caa7b35a521e3f95813 \ + --hash=sha256:334644fbac4eff73d985a17a91226df55d0f394160c4cfb880e084c8f7161cac \ + --hash=sha256:34e261f78cb6ceaaa36f42f2613f4380d94d9c759a9c73c769ee6e0247364632 \ + --hash=sha256:363e05fa78e15116c3c32c210ee36884fd6b9afa6d440e47112c3bd511d64cb6 \ + --hash=sha256:398aff33cee2767e3e781d2554c54bd0dff386bb437581e0d8011fde1a942ec1 \ + --hash=sha256:3d22a20b1fb1632cc72c22f95f7b0d2961c3e1c235f245ba4c606c4771035659 \ + --hash=sha256:42a494cee34437f05546455144f2b5d9ac09b1face62bcfce597d2e521066688 \ + --hash=sha256:42e2f76b9455f5a9a844f770bf3e200ed3da0e15f5df3db9c31fe80b04b3d004 \ + --hash=sha256:42f6930c31dc7f50732c9ae793c2786c7b6b044195967bbdde40bb9be81c4cc0 \ + --hash=sha256:456a61fa52d579ebf9df2e9552ead5129855dbaff6c1e5a9b1bc408809bdc062 \ + --hash=sha256:471cee653ae88de62096552e6d24ccb4a5adb8c8c9f10b5054d0122c15bf2779 \ + --hash=sha256:49cbc70e6542d4ccccb936558d1064a8012541e78f821f955cff24e357776c94 \ + --hash=sha256:4a7c934f7360e8cd64fe9efadcbd10c7c6364f531e432b9a4bf5ccbc9e0e8b50 \ + --hash=sha256:4be96343e422f2dfcd12ab5c9f5aebe03f82f737c6bffeca6830b3875cb44aab \ + --hash=sha256:4f42141fc14250de6dde5ee7ea4432be017252d91f19c5ad043c084cea629cac \ + --hash=sha256:507a24c282e0f42f8ed737cf048572cbf580468da5555764a8331735e9c736b6 \ + --hash=sha256:51b31d1c98274844cfd7838ce00bfc27c7423a4dc00fc0772fc3331c2cc90676 \ + --hash=sha256:58acb8ab8e295e6c5ea12f888cbb13cf21511ef2a3303a23f4325c29d17fe5c1 \ + --hash=sha256:5a59cc1c4442bc3d5c703bf720b51138d0bfc173618807c9ee2490a7541dd3d9 \ + --hash=sha256:5bb4e7ea95dcd6a014a6fef62e62467d67d8e582326443f3d68e71d6320a9fcf \ + --hash=sha256:5c58fe613dc5e5336357eff555824a314d8e43282600435c8d1cb6a7a2fedd13 \ + --hash=sha256:5e7cecbaadb83884793e05828cee59b210b24583b9c7425d0ba6a754fe22eb4e \ + --hash=sha256:616f097f2fe415bc92a247f02e11f634e1f9e9a83d327e3c915c15089c87869e \ + --hash=sha256:63bbfd5ded17c4840ac07cd8f1c21ba9d9708141f840b324f422f41b207e3973 \ + --hash=sha256:64faea20f4e2613363a1a9b9c7dd73058f3ecd00133a511e72ad7c511658f527 \ + --hash=sha256:661c298b4821edebead0c91edd2b00374d67ad7c5a1f7a91d4442633b79d6a72 \ + --hash=sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890 \ + --hash=sha256:6a8dddef476fab96d066d578fc88526767b836ab5ab21754e1d5bf3879c31c7c \ + --hash=sha256:6e192623c49c94421616a5778fba35cf0d5a8d000650c1967ef4448ee5cdd990 \ + --hash=sha256:7225e4514edb64eb6740324353e0da0711954fd8d7da4576755b1c6e09b697cd \ + --hash=sha256:75f80557d1389eddbd0de2681f6a390a0c5338c31ddaa821381c203fc3fd50d9 \ + --hash=sha256:770de9db11e84213beec501cfcaa013b019820ca881e03344dea5844f7876d94 \ + --hash=sha256:7750c6449dff7864bb9bb27ddfb0267756189201a3afc911d82b3caacd70dfc3 \ + --hash=sha256:7bde5e4cc5c10140859842b9d383af292b22639a4dffb725314baf45968cef80 \ + --hash=sha256:7ce713ace7c0e4520535b42b77eaa742c16dab813978064913e5a3cf82973b41 \ + --hash=sha256:7da0c5eff80f0197f3b3d1232ec5a682a9325f4ae9016a78f5f5ca35f9ced1f5 \ + --hash=sha256:7dbb61fe3a7699468030f71bbe5f8a0e326a151daa91beb11a6fc1f980c55e1c \ + --hash=sha256:811bd1e21d32de12efca32393a0ab3f5133b54fce9bd44b8bd77ab07da14bf6a \ + --hash=sha256:8ef53b2de9bcb9197d31854256575d59dbac0cba72ac627bb291ef5eceb74be4 \ + --hash=sha256:937c0052c05a31ca1daf18de3158eed4dbfcb9cc107adbea227728d647be701e \ + --hash=sha256:9d2055050ea716bd38b7f7f1579c275386646b4894c155a3e2f3cd62ed41b7c6 \ + --hash=sha256:9f8d177621de5cb38ee3e731eda45d421db093ec0739f46a5594babda7987a98 \ + --hash=sha256:a2d7755bef5a12ed488f4ef1f1b69ee9191d7396083b755a5d2295f6edb4768b \ + --hash=sha256:a48d62ab9d6f4f98c983223a547af44be6ca3691074c31cecced6facd3ba2dc1 \ + --hash=sha256:a4f00aa42f75d6e4595e8866e748cc1705adc0cddfeb2ca86d0d03993d63ba03 \ + --hash=sha256:a6e721d4b0e45d5b65e87534470e67b18dcd092c83f68fba09f152b9cbc061af \ + --hash=sha256:a730a083190634c65cca36ba5f489531576ebd79bcd5c8e172130f6453127231 \ + --hash=sha256:a931079504ecc49efed7744c476a5c343a92fabf66dec2db95edb1b2fdc770e2 \ + --hash=sha256:aa9511c62d14da7aacc9b4bf51f3f697a621e83b2d6919008243c3aad168eea3 \ + --hash=sha256:ab36d55f9ed2d067327667c2fea18dda018eb628dd6347aa01dda6cf1f5d3836 \ + --hash=sha256:ad2c86c495b899d862ea0f4b42891b8713a3bd45dd4105c7fd51c2a72f39f3a5 \ + --hash=sha256:aeae0e330c9f6acd681f647d46cefd30c29f93e3392882e792e82080c9691399 \ + --hash=sha256:b0431303acaea1089ad4b3e9ce4e6518193def1118d4073ca848635ee4ea2e96 \ + --hash=sha256:b5bdfd1c873d4e093aabc0ca84c4ca6dbc4f752afb5c86f146d9742580c9da2e \ + --hash=sha256:baed1e86cc735622097354b9d1281406caf42ff42a886d29faa8e8d1630333be \ + --hash=sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf \ + --hash=sha256:c26608d2222fb1e94487e4a387d85f13eb55d5ed725cb25a0c589ac4ee60e7bc \ + --hash=sha256:c7659f22557c5a0bc4855cd635f55edec690cc008a40768527762cb9fb263455 \ + --hash=sha256:c8c69575568085ba0b1b10c0249d779a214aea6f6522e949a0fc9fb0fcb449d0 \ + --hash=sha256:c8d2c9fd1f2d16f780d15127abb050d13d1a76c03a4bd87d7e4980e45e511e12 \ + --hash=sha256:ca82be1a1d406ecfe1d25dc16cb33488e5a16bf4438c9fb590484ea29d92478b \ + --hash=sha256:cc572dace3f60ef98d7b12ff411d20f5362feb31a0439eab0085bbfd349982d7 \ + --hash=sha256:d18e5ac0f2f03f4f518d3e23db0f0cad7faa1da8620e9c09461d443bbf6e6692 \ + --hash=sha256:d28630f5854ab07ab1fd4aba756de52326c82e6be15d414b12793f1975048b54 \ + --hash=sha256:d9c275eaacd24aa73f94ffd6de08fc3f932424d8b6c376f4bed7cde376fe7bc3 \ + --hash=sha256:da0e573f9f97159390c89d9f1a9e41908b66d408cc5b58d08cf3847d844c531b \ + --hash=sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be \ + --hash=sha256:dddad92b554513a31f272570678ba307fb9f618f05e3d4a5eacafff9eae03e1d \ + --hash=sha256:df423d40ee8654634421812bc3b196da3f9bd7d32929da813f8394c4348a5358 \ + --hash=sha256:df913725b79db7bcf03448f36b7bf8815363417d5b58deecf9305e3e30f0f21a \ + --hash=sha256:e0bcb7e0f677f543555d2adff3bf19c05f66cdb4796e5ff602442ab2fe3c4ef7 \ + --hash=sha256:e2d65b31f36619cda3999b78b2aa9632e76b78448e7a56fc4240824200e7c4fc \ + --hash=sha256:e6e8cff14d6fb0be70a09c0bdc58096f501952d04624ebf867e0e56da2df8960 \ + --hash=sha256:f16c709686a78c727bbbf059f92b0bf41c6fc60deec706d2dc19f529175a6125 \ + --hash=sha256:f24fb43132a4c6b4cb4eb029492919b2db645be6808d738f244fd146c03c32cb \ + --hash=sha256:f53e442b08449d42821fa4a4fba000095af9f62742a500f978a9f557ec44339a \ + --hash=sha256:f5cfbc5fe74540d335175b656c725d74d90e3730c626d92575eea35029d9afaa \ + --hash=sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf \ + --hash=sha256:f8ec5e643a9a937f64e1999eb9f75d072263751912dc5cd06d3c85f8f44be7c3 \ + --hash=sha256:fb92203a88b3d3053034db775110081c49d28be6551923805e039924093761e4 \ + --hash=sha256:fcd22650c908d7b7da162bbfaab594a1227a15d1643a98c68b122ac642fa2264 + # via cryptography +cryptography==50.0.1 \ + --hash=sha256:01f41478cf33fc605a6a089cd56d28b45c6c0b45a1928b61797f2621a04bac71 \ + --hash=sha256:05ba322c4da95b262a212c345af888ef2c37c88c0509756ea00a0e6d68850f23 \ + --hash=sha256:16c5ecd954b3330ebfb6605eca4fd952da8bef376551d5cc264534e3770a9ee6 \ + --hash=sha256:2a93d05e34d5f67fba6f891fe85d929999baa7195e853923ea6d7576c9e68c5e \ + --hash=sha256:2b34d76a652ea2b6faf777c35df230c5637842cd904e04f16230c3f9f03e4361 \ + --hash=sha256:2ebbfb0f1fed745e91796e3e1080a1440423fdae8ece1b995a1d80883a409054 \ + --hash=sha256:30a125032e5642a21ff816e021152bd4e7e94f03eff3f4b7fca41cd22bc3110f \ + --hash=sha256:330fbb252391c596f1ae42c5754449dc924e6ad012dca8efe0d703f9f2d12ec6 \ + --hash=sha256:359e62deae718bce96170e223fdcb6357e4fbd3bb7a3a75f4430763532560e49 \ + --hash=sha256:407fe2b6db00939c05c0e945e9914238f2f0a430974839429dafc82b1ee6bee5 \ + --hash=sha256:42be3bb70596b3abe4ac097b75be223e8b3ab614a0e5de068e3dcc54d71d6149 \ + --hash=sha256:4c4188f7c0cf655be5c06342b817ed0f9595b69ffa2b12026e5353eed29dea88 \ + --hash=sha256:51593d180cf6d179bde5c5d065bed81386b1f381656ae7d042b7ffc87a9895ad \ + --hash=sha256:51afcfceb15597cf2635068e4ac9a56b2abde622edde17f37d85fd7b5306497a \ + --hash=sha256:53e279950892dc102c6b4e52af03ae5ea92fac572a1ddab78ca73a997f62b69f \ + --hash=sha256:55d16b1ef3ee0958d893a977b19777887e546c9954ea81b200c3301a864013f2 \ + --hash=sha256:5dd9bda1c12b4162f6ff568eeb5e0ff956c28d14406e875cfe8a63a2d414ff20 \ + --hash=sha256:5fe002589592ed749ce77fe0695fcbd3500dd61d7d6db5858a7544c612fa8e45 \ + --hash=sha256:5fe939deeb161024a6be98229c953b6591fef1f41214497a78fe793a244c017f \ + --hash=sha256:693c99b49bd37d0d096e4334c10232c77248c415b98d35236094cdf96d57258b \ + --hash=sha256:76de83fbd91ac49c0feaaa983d0748fd7a53176afac5fb3bf7478d244f0eb527 \ + --hash=sha256:79bf008d1f9af6071c797ad133e39915dfee7614f18f18f4db9072eb715064a3 \ + --hash=sha256:804728ce710890870f3aaa344b2e161172d258d768ac139d02cfd9092d0d94e6 \ + --hash=sha256:8921d58f426793c5f1b47f0b59575780de9a095214958d0eb37d909593db8367 \ + --hash=sha256:8df2de9102026855887e4587084f6eabd80ed0f345b8ad8a7ac27ab9bf4723e0 \ + --hash=sha256:9cb3cb952cf5a8abd50c782a98a89d71699715e802fe349704b47f2425b42a94 \ + --hash=sha256:9dde0a357190eb3b1da1bb9ab750e9c85cba82ca5977aa0836cbb94e92611239 \ + --hash=sha256:9ebcdd5519be9b652a46f507817a74591774fc3d6923ac364e4dfa64e36b291b \ + --hash=sha256:a0b1a59e3a089064a0ec309e9428c8e3ae4e161419d20ac33600767e83fc658a \ + --hash=sha256:a255449073358275b64b67d3f595f268bbef70e72b6edb65e0c70c735bf739c9 \ + --hash=sha256:a8f40ea47330e71b594a7e246898f93177c259490c63183dbaf9e571d71ed9a5 \ + --hash=sha256:ac02b07824d4d1001bd4367599f839c19cb171924c796e52c23508ac14c2c0cc \ + --hash=sha256:aed8db4f6d71c51efb89530e12d9464e7bf2923d46c3205dc794a2a93f8c0648 \ + --hash=sha256:b8f852c65863251b9e3a1b8c150ce21e59b522dbb6a7d4bc80e680d38388e986 \ + --hash=sha256:be224a65493ec5b74a158ff22a5522ce4a5ca1e543c647a3a4730d4a09e5f959 \ + --hash=sha256:ca83d00d9e69cd5eb63f2e69c3a5a59e0cecae5ae14c6ae0b35830fe3b37bad0 \ + --hash=sha256:cbf74a81765ee67413503ca6e26dcc4f6f5a519822436cc0a1b97aab6c1b8a17 \ + --hash=sha256:d63ae8f6481fec907ac0f588eee8a90aefde112c633131fe540e5711ddbb5a4e \ + --hash=sha256:e22dfed744bd4002e909464cb23d2f0b05c6f3113a79ef2e9864a53db737c733 \ + --hash=sha256:e2ca8fd1b6b4b82a1c4cb02841d0837e3c12336c2e24b520ab8ab3b969733d8f \ + --hash=sha256:e74591e283fe6eb956416c929eb58262a719fe0311fd9054c62c3350ed8760d8 \ + --hash=sha256:f74455bb086a85d5e81246412602aaa97ed095e504cd40dd261ef50be42205bf \ + --hash=sha256:fb4b9672d389c738b175c4166e78310f8a70358886aacd9173ee03a85ffdc671 \ + --hash=sha256:fc3ed7ebd2a8c96f5b166de0ab9b624996bef3b07bbeb19364dfb78222c22c80 \ + --hash=sha256:fd3718b960d0b5dd213cdf03f3bcb7000e69dda0de8b956061947ff6bcff5558 \ + --hash=sha256:ff838d62ec1bfce4f9ba7fa16f4a7b554cd8d0c299e6be37502161a660c84eef + # via ansible-core +jinja2==3.1.6 \ + --hash=sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d \ + --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 + # via ansible-core +markupsafe==3.0.3 \ + --hash=sha256:0303439a41979d9e74d18ff5e2dd8c43ed6c6001fd40e5bf2e43f7bd9bbc523f \ + --hash=sha256:068f375c472b3e7acbe2d5318dea141359e6900156b5b2ba06a30b169086b91a \ + --hash=sha256:0bf2a864d67e76e5c9a34dc26ec616a66b9888e25e7b9460e1c76d3293bd9dbf \ + --hash=sha256:0db14f5dafddbb6d9208827849fad01f1a2609380add406671a26386cdf15a19 \ + --hash=sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf \ + --hash=sha256:0f4b68347f8c5eab4a13419215bdfd7f8c9b19f2b25520968adfad23eb0ce60c \ + --hash=sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175 \ + --hash=sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219 \ + --hash=sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb \ + --hash=sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6 \ + --hash=sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab \ + --hash=sha256:15d939a21d546304880945ca1ecb8a039db6b4dc49b2c5a400387cdae6a62e26 \ + --hash=sha256:177b5253b2834fe3678cb4a5f0059808258584c559193998be2601324fdeafb1 \ + --hash=sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce \ + --hash=sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218 \ + --hash=sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634 \ + --hash=sha256:1ba88449deb3de88bd40044603fafffb7bc2b055d626a330323a9ed736661695 \ + --hash=sha256:1cc7ea17a6824959616c525620e387f6dd30fec8cb44f649e31712db02123dad \ + --hash=sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73 \ + --hash=sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c \ + --hash=sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe \ + --hash=sha256:2a15a08b17dd94c53a1da0438822d70ebcd13f8c3a95abe3a9ef9f11a94830aa \ + --hash=sha256:2f981d352f04553a7171b8e44369f2af4055f888dfb147d55e42d29e29e74559 \ + --hash=sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa \ + --hash=sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37 \ + --hash=sha256:3537e01efc9d4dccdf77221fb1cb3b8e1a38d5428920e0657ce299b20324d758 \ + --hash=sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f \ + --hash=sha256:38664109c14ffc9e7437e86b4dceb442b0096dfe3541d7864d9cbe1da4cf36c8 \ + --hash=sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d \ + --hash=sha256:3b562dd9e9ea93f13d53989d23a7e775fdfd1066c33494ff43f5418bc8c58a5c \ + --hash=sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97 \ + --hash=sha256:4bd4cd07944443f5a265608cc6aab442e4f74dff8088b0dfc8238647b8f6ae9a \ + --hash=sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19 \ + --hash=sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9 \ + --hash=sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9 \ + --hash=sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc \ + --hash=sha256:591ae9f2a647529ca990bc681daebdd52c8791ff06c2bfa05b65163e28102ef2 \ + --hash=sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4 \ + --hash=sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354 \ + --hash=sha256:6b5420a1d9450023228968e7e6a9ce57f65d148ab56d2313fcd589eee96a7a50 \ + --hash=sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698 \ + --hash=sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9 \ + --hash=sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b \ + --hash=sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc \ + --hash=sha256:7be7b61bb172e1ed687f1754f8e7484f1c8019780f6f6b0786e76bb01c2ae115 \ + --hash=sha256:7c3fb7d25180895632e5d3148dbdc29ea38ccb7fd210aa27acbd1201a1902c6e \ + --hash=sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485 \ + --hash=sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f \ + --hash=sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12 \ + --hash=sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025 \ + --hash=sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009 \ + --hash=sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d \ + --hash=sha256:949b8d66bc381ee8b007cd945914c721d9aba8e27f71959d750a46f7c282b20b \ + --hash=sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a \ + --hash=sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5 \ + --hash=sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f \ + --hash=sha256:a320721ab5a1aba0a233739394eb907f8c8da5c98c9181d1161e77a0c8e36f2d \ + --hash=sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1 \ + --hash=sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287 \ + --hash=sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6 \ + --hash=sha256:bc51efed119bc9cfdf792cdeaa4d67e8f6fcccab66ed4bfdd6bde3e59bfcbb2f \ + --hash=sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581 \ + --hash=sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed \ + --hash=sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b \ + --hash=sha256:c0c0b3ade1c0b13b936d7970b1d37a57acde9199dc2aecc4c336773e1d86049c \ + --hash=sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026 \ + --hash=sha256:c4ffb7ebf07cfe8931028e3e4c85f0357459a3f9f9490886198848f4fa002ec8 \ + --hash=sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676 \ + --hash=sha256:d2ee202e79d8ed691ceebae8e0486bd9a2cd4794cec4824e1c99b6f5009502f6 \ + --hash=sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e \ + --hash=sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d \ + --hash=sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d \ + --hash=sha256:de8a88e63464af587c950061a5e6a67d3632e36df62b986892331d4620a35c01 \ + --hash=sha256:df2449253ef108a379b8b5d6b43f4b1a8e81a061d6537becd5582fba5f9196d7 \ + --hash=sha256:e1c1493fb6e50ab01d20a22826e57520f1284df32f2d8601fdd90b6304601419 \ + --hash=sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795 \ + --hash=sha256:e2103a929dfa2fcaf9bb4e7c091983a49c9ac3b19c9061b6d5427dd7d14d81a1 \ + --hash=sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5 \ + --hash=sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d \ + --hash=sha256:e8fc20152abba6b83724d7ff268c249fa196d8259ff481f3b1476383f8f24e42 \ + --hash=sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe \ + --hash=sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda \ + --hash=sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e \ + --hash=sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737 \ + --hash=sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523 \ + --hash=sha256:f42d0984e947b8adf7dd6dde396e720934d12c506ce84eea8476409563607591 \ + --hash=sha256:f71a396b3bf33ecaa1626c255855702aca4d3d9fea5e051b41ac59a9c1c41edc \ + --hash=sha256:f9e130248f4462aaa8e2552d547f36ddadbeaa573879158d721bbd33dfe4743a \ + --hash=sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50 + # via jinja2 +packaging==26.3 \ + --hash=sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79 \ + --hash=sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c + # via ansible-core +pycparser==3.0 ; implementation_name != 'PyPy' and platform_python_implementation != 'PyPy' \ + --hash=sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29 \ + --hash=sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992 + # via cffi +pyyaml==6.0.3 \ + --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ + --hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a \ + --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ + --hash=sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956 \ + --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ + --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c \ + --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \ + --hash=sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a \ + --hash=sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0 \ + --hash=sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b \ + --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \ + --hash=sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6 \ + --hash=sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7 \ + --hash=sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e \ + --hash=sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007 \ + --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \ + --hash=sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4 \ + --hash=sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9 \ + --hash=sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295 \ + --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \ + --hash=sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0 \ + --hash=sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e \ + --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \ + --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \ + --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \ + --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \ + --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \ + --hash=sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b \ + --hash=sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69 \ + --hash=sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5 \ + --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \ + --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \ + --hash=sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369 \ + --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \ + --hash=sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824 \ + --hash=sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198 \ + --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \ + --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \ + --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \ + --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \ + --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \ + --hash=sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b \ + --hash=sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00 \ + --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \ + --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \ + --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \ + --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \ + --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \ + --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \ + --hash=sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4 \ + --hash=sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b \ + --hash=sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf \ + --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \ + --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \ + --hash=sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8 \ + --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \ + --hash=sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da \ + --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \ + --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ + --hash=sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c \ + --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \ + --hash=sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f \ + --hash=sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917 \ + --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \ + --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \ + --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \ + --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \ + --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \ + --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \ + --hash=sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3 \ + --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \ + --hash=sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926 \ + --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 + # via ansible-core +resolvelib==1.2.1 \ + --hash=sha256:7d08a2022f6e16ce405d60b68c390f054efcfd0477d4b9bd019cc941c28fad1c \ + --hash=sha256:fb06b66c8da04172d9e72a21d7d06186d8919e32ae5ab5cdf5b9d920be805ac2 + # via ansible-core diff --git a/cloud-init/tests/lint.py b/cloud-init/tests/lint.py new file mode 100755 index 0000000..38b9525 --- /dev/null +++ b/cloud-init/tests/lint.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python3 +"""Check the invariants cloud-init/README.md promises for a user-data file. + +Usage: lint.py (normally via `make lint-cloud-init`) + +`cloud-init schema` checks the file's shape, including the `#cloud-config` header. This +script checks what the schema cannot: + * no secret anywhere: no key whose name looks secret-bearing (RPC URL, password, + token, keystore contents, private key), no credentials embedded in a URL, and only + the known keys in bootstrap.env; + * the node is installed from the signed release, and its wallet is generated on the + host; + * the inventory puts localhost in decdn_nodes (site.yml's host pattern) with a local + connection, and names a keyed admin account (baseline's lockout guard); + * stage 1 (the embedded decdn-bootstrap script) passes shellcheck and is what + runcmd runs; + * cloud-init/collections.lock.yml pins every collection in ansible/requirements.yml, + at a version inside its range. + +Exit 0 when all of that holds, 1 on a violated invariant (the message says which), and +2 when the file cannot be read or parsed. +""" + +import json +import re +import shutil +import subprocess +import sys +from pathlib import Path + +REPO = Path(__file__).resolve().parents[2] +ENV_PATH = "/etc/decdn-bootstrap/bootstrap.env" +INVENTORY_PATH = "/etc/decdn-bootstrap/inventory.yml" +STAGE1_PATH = "/usr/local/sbin/decdn-bootstrap" +ENV_KEYS = {"DEVOPS_REPO", "DEVOPS_REF", "DECDN_BOOTSTRAP_ANSIBLE_ARGS"} + +# A key whose NAME suggests it carries a secret. Public-key material is fine +# (baseline_sudo_users[].keys), so match whole words, not "key" alone. +SECRET_KEY = re.compile(r"passw|secret|token|rpc_url|private|api_?key|keystore|chpasswd", re.I) +# Named knobs that match SECRET_KEY but carry no secret. +SECRET_KEY_ALLOW = {"decdn_node_generate_keystore"} +# user:password@ (or token@) in any URL. +URL_CREDENTIALS = re.compile(r"[a-z][a-z0-9+.-]*://[^/\s@]+@", re.I) + +errors = [] + + +class ParseError(Exception): + pass + + +def load_yaml(text): + """Parse YAML via yq (mikefarah v4), as the repo's other checks do; no PyYAML needed.""" + if not shutil.which("yq"): + print("lint.py: needs yq (mikefarah v4)", file=sys.stderr) + sys.exit(2) + r = subprocess.run(["yq", "-p=yaml", "-o=json", "."], input=text, text=True, + capture_output=True, check=False) + if r.returncode != 0: + raise ParseError(r.stderr.strip()) + return json.loads(r.stdout) if r.stdout.strip() else None + + +def violation(msg): + errors.append(msg) + + +def walk_keys(node, path=""): + """Yield (dotted path, key) for every mapping key in a parsed YAML tree.""" + if isinstance(node, dict): + for k, v in node.items(): + p = f"{path}.{k}" if path else str(k) + yield p, str(k) + yield from walk_keys(v, p) + elif isinstance(node, list): + for i, v in enumerate(node): + yield from walk_keys(v, f"{path}[{i}]") + + +def version_tuple(v): + return tuple(int(x) for x in re.findall(r"\d+", v)) + + +def satisfies(version, spec): + """Check a Galaxy-style spec made of comma-separated >=, >, <=, <, ==, != and *.""" + ops = { + ">=": lambda a, b: a >= b, ">": lambda a, b: a > b, + "<=": lambda a, b: a <= b, "<": lambda a, b: a < b, + "==": lambda a, b: a == b, "!=": lambda a, b: a != b, + } + for part in str(spec).split(","): + part = part.strip() + if part in ("", "*"): + continue + m = re.fullmatch(r"(>=|<=|==|!=|>|<)?\s*(\S+)", part) + op, ver = (m.group(1) or "=="), m.group(2) + if not ops[op](version_tuple(version), version_tuple(ver)): + return False + return True + + +def check_lock(): + try: + wanted = load_yaml((REPO / "ansible/requirements.yml").read_text())["collections"] + lock = load_yaml((REPO / "cloud-init/collections.lock.yml").read_text())["collections"] + except (OSError, ParseError, KeyError, TypeError) as e: + print(f"lint.py: cannot read the collection requirements or lock: {e}", file=sys.stderr) + sys.exit(2) + pinned = {c["name"]: str(c.get("version", "")) for c in lock} + for name, v in pinned.items(): + if not re.fullmatch(r"\d+\.\d+\.\d+", v): + violation(f"collections.lock.yml: {name} must be pinned to an exact version, not {v!r}") + for c in wanted: + name, spec = c["name"], c.get("version", "*") + if name not in pinned: + violation(f"collections.lock.yml does not pin {name} (listed in ansible/requirements.yml)") + elif not satisfies(pinned[name], spec): + violation(f"collections.lock.yml pins {name} {pinned[name]}, outside ansible/requirements.yml's {spec}") + + +def main(): + if len(sys.argv) != 2: + print(__doc__, file=sys.stderr) + sys.exit(2) + path = Path(sys.argv[1]) + try: + raw = path.read_text() + doc = load_yaml(raw) + except (OSError, ParseError) as e: + print(f"lint.py: cannot parse {path}: {e}", file=sys.stderr) + sys.exit(2) + if not isinstance(doc, dict): + print(f"lint.py: {path} is not a YAML mapping", file=sys.stderr) + sys.exit(2) + + files = {f.get("path"): f for f in doc.get("write_files") or [] if isinstance(f, dict)} + for p in (ENV_PATH, INVENTORY_PATH, STAGE1_PATH): + if p not in files: + violation(f"write_files has no {p}") + for p in (ENV_PATH, INVENTORY_PATH): + if p in files and str(files[p].get("permissions")) != "0600": + violation(f"{p} must be written 0600") + + # --- No secrets --------------------------------------------------------------- + try: + inventory = load_yaml(files.get(INVENTORY_PATH, {}).get("content", "")) or {} + except ParseError as e: + print(f"lint.py: the embedded {INVENTORY_PATH} does not parse: {e}", file=sys.stderr) + sys.exit(2) + for where, tree in (("user-data", doc), ("inventory", inventory)): + for p, k in walk_keys(tree): + if SECRET_KEY.search(k) and k not in SECRET_KEY_ALLOW: + violation(f"{where}: {p} looks secret-bearing; secrets never go in user-data") + for i, line in enumerate(raw.splitlines(), 1): + if URL_CREDENTIALS.search(line): + violation(f"line {i}: a URL with embedded credentials") + if "DECDN_RPC_URL" in line and not line.lstrip().startswith("#"): + violation(f"line {i}: DECDN_RPC_URL belongs in /etc/decdn/decdn.env on the host, not in user-data") + + env = {} + for line in files.get(ENV_PATH, {}).get("content", "").splitlines(): + if line.strip() and not line.lstrip().startswith("#"): + k, _, v = line.partition("=") + env[k.strip()] = v.strip() + for k in env.keys() - ENV_KEYS: + violation(f"bootstrap.env: unexpected key {k} (allowed: {', '.join(sorted(ENV_KEYS))})") + if not env.get("DEVOPS_REPO", "").startswith("https://"): + violation("bootstrap.env: DEVOPS_REPO must be an https:// URL") + + # --- The node and the inventory ------------------------------------------------- + group = inventory.get("decdn_nodes") or {} + host = (group.get("hosts") or {}).get("localhost") + if not isinstance(host, dict) or host.get("ansible_connection") != "local": + violation("inventory: localhost must be in decdn_nodes with ansible_connection: local") + iv = group.get("vars") or {} + if iv.get("decdn_node_install_method") != "release": + violation("inventory: decdn_node_install_method must be release (the only method that needs no control machine)") + if iv.get("decdn_node_generate_keystore") is not True: + violation("inventory: decdn_node_generate_keystore must be true (the wallet is generated on the host)") + if iv.get("decdn_verify_release_signature", True) is not True: + violation("inventory: decdn_verify_release_signature must not be turned off") + users = iv.get("baseline_sudo_users") or [] + if not users or not all(isinstance(u, dict) and u.get("name") and u.get("keys") for u in users): + violation("inventory: baseline_sudo_users needs at least one named account, each with keys (lockout guard)") + + # --- Stage 1 -------------------------------------------------------------------- + runcmd = [c if isinstance(c, str) else " ".join(map(str, c)) for c in doc.get("runcmd") or []] + if not any(STAGE1_PATH in c for c in runcmd): + violation(f"runcmd does not run {STAGE1_PATH}") + stage1 = files.get(STAGE1_PATH, {}).get("content", "") + if stage1: + if not shutil.which("shellcheck"): + print("lint.py: needs shellcheck", file=sys.stderr) + sys.exit(2) + sc = subprocess.run(["shellcheck", "-s", "bash", "-"], input=stage1, text=True, + capture_output=True, check=False) + if sc.returncode != 0: + violation(f"{STAGE1_PATH} fails shellcheck:\n{sc.stdout}") + + check_lock() + + for e in errors: + print(f" - {e}", file=sys.stderr) + if errors: + print(f"{path} violates an invariant (see cloud-init/tests/lint.py)", file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/cloud-init/user-data.yaml b/cloud-init/user-data.yaml new file mode 100644 index 0000000..4035316 --- /dev/null +++ b/cloud-init/user-data.yaml @@ -0,0 +1,134 @@ +#cloud-config +# deCDN node bootstrap for a fresh Debian 12/13 or Ubuntu 24.04/26.04 VM (x86_64 or +# aarch64). Paste this into your provider's "user data" / "cloud-init" field when creating +# the server. The full walkthrough is in cloud-init/README.md of github.com/decdn/devops. +# +# Edit every value marked MUST-EDIT first. The bootstrap refuses to run while any +# CHANGE_ME is left. +# +# NEVER PUT A SECRET IN THIS FILE. Any local process can read user-data from the instance +# metadata service, and your provider keeps a copy in its console and API. That rules out +# the RPC URL (it may embed an API key), passwords, keystores and tokens. Everything below +# is public: an SSH *public* key, a version, a region. The RPC URL is written over SSH after +# the first boot, and the node's wallet is generated on the host. +# +# On first boot the host clones github.com/decdn/devops at DEVOPS_REF, installs a pinned +# Ansible, and hardens itself (SSH, firewall, automatic patching). It then stops and waits +# for /etc/decdn/decdn.env. `cloud-init status --wait` returns when it is done. +--- +package_update: true +packages: + - git + - python3-venv + - ca-certificates + - sudo + +write_files: + # Which revision of github.com/decdn/devops the host runs. Pin a full 40-character + # commit SHA (verified after checkout) or a release tag. Branch names are refused. + - path: /etc/decdn-bootstrap/bootstrap.env + owner: root:root + permissions: "0600" + content: | + DEVOPS_REPO=https://github.com/decdn/devops.git + DEVOPS_REF=CHANGE_ME + DECDN_BOOTSTRAP_ANSIBLE_ARGS= + + # The Ansible inventory for this host. Everything the playbook needs that is not a + # secret lives here. More knobs: ansible/roles/*/defaults/main.yml. + - path: /etc/decdn-bootstrap/inventory.yml + owner: root:root + permissions: "0600" + content: | + --- + decdn_nodes: + hosts: + localhost: + ansible_connection: local + ansible_python_interpreter: /usr/bin/python3 + vars: + # Your admin account. Hardening disables root and password SSH logins, so this + # account and its key are how you log back in. Many providers inject their key + # for root only, so list the key here too. + baseline_sudo_users: + - name: CHANGE_ME # MUST-EDIT: login name, e.g. "alice" + keys: + - "CHANGE_ME" # MUST-EDIT: your SSH public key, "ssh-ed25519 AAAA… you@laptop" + # Under cloud-init the "runner" is root on this host, not you, so there is + # nobody to auto-detect. + baseline_sudo_autodetect_runner: false + # Optional: accept SSH only from these CIDRs. + # ssh_allow_cidrs: ["203.0.113.0/24"] + + # The node, installed from the GPG-verified release tarballs. + decdn_node_install_method: release + decdn_node_version: "CHANGE_ME" # MUST-EDIT: an upstream release, e.g. "0.2.0" + # Until upstream publishes a release, point this at a mirror that serves + # v/{decdn-node,decdn}--.tar.gz, SHA256SUMS and + # SHA256SUMS.asc. The signature is still checked against deCDN's release key. + # decdn_node_release_base: "https://mirror.example/decdn/releases/download" + decdn_node_generate_keystore: true # the wallet is generated on the host + decdn_network: arbitrum-sepolia + decdn_region: "CHANGE_ME" # MUST-EDIT: ISO 3166-1 alpha-2 of the VM's location, e.g. "DE" + + # Stage 1: fetch and verify the pinned checkout, then hand over to its + # cloud-init/bootstrap.sh. Re-run it by hand after writing /etc/decdn/decdn.env, or + # after changing DEVOPS_REF: `sudo decdn-bootstrap`. + - path: /usr/local/sbin/decdn-bootstrap + owner: root:root + permissions: "0755" + content: | + #!/usr/bin/env bash + # Stage 1 of the deCDN cloud-init bootstrap (cloud-init/README.md in decdn/devops). + set -euo pipefail + conf=/etc/decdn-bootstrap/bootstrap.env + checkout=/opt/decdn-devops + die() { echo "decdn-bootstrap: $*" >&2; exit 1; } + + [[ $EUID -eq 0 ]] || die "run as root (sudo decdn-bootstrap)" + [[ $(stat -c '%u %a' "$conf") == "0 600" ]] || die "$conf must be owned by root, mode 0600" + # KEY=value lines, read literally: never sourced, so a value is data, not shell. + DEVOPS_REPO='' DEVOPS_REF='' DECDN_BOOTSTRAP_ANSIBLE_ARGS='' + while IFS= read -r line || [[ -n $line ]]; do + [[ $line =~ ^[[:space:]]*(#|$) ]] && continue + [[ $line =~ ^([A-Z_]+)=(.*)$ ]] || die "$conf: not a KEY=value line: $line" + case ${BASH_REMATCH[1]} in + DEVOPS_REPO) DEVOPS_REPO=${BASH_REMATCH[2]} ;; + DEVOPS_REF) DEVOPS_REF=${BASH_REMATCH[2]} ;; + DECDN_BOOTSTRAP_ANSIBLE_ARGS) DECDN_BOOTSTRAP_ANSIBLE_ARGS=${BASH_REMATCH[2]} ;; + *) die "$conf: unknown key ${BASH_REMATCH[1]}" ;; + esac + done <"$conf" + [[ -n $DEVOPS_REPO ]] || die "DEVOPS_REPO is empty in $conf" + [[ -n $DEVOPS_REF && $DEVOPS_REF != CHANGE_ME ]] \ + || die "set DEVOPS_REF in $conf to a commit SHA or release tag of $DEVOPS_REPO" + export DECDN_BOOTSTRAP_ANSIBLE_ARGS + export HOME="${HOME:-/root}" + + if [[ ! -d $checkout/.git ]]; then + git clone --quiet --no-checkout "$DEVOPS_REPO" "$checkout" + fi + git -C "$checkout" remote set-url origin "$DEVOPS_REPO" + git -C "$checkout" fetch --quiet --force --tags origin + + if [[ $DEVOPS_REF =~ ^[0-9a-f]{40}$ ]]; then + want=$DEVOPS_REF + git -C "$checkout" cat-file -e "$want^{commit}" 2>/dev/null \ + || git -C "$checkout" fetch --quiet origin "$want" + elif want=$(git -C "$checkout" rev-parse -q --verify "refs/tags/$DEVOPS_REF^{commit}"); then + echo "decdn-bootstrap: tag $DEVOPS_REF is commit $want" + else + die "DEVOPS_REF=$DEVOPS_REF is neither a full 40-character commit SHA nor a tag of $DEVOPS_REPO" + fi + git -C "$checkout" -c advice.detachedHead=false checkout --quiet --force --detach "$want" + [[ $(git -C "$checkout" rev-parse HEAD) == "$want" ]] || die "checkout is not at $want" + echo "decdn-bootstrap: running $DEVOPS_REPO at $want" + exec "$checkout/cloud-init/bootstrap.sh" + +runcmd: + - [/usr/local/sbin/decdn-bootstrap] + +final_message: >- + deCDN bootstrap finished after $UPTIME s. `cat /var/lib/decdn-bootstrap/state` shows + where it stopped; "awaiting-secret" means SSH in, write /etc/decdn/decdn.env and run + `sudo decdn-bootstrap` (cloud-init/README.md). diff --git a/compose/README.md b/compose/README.md index 27cada9..ba0dec0 100644 --- a/compose/README.md +++ b/compose/README.md @@ -8,7 +8,7 @@ Pick this path for a single machine you already run Docker on. For a fleet, or a you want hardened from scratch (firewall, SSH, auto-patching), use the [Ansible project](../ansible/README.md); on Kubernetes, the [Helm chart](../charts/decdn-node/README.md). [`docs/requirements.md`](../docs/requirements.md) -compares the three. +compares the paths. > **Upstream has not published a release yet**, so there is no signed image to pin. > `compose.yaml` only takes an image by digest, and a locally built image has no diff --git a/docs/requirements.md b/docs/requirements.md index f32d4f1..fe6105f 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -1,25 +1,27 @@ # Requirements and choosing a deploy path -What a deCDN node needs from its host and network, and which of this repo's three +What a deCDN node needs from its host and network, and which of this repo's four deploy paths fits. Protocol facts (bond sizing, fees) are not here: they come from the deCDN ADRs. ## Choosing a path -| | Ansible | Docker Compose | Helm | -|---|---|---|---| -| Target | VMs, bare metal | one host running Docker | Kubernetes | -| Host hardening (firewall, SSH, patching) | yes, `baseline` | no, yours | no, the cluster's | -| Fleets | yes, one inventory | one host per compose project | one release per node | -| Secrets | host file or git-ignored inventory | host file | operator-created Secrets | -| Install source | signed release tarball, or local build | image by digest (enforced) | image by digest (recommended) or tag | -| Chain config | `decdn_network` profile | `decdn config init --chain` | explicit values | -| Monitoring | opt-in Grafana Cloud agent | bring your own | ServiceMonitor, PrometheusRule, dashboards | -| Backup / decommission | `make backup` / `make decommission` | manual commands | PVC snapshot | -| Guide | [ansible/README.md](../ansible/README.md) | [compose/README.md](../compose/README.md) | [charts/decdn-node/README.md](../charts/decdn-node/README.md) | +| | Ansible | cloud-init | Docker Compose | Helm | +|---|---|---|---|---| +| Target | VMs, bare metal | one cloud VM, no control machine | one host running Docker | Kubernetes | +| Host hardening (firewall, SSH, patching) | yes, `baseline` | yes, `baseline` | no, yours | no, the cluster's | +| Fleets | yes, one inventory | one user-data per VM | one host per compose project | one release per node | +| Secrets | host file or git-ignored inventory | host file, written over SSH | host file | operator-created Secrets | +| Install source | signed release tarball, or local build | signed release tarball | image by digest (enforced) | image by digest (recommended) or tag | +| Chain config | `decdn_network` profile | `decdn_network` profile | `decdn config init --chain` | explicit values | +| Monitoring | opt-in Grafana Cloud agent | opt-in Grafana Cloud agent (token in a host file) | bring your own | ServiceMonitor, PrometheusRule, dashboards | +| Backup / decommission | `make backup` / `make decommission` | the Ansible targets, from a workstation inventory | manual commands | PVC snapshot | +| Guide | [ansible/README.md](../ansible/README.md) | [cloud-init/README.md](../cloud-init/README.md) | [compose/README.md](../compose/README.md) | [charts/decdn-node/README.md](../charts/decdn-node/README.md) | If you are unsure: a VPS or dedicated server you control end to end is the Ansible -path. It is the only one that hardens the host as well as installing the node. +path, or cloud-init for a single VM when you would rather not run Ansible from a +workstation. They are the only two that harden the host as well as installing the node, +and they run the same roles. ## Platforms @@ -28,6 +30,7 @@ path. It is the only one that hardens the host as well as installing the node. | OS | Debian 12 (bookworm), Debian 13 (trixie), Ubuntu 24.04 (noble), Ubuntu 26.04 (resolute) | molecule converges `decdn_node` on all four in systemd containers; `grafana_alloy`'s install path on Debian 12 only (its disabled path on all four); `baseline` on real hosts | | Architecture | x86_64, aarch64 | upstream builds both; the Ansible role derives the target from the host | | Ansible (control machine) | ansible-core ≥ 2.15 | CI runs the current release | +| cloud-init | the provider image's own; the bootstrap installs its pinned ansible-core on the host | the user-data is booted with the distro's cloud-init in Debian 12 and Ubuntu 26.04 containers, to a running node | | Kubernetes | ≥ 1.25 | rendered and validated with kubeconform against 1.30 | | Docker Compose | v2 with `env_file.required` support (2.24+) | rendered in CI | diff --git a/tests/scripts-test.sh b/tests/scripts-test.sh index 10884b3..1d0864b 100755 --- a/tests/scripts-test.sh +++ b/tests/scripts-test.sh @@ -1,9 +1,10 @@ #!/usr/bin/env bash # Tests for the repo's own guard rails that no molecule scenario or chart render # exercises: the ansible/ Makefile's scoping guards, the release gate, the -# lint-compose invariants (negative cases), and — with UPSTREAM= — -# the upstream-mirror generators' exit codes. `make test-scripts` runs it; CI's -# `scripts` job does too. Needs make, docker (compose v2), jq. +# lint-compose and lint-cloud-init invariants (negative cases), and — with +# UPSTREAM= — the upstream-mirror generators' exit codes. +# `make test-scripts` runs it; CI's `scripts` job does too. Needs make, docker +# (compose v2), jq; the cloud-init cases also need cloud-init, shellcheck and yq. set -euo pipefail repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" @@ -76,6 +77,39 @@ variant "capabilities kept" 's/^(\s*)cap_drop: \[ALL\]$/\1cap_drop: [NET_RAW variant "published port" 's/^(\s*)network_mode: host$/\1ports: ["127.0.0.1:9090:9090"]/' variant "tag instead of digest" 's#^(\s*)image: .*#\1image: ghcr.io/decdn/decdn-node:latest#' +# --- lint-cloud-init negatives: each broken variant must be rejected ------------------- +if command -v cloud-init >/dev/null; then + userdata="$repo/cloud-init/user-data.yaml" + expect 0 "lint-cloud-init accepts cloud-init/user-data.yaml" make -s -C "$repo" lint-cloud-init + sed '1s/^#cloud-config$/# cloud-config/' "$userdata" > "$work/ci-header.yaml" + if make -s -C "$repo" lint-cloud-init CLOUD_INIT_FILE="$work/ci-header.yaml" >"$work/out" 2>&1 \ + || ! grep -q 'is not a valid cloud-config' "$work/out"; then + cat "$work/out" >&2; fail "lint-cloud-init did not reject a missing #cloud-config header as invalid" + fi + pass "lint-cloud-init rejects: no #cloud-config header (schema)" + ci_variant() { # + sed -E "$2" "$userdata" > "$work/ci-$1.yaml" + cmp -s "$userdata" "$work/ci-$1.yaml" && fail "variant $1 did not change user-data.yaml" + if make -s -C "$repo" lint-cloud-init CLOUD_INIT_FILE="$work/ci-$1.yaml" >"$work/out" 2>&1; then + fail "lint-cloud-init accepted: $1" + fi + grep -q 'violates an invariant' "$work/out" || { cat "$work/out" >&2; fail "lint-cloud-init failed for another reason: $1"; } + pass "lint-cloud-init rejects: $1" + } + ci_variant "RPC URL in bootstrap.env" 's#^(\s*)DECDN_BOOTSTRAP_ANSIBLE_ARGS=$#\1DECDN_RPC_URL=https://rpc.example/key#' + ci_variant "RPC URL in the inventory" 's#^(\s*)decdn_network: arbitrum-sepolia$#\1decdn_rpc_url: "https://rpc.example/"#' + ci_variant "credentials in a URL" 's#^(\s*)DEVOPS_REPO=https://#\1DEVOPS_REPO=https://user:pw@#' + ci_variant "manual install method" 's/^(\s*)decdn_node_install_method: release$/\1decdn_node_install_method: manual/' + ci_variant "no host-generated wallet" 's/^(\s*)decdn_node_generate_keystore: true(.*)$/\1decdn_node_generate_keystore: false\2/' + ci_variant "localhost outside decdn_nodes" 's/^(\s*)decdn_nodes:$/\1decdn_hosts:/' + ci_variant "admin account without keys" '/^\s*keys:$/,+1d' + ci_variant "stage 1 not run" 's#^ - \[/usr/local/sbin/decdn-bootstrap\]$# - [/bin/true]#' + # shellcheck disable=SC2016 # a literal $DEVOPS_REPO: the variant unquotes it in stage 1 + ci_variant "shellcheck-dirty stage 1" 's#git clone --quiet --no-checkout "\$DEVOPS_REPO"#git clone --quiet --no-checkout $DEVOPS_REPO#' +else + skipped+=("lint-cloud-init negatives (needs cloud-init on PATH; CI installs it)") +fi + # --- upstream-mirror generators (optional: needs a decdn/decdn checkout) -------------- if [[ -n "${UPSTREAM:-}" ]]; then expect 0 "network profiles current" "$repo/scripts/sync-network-profiles.py" "$UPSTREAM" --check From 7b666c17f3f8b3cc57a7257e93bf950a37a6b3f7 Mon Sep 17 00:00:00 2001 From: Ant Somers Date: Thu, 24 Sep 2026 06:08:00 +0300 Subject: [PATCH 2/2] fix(cloud-init): close lint bypasses and record every failed run Address the Copilot review and a four-agent review (code, tests, silent failures, docs) of #73. Lint (cloud-init/tests/lint.py): - write_files may write only the bootstrap's own four files, as plain text. Secrets in /etc/decdn/decdn.env, /etc/grafana-alloy.env or b64 content no longer pass. - Reject decdn_extra_env, and any NAME=value assignment of a secret-looking variable in file contents or commands. - The trust knobs (install method, keystore generation, signature verification) may be set only in decdn_nodes.vars, where they are checked. A host var no longer overrides them. decdn_release_keyring and decdn_env_file are refused, and the inventory may hold only the decdn_nodes group. - runcmd must be exactly stage 1, so `|| true` cannot mask a failure. - No mention of the test-only baseline switch anywhere. - Allowlist non-secret knobs with secret-looking names (baseline_sudo_passwordless, keystore/secret file paths). Bootstrap: - Drop DECDN_BOOTSTRAP_ANSIBLE_ARGS: user-data carried free-form ansible arguments (extra-vars, --skip-tags baseline). Molecule now uses a fixed test-only marker file that bootstrap.sh honours and the lint rejects. - Stage 1 takes a lock and records "running". An EXIT trap plus signal traps in both stages record "failed" on any non-zero exit, including stage-1 refusals and a dropped SSH session. - The login hint is now written by the user-data, so it exists even if stage 1 fails. It reports a "running" state whose process is gone as interrupted. - Refuse to run phase 1 unless `--tags baseline` still selects baseline tasks, so a renamed tag cannot report an unhardened host as hardened. Tests: - Every lint-cloud-init negative case now names the invariant it expects: 22 variants, plus a positive case for the allowlisted knobs. - The harness fails instead of skipping in CI when cloud-init is missing, and make lint-cloud-init reports a missing cloud-init as such. - Molecule checks that stage 1 refuses a branch name, the placeholder ref, an unknown key and a loose mode, each recording "failed" with the hint saying so and the checkout unmoved. It then upgrades to a tag on a newer commit before the secret run. Docs: fix the boot order, hint behaviour, admin login timing, the wallet address (via `decdn whoami`), tag mutability, the collection bump procedure and several stale comments. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/ci.yml | 3 +- AGENTS.md | 2 +- CONTRIBUTING.md | 4 +- Makefile | 17 ++- ansible/.ansible-lint | 5 + ansible/molecule/cloud-init/converge.yml | 19 ++- .../molecule/cloud-init/includes/refuse.yml | 47 +++++++ ansible/molecule/cloud-init/molecule.yml | 23 ++-- ansible/molecule/cloud-init/prepare.yml | 10 +- ansible/molecule/cloud-init/side_effect.yml | 63 ++++++++- ansible/molecule/cloud-init/verify.yml | 18 ++- cloud-init/README.md | 91 +++++++----- cloud-init/bootstrap.sh | 66 ++++----- cloud-init/collections.lock.yml | 3 +- cloud-init/requirements.in | 2 +- cloud-init/tests/lint.py | 130 +++++++++++++----- cloud-init/user-data.yaml | 49 ++++++- tests/scripts-test.sh | 53 +++++-- 18 files changed, 462 insertions(+), 143 deletions(-) create mode 100644 ansible/molecule/cloud-init/includes/refuse.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c575b00..4b4ca3e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -212,7 +212,8 @@ jobs: # promises (no secrets, release install, host-generated wallet, localhost in # decdn_nodes, a keyed admin account, shellcheck-clean stage 1, a collection lock # covering ansible/requirements.yml). The boot itself is the molecule `cloud-init` - # scenario (molecule.yml). Runs only when cloud-init/ or the root Makefile changed. + # scenario (molecule.yml). Runs only when cloud-init/, ansible/requirements.yml, the + # root Makefile or ci.yml changed. cloud-init: needs: changes if: needs.changes.outputs.cloud-init == 'true' diff --git a/AGENTS.md b/AGENTS.md index c0227b3..bff92cf 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -143,7 +143,7 @@ addresses) the repo carries, and they carry their upstream commit. `ansible/requirements.yml` changes, re-sync the lock: `make lint-cloud-init` checks it covers the requirements. The molecule `cloud-init` scenario boots the real user-data through cloud-init (skipping `baseline`) against a locally signed release - mirror, and is the suite's only release-mode coverage. + mirror, and is the suite's only coverage of the release download and verify path. - **`compose/`** — the same node under Docker Compose on one host: the upstream image, always by digest (`compose.yaml` builds `DECDN_IMAGE_REPO@DECDN_IMAGE_DIGEST`), the diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 64c4ba7..0ab117b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -33,7 +33,7 @@ targets, so a local pass means a CI pass. Deploy targets live in | `make lint-helm` | chart: `helm lint --strict`, positive/negative render tests, kubeconform (digest-pinned image), the shared schema-key check (needs `helm`, `yq`, `python3` ≥ 3.11, Docker). Set `DECDN_CLI=` to also run the real `decdn config validate` (CI can't). | | `make lint-alloy` | renders `roles/grafana_alloy`'s templates and validates them with the **real** digest-pinned Alloy binary. The molecule stub exits 0 for everything, so this is the only gate that proves the config loads. `ALLOY_BIN=` skips the download. | | `make lint-compose` | renders `compose/compose.yaml` with its example env and asserts its security invariants | -| `make lint-cloud-init` | `cloud-init schema` on `cloud-init/user-data.yaml`, then `cloud-init/tests/lint.py`: no secrets, `release` install with a host-generated wallet, localhost in `decdn_nodes`, a keyed admin account, a shellcheck-clean stage 1, and a collection lock that covers `ansible/requirements.yml` (needs `cloud-init`, `shellcheck`, `yq`). `CLOUD_INIT_FILE=` checks your own filled-in copy. | +| `make lint-cloud-init` | `cloud-init schema` on `cloud-init/user-data.yaml`, then `cloud-init/tests/lint.py`: no secrets (only the bootstrap's own files, no secret-looking keys or assignments), no hardening skip, a `release` install verified against the vendored key with a host-generated wallet (trust knobs only in `decdn_nodes.vars`), localhost in `decdn_nodes`, a keyed admin account, `runcmd` exactly stage 1, shellcheck-clean scripts, and a collection lock that covers `ansible/requirements.yml` (needs `cloud-init`, `shellcheck`, `yq`). `CLOUD_INIT_FILE=` checks your own filled-in copy. | | `make test-scripts` | `tests/scripts-test.sh`: the `ansible/Makefile` scoping guards (dry runs), the release gate, and the negative cases of `lint-compose` and `lint-cloud-init` (the latter skipped without `cloud-init` on PATH). `UPSTREAM=` adds the sync generators' exit codes. | | `make security` | KICS IaC scan of `ansible/`, the rendered chart and `compose/` (digest-pinned engine, fail on HIGH) | | `make galaxy-check` | build the `decdn.node` collection and run galaxy-importer's checks | @@ -111,7 +111,7 @@ reports the two differently. the four `setup-helm` `version:` inputs (`ci.yml`'s `helm` and `kics` jobs, both jobs in `release.yml`); the collection versions in `ansible/requirements.yml`, and their exact pins in `cloud-init/collections.lock.yml` (the full transitive set, from - a `make deps` resolve); ansible-core in `cloud-init/requirements.in`, followed by a + a fresh `make deps` resolve); ansible-core in `cloud-init/requirements.in`, followed by a recompile of the hash-locked `requirements.txt` (command in `cloud-init/README.md`); and the local yamllint hook's `additional_dependencies` pin. - **Bump the `cache-epoch:` counter in `ansible/requirements.yml` to make CI diff --git a/Makefile b/Makefile index 9be7175..aff96ae 100644 --- a/Makefile +++ b/Makefile @@ -92,12 +92,15 @@ lint-compose: ## render compose/ with its examples and check its security @echo "compose invariants hold" # The cloud-init user-data (cloud-init/README.md): `cloud-init schema` for its shape, then -# cloud-init/tests/lint.py for what a schema cannot see. That covers no secrets, release -# install with a host-generated wallet, localhost in decdn_nodes, a keyed admin account, -# a shellcheck-clean stage 1, and a collection lock that covers ansible/requirements.yml. -# CLOUD_INIT_FILE is overridable so tests/scripts-test.sh can feed it broken variants. +# cloud-init/tests/lint.py for what a schema cannot see. That covers no secrets, no +# hardening skip, a signed release install with a host-generated wallet, localhost in +# decdn_nodes, a keyed admin account, runcmd exactly stage 1, shellcheck-clean scripts, +# and a collection lock that covers ansible/requirements.yml. +# CLOUD_INIT_FILE is overridable so operators can check their filled-in copy and +# tests/scripts-test.sh can feed it broken variants. CLOUD_INIT_FILE ?= cloud-init/user-data.yaml lint-cloud-init: ## schema-check cloud-init/user-data.yaml and its invariants (needs cloud-init, shellcheck, yq) + @command -v cloud-init >/dev/null || { echo "lint-cloud-init: needs cloud-init on PATH" >&2; exit 2; } @cloud-init schema -c '$(CLOUD_INIT_FILE)' >/dev/null 2>&1 \ || { cloud-init schema -c '$(CLOUD_INIT_FILE)' 2>&1 | grep -v WARNING >&2; \ echo "lint-cloud-init: $(CLOUD_INIT_FILE) is not a valid cloud-config (see above)" >&2; exit 2; } @@ -105,9 +108,9 @@ lint-cloud-init: ## schema-check cloud-init/user-data.yaml and its invariant @echo "cloud-init invariants hold" # The guard rails nothing else exercises: ansible/Makefile's scoping guards, the -# release gate, lint-compose's negative cases, and (with UPSTREAM=) -# the upstream-mirror generators' exit codes. CI job `scripts`. -test-scripts: ## test the Makefile guards, release gate and lint-compose negatives (needs docker, jq) +# release gate, the lint-compose and lint-cloud-init negative cases, and (with +# UPSTREAM=) the upstream-mirror generators' exit codes. CI job `scripts`. +test-scripts: ## test the Makefile guards, release gate, lint-compose and lint-cloud-init negatives (needs docker, jq, cloud-init, yq) tests/scripts-test.sh lint-helm: ## helm lint + render tests + kubeconform + shared schema-key check (needs helm, yq, python3>=3.11, docker) diff --git a/ansible/.ansible-lint b/ansible/.ansible-lint index d60205f..3ad8629 100644 --- a/ansible/.ansible-lint +++ b/ansible/.ansible-lint @@ -17,3 +17,8 @@ skip_list: exclude_paths: - build/ - galaxy/ + +# Task files a molecule scenario includes (ansible-lint would otherwise read them as +# playbooks, and a `tasks/` directory would make the scenario look like a role). +kinds: + - tasks: "**/molecule/*/includes/*.yml" diff --git a/ansible/molecule/cloud-init/converge.yml b/ansible/molecule/cloud-init/converge.yml index 32ddf04..3967f8e 100644 --- a/ansible/molecule/cloud-init/converge.yml +++ b/ansible/molecule/cloud-init/converge.yml @@ -54,8 +54,6 @@ replace: '\1DEVOPS_REPO=file:///srv/devops-src' - regexp: '^(\s*)DEVOPS_REF=CHANGE_ME$' replace: '\1DEVOPS_REF={{ src_sha }}' - - regexp: '^(\s*)DECDN_BOOTSTRAP_ANSIBLE_ARGS=$' - replace: '\1DECDN_BOOTSTRAP_ANSIBLE_ARGS=--skip-tags baseline' - regexp: '^(\s*)- name: CHANGE_ME\b.*$' replace: '\1- name: molecule' - regexp: '^(\s*)- "CHANGE_ME"(.*)$' @@ -79,6 +77,23 @@ changed_when: false failed_when: leftover.rc != 1 + - name: Create the bootstrap's config directory + ansible.builtin.file: + path: /etc/decdn-bootstrap + state: directory + owner: root + group: root + mode: "0755" + + # Outside the user-data on purpose: the lint rejects a user-data that mentions it. + - name: Skip baseline in the container (bootstrap.sh's test-only switch) + ansible.builtin.copy: + dest: /etc/decdn-bootstrap/TEST-ONLY-skip-baseline + content: "" + owner: root + group: root + mode: "0600" + - name: Write the NoCloud meta-data ansible.builtin.copy: dest: "{{ seed }}/meta-data" diff --git a/ansible/molecule/cloud-init/includes/refuse.yml b/ansible/molecule/cloud-init/includes/refuse.yml new file mode 100644 index 0000000..32a0882 --- /dev/null +++ b/ansible/molecule/cloud-init/includes/refuse.yml @@ -0,0 +1,47 @@ +--- +# One refusal case for ../side_effect.yml: write the broken bootstrap.env, run stage 1, and +# check that it refused for the stated reason, recorded "failed" and did not move the +# checkout. +- name: "Write a bootstrap.env with {{ refusal.name }}" + ansible.builtin.copy: + dest: "{{ env_file }}" + content: "{{ refusal.content }}" + owner: root + group: root + mode: "{{ refusal.mode }}" + +- name: "Run stage 1 with {{ refusal.name }}" + ansible.builtin.command: + cmd: /usr/local/sbin/decdn-bootstrap + register: refused + changed_when: false + failed_when: false + +- name: "Read the state after {{ refusal.name }}" + ansible.builtin.slurp: + src: /var/lib/decdn-bootstrap/state + register: refused_state + +- name: "Run the login hint after {{ refusal.name }}" + ansible.builtin.command: + cmd: sh /etc/profile.d/decdn-bootstrap.sh + register: refused_hint + changed_when: false + +- name: "Read the checkout's commit after {{ refusal.name }}" + ansible.builtin.slurp: + src: /opt/decdn-devops/.git/HEAD + register: refused_head + +- name: "Assert stage 1 refused {{ refusal.name }}" + ansible.builtin.assert: + that: + - refused.rc == 1 + - refused.stderr is search(refusal.expect, ignorecase=false) + - refused_state.content | b64decode | trim == 'failed' + - refused_hint.stdout is search('FAILED') + - refused_head.content | b64decode | trim == pinned_sha + fail_msg: >- + rc={{ refused.rc }} stderr={{ refused.stderr | to_json }} + state={{ refused_state.content | b64decode | trim }} hint={{ refused_hint.stdout | to_json }} + head={{ refused_head.content | b64decode | trim }} (pinned {{ pinned_sha }}) diff --git a/ansible/molecule/cloud-init/molecule.yml b/ansible/molecule/cloud-init/molecule.yml index 567bc8d..92ba1bb 100644 --- a/ansible/molecule/cloud-init/molecule.yml +++ b/ansible/molecule/cloud-init/molecule.yml @@ -8,21 +8,24 @@ # converge seeds NoCloud with the user-data (placeholders filled, the repo, mirror # and keyring pointed at the local ones) and runs cloud-init's boot stages. # It then asserts that the host stopped cleanly at the secret gate. -# side_effect writes 0600 /etc/decdn/decdn.env over "SSH", as the operator would, and -# re-runs `decdn-bootstrap`. -# verify checks a release-mode install, a host-generated wallet, a running -# daemon, and the pinned collection set. +# side_effect checks that stage 1 refuses a branch name, the placeholder ref, an +# unknown bootstrap.env key and a group-readable bootstrap.env, each +# recording "failed" for the login hint. It then moves DEVOPS_REF to a tag +# on a newer commit, writes 0600 /etc/decdn/decdn.env over "SSH", as the +# operator would, and re-runs `decdn-bootstrap`. +# verify checks the checkout is at the tag, a release-mode install, a +# host-generated wallet, a running daemon, and the pinned collection set. # # Release mode runs here and nowhere else in the suite: the role's download, GPG # signature check, SHA256SUMS check, extract and version stamp, against a mirror, # because upstream has not cut a release yet. # -# Coverage boundary: `baseline` is skipped through the user-data's own -# DECDN_BOOTSTRAP_ANSIBLE_ARGS (--skip-tags baseline). Its host hardening (nftables, -# DevSec sysctl/sshd, fail2ban) means nothing in a throwaway container, the same as in -# every other scenario (see ../default/molecule.yml). So this scenario proves the -# bootstrap plumbing and the node path, not the hardening. `cloud-init/README.md` says -# the same. +# Coverage boundary: `baseline` is skipped through bootstrap.sh's test-only switch +# (/etc/decdn-bootstrap/TEST-ONLY-skip-baseline, written by converge outside the +# user-data). Its host hardening (nftables, DevSec sysctl/sshd, fail2ban) means nothing +# in a throwaway container, the same as in every other scenario (see +# ../default/molecule.yml). So this scenario proves the bootstrap plumbing and the node +# path, not the hardening. `cloud-init/README.md` says the same. # # Oldest and newest supported distributions only. They bracket the controller Pythons # that cloud-init/requirements.txt pins two ansible-core versions for (3.11 gets 2.19, diff --git a/ansible/molecule/cloud-init/prepare.yml b/ansible/molecule/cloud-init/prepare.yml index 5b6ac9b..adbbec7 100644 --- a/ansible/molecule/cloud-init/prepare.yml +++ b/ansible/molecule/cloud-init/prepare.yml @@ -62,14 +62,20 @@ src: "{{ scratch }}/devops-src.tgz" dest: /srv/devops-src - - name: Commit it, so stage 1 has a SHA to pin and verify + # Two commits: converge pins the first by SHA; side_effect then moves the host to a + # tag on the second, the "upgrade the deployment code" path. The branch is named + # so side_effect can check that stage 1 refuses a branch name. + - name: Commit it, so stage 1 has a SHA to pin and a tag to move to ansible.builtin.shell: cmd: | set -euo pipefail - git init -q + git init -q -b molecule git add -A git -c user.name=molecule -c user.email=molecule@molecule.invalid commit -qm "molecule snapshot" git rev-parse HEAD > /srv/devops-src.sha + git -c user.name=molecule -c user.email=molecule@molecule.invalid commit -q --allow-empty -m "molecule bump" + git tag v0.0.0-molecule + git rev-parse HEAD > /srv/devops-src.tag-sha chdir: /srv/devops-src executable: /bin/bash creates: /srv/devops-src/.git diff --git a/ansible/molecule/cloud-init/side_effect.yml b/ansible/molecule/cloud-init/side_effect.yml index 5eff77c..cfe872b 100644 --- a/ansible/molecule/cloud-init/side_effect.yml +++ b/ansible/molecule/cloud-init/side_effect.yml @@ -1,11 +1,66 @@ --- -# The operator's part, as cloud-init/README.md tells it: SSH in, write the RPC endpoint to -# a 0600 file on the host, and re-run the bootstrap. The value is a placeholder; the stub -# daemon never dials it. -- name: Side effect — provision the secret and re-run the bootstrap +# The operator's part, as cloud-init/README.md tells it, plus the mistakes stage 1 must +# refuse: +# 1. Each broken bootstrap.env must make `decdn-bootstrap` exit non-zero with its reason, +# record "failed", and make the login hint say so. The branch-name case matters +# most: a branch is unpinned code run as root. +# 2. Then the real thing: move DEVOPS_REF to a tag on a newer commit (the upgrade +# path), write the RPC endpoint to a 0600 file on the host, and re-run. The value +# is a placeholder; the stub daemon never dials it. +- name: Side effect — refusals, then the secret and a re-run at a tag hosts: all become: true + vars: + env_file: /etc/decdn-bootstrap/bootstrap.env + refusals: + - name: a branch name + content: "DEVOPS_REPO=file:///srv/devops-src\nDEVOPS_REF=molecule\n" + mode: "0600" + expect: neither a full 40-character commit SHA nor a tag + - name: the placeholder ref + content: "DEVOPS_REPO=file:///srv/devops-src\nDEVOPS_REF=CHANGE_ME\n" + mode: "0600" + expect: set DEVOPS_REF + - name: an unknown key + content: "DEVOPS_REPO=file:///srv/devops-src\nDEVOPS_REF=CHANGE_ME\nFOO=bar\n" + mode: "0600" + expect: unknown key FOO + - name: a group-readable bootstrap.env + content: "DEVOPS_REPO=file:///srv/devops-src\nDEVOPS_REF=v0.0.0-molecule\n" + mode: "0640" + expect: must be owned by root, mode 0600 tasks: + - name: Read the working bootstrap.env + ansible.builtin.slurp: + src: "{{ env_file }}" + register: good_env + + - name: Read the SHA converge pinned (recorded by prepare.yml) + ansible.builtin.slurp: + src: /srv/devops-src.sha + register: pinned_sha_file + + - name: Decode it + ansible.builtin.set_fact: + pinned_sha: "{{ pinned_sha_file.content | b64decode | trim }}" + + - name: Refuse each broken bootstrap.env + ansible.builtin.include_tasks: includes/refuse.yml + loop: "{{ refusals }}" + loop_control: + loop_var: refusal + label: "{{ refusal.name }}" + + - name: Pin the tag on the newer commit (the upgrade path) + ansible.builtin.copy: + dest: "{{ env_file }}" + content: >- + {{ good_env.content | b64decode + | regex_replace('(?m)^DEVOPS_REF=.*$', 'DEVOPS_REF=v0.0.0-molecule') }} + owner: root + group: root + mode: "0600" + - name: Create /etc/decdn ansible.builtin.file: path: /etc/decdn diff --git a/ansible/molecule/cloud-init/verify.yml b/ansible/molecule/cloud-init/verify.yml index 0eb9ef2..91ff387 100644 --- a/ansible/molecule/cloud-init/verify.yml +++ b/ansible/molecule/cloud-init/verify.yml @@ -1,6 +1,7 @@ --- -# After the secret: the node came from the (mirrored, signed) release, generated its own -# wallet and is running. The toolchain is exactly what cloud-init/ pins. +# After the secret: the host moved to the pinned tag, the node came from the (mirrored, +# signed) release, generated its own wallet and is running. The toolchain is exactly +# what cloud-init/ pins. - name: Verify hosts: all become: true @@ -18,6 +19,17 @@ src: /var/lib/decdn-bootstrap/state register: bootstrap_state + # side_effect moved DEVOPS_REF from converge's SHA to a tag on a newer commit. + - name: Read the tagged commit (recorded by prepare.yml) + ansible.builtin.slurp: + src: /srv/devops-src.tag-sha + register: tag_sha + + - name: Read the checkout's commit + ansible.builtin.slurp: + src: /opt/decdn-devops/.git/HEAD + register: checkout_head + - name: Read the installed-version stamp (release mode only writes it) ansible.builtin.slurp: src: /usr/local/lib/decdn/installed-version @@ -45,6 +57,7 @@ ansible.builtin.assert: that: - bootstrap_state.content | b64decode | trim == 'complete' + - checkout_head.content | b64decode | trim == tag_sha.content | b64decode | trim - stamp.content | b64decode | trim == '0.0.0-molecule-stub' - wallet.results | rejectattr('stat.exists') | list | length == 0 - wallet.results | map(attribute='stat.mode') | unique == ['0600'] @@ -52,6 +65,7 @@ - login_hint.stdout == '' fail_msg: >- state={{ bootstrap_state.content | b64decode | trim }} + head={{ checkout_head.content | b64decode | trim }} (tag {{ tag_sha.content | b64decode | trim }}) stamp={{ stamp.content | b64decode | trim }} wallet={{ wallet.results | map(attribute='stat') | list }} service={{ ansible_facts.services['decdn-node.service'] | default('absent') }} diff --git a/cloud-init/README.md b/cloud-init/README.md index ca900b0..a19e135 100644 --- a/cloud-init/README.md +++ b/cloud-init/README.md @@ -17,38 +17,44 @@ repeated deploys from your workstation, use the [Ansible project](../ansible/REA directly. [`docs/requirements.md`](../docs/requirements.md) compares the paths. > **Upstream has not published a release yet.** The node installs only from a -> GPG-verified release tarball (`release` mode). The `manual` mode copies binaries -> from a control machine, which this path does not have. Until a release exists, serve +> GPG-verified release tarball (`release` mode). The `manual` mode would install +> binaries that nothing verified, so the lint refuses it. Until a release exists, serve > `v/{decdn-node,decdn}--.tar.gz`, `SHA256SUMS` and > `SHA256SUMS.asc` from a mirror, and set `decdn_node_release_base` in the user-data > to point at it. The signature is still checked against deCDN's release key. ## What happens at boot -1. cloud-init installs `git`, `python3-venv`, `ca-certificates` and `sudo`, then writes - three files: +1. cloud-init writes four files early in boot: - `/etc/decdn-bootstrap/bootstrap.env`: which revision of this repo to run. - `/etc/decdn-bootstrap/inventory.yml`: your non-secret settings. - `/usr/local/sbin/decdn-bootstrap`: stage 1 of the bootstrap. -2. **Stage 1** (`decdn-bootstrap`) clones this repo into `/opt/decdn-devops` at - `DEVOPS_REF`. If the ref is a full commit SHA, it checks that the checkout really is - at that commit. It then runs the checkout's [`bootstrap.sh`](bootstrap.sh). + - `/etc/profile.d/decdn-bootstrap.sh`: the login hint (below). + + In its final stage it installs `git`, `python3-venv`, `ca-certificates` and `sudo`, + then runs stage 1. +2. **Stage 1** (`decdn-bootstrap`) takes a lock, so only one run happens at a time, and + records `running`. It clones this repo into `/opt/decdn-devops` at `DEVOPS_REF`. If + the ref is a full commit SHA, it checks that the checkout really is at that commit. + It then runs the checkout's [`bootstrap.sh`](bootstrap.sh). 3. **Stage 2** (`bootstrap.sh`) installs the pinned toolchain: - ansible-core into `/opt/decdn-bootstrap/venv`, from [`requirements.txt`](requirements.txt) with pip's hash checking on; - the Galaxy collections at the exact versions in [`collections.lock.yml`](collections.lock.yml). - It then syntax-checks the playbook and checks that the inventory puts localhost in - `decdn_nodes`. + It then syntax-checks the playbook. It also checks that the inventory puts localhost + in `decdn_nodes`, and that `--tags baseline` still selects the baseline role. 4. With no `/etc/decdn/decdn.env` yet, stage 2 runs only the `baseline` role and records `awaiting-secret`. With the file present, it runs the whole playbook and records `complete`. The state is in `/var/lib/decdn-bootstrap/state`: `running`, `awaiting-secret`, -`complete` or `failed`. A login hint (`/etc/profile.d/decdn-bootstrap.sh`) prints the -next step while the bootstrap is unfinished. The full log is in -`/var/log/cloud-init-output.log`. +`complete` or `failed`. Any failed run records `failed`, including one that stage 1 +refused or a signal interrupted. The login hint (`/etc/profile.d/decdn-bootstrap.sh`) +prints the next step whenever the state is not `complete`. It also says when a +`running` bootstrap is no longer alive, for example after a reboot mid-run. The full +log is in `/var/log/cloud-init-output.log`. ## Set up @@ -67,15 +73,18 @@ next step while the bootstrap is unfinished. The full log is in - `decdn_node_release_base`, for a mirror (see the note above); - `decdn_network`, `arbitrum-sepolia` today. - Any other role knob can go in the same `vars:` block - (`ansible/roles/*/defaults/main.yml`). Check the file before you paste it: + Any other non-secret role knob can go in the same `vars:` block + (`ansible/roles/*/defaults/main.yml`). The knobs that decide the install's trust + (install method, wallet generation, signature verification) stay there too. The lint + refuses them as host vars, and refuses `decdn_release_keyring` and `decdn_env_file` + outright. Check the file before you paste it: ```bash make lint-cloud-init CLOUD_INIT_FILE=path/to/your-user-data.yaml ``` - It fails on your edited copy only if an invariant breaks, for example a secret - added or `release` mode changed. + It fails on your edited copy only if an invariant breaks. Examples: a secret added, + another file written, or `release` mode changed. 2. **Create the VM** with the file as its user data. Examples: - **Hetzner Cloud:** "Cloud config" field, or `hcloud server create --user-data-from-file`. @@ -95,6 +104,10 @@ next step while the bootstrap is unfinished. The full log is in ssh @ cat /var/lib/decdn-bootstrap/state # awaiting-secret ``` + Your admin account exists only once `baseline` has run, near the end of the first + boot. Until then, and after a failure before that point, log in the way your + provider set up (often `root` with the injected key). + `status: error` means the bootstrap failed. The reason is at the end of `/var/log/cloud-init-output.log`. Fix it (usually a value in `/etc/decdn-bootstrap/inventory.yml`), then run `sudo decdn-bootstrap`. @@ -133,18 +146,22 @@ next step while the bootstrap is unfinished. The full log is in --multiaddr /ip4//udp/4433/quic-v1 --dry-run ``` - The wallet's address is in `/var/lib/decdn/keystore.json`. Fund it before you run - the command without `--dry-run`. + `decdn whoami` prints the wallet's address. The address is encrypted inside the + keystore, so the command needs the keystore password, which is in the root-only + `/etc/decdn/keystore.password`. Fund the wallet before you run the command without + `--dry-run`. ## Operate - **Re-run or change settings:** edit `/etc/decdn-bootstrap/inventory.yml`, then run - `sudo decdn-bootstrap`. Every run converges the whole host again, as `make deploy` - does. + `sudo decdn-bootstrap`. Once `/etc/decdn/decdn.env` exists, every run converges the + whole host again, as `make deploy` does. Before that, runs apply `baseline` only. - **Upgrade the deployment code:** set `DEVOPS_REF` in `/etc/decdn-bootstrap/bootstrap.env` to the new SHA or tag, then run `sudo decdn-bootstrap`. It fetches, verifies and re-installs the toolchain pinned at - that revision. Nothing pulls on a timer: the host only runs code you pinned. + that revision. Nothing pulls on a timer: the host only runs code you pinned. A tag is + resolved again on every run, so if someone re-points it, the next run follows. Pin a + SHA if that matters to you. - **Upgrade the node:** change `decdn_node_version` in the inventory and re-run. - **Back up, migrate or decommission:** the host is an ordinary Ansible node. Add it to an inventory on your workstation with the same variables and use `make backup`, @@ -159,16 +176,23 @@ next step while the bootstrap is unfinished. The full log is in only public material: an SSH public key, a version, a region, a repo URL. The RPC URL is written over SSH, and the wallet is generated on the host. `make lint-cloud-init` fails on: - - any secret-looking key (RPC URL, password, token, private key, keystore); + - any file written besides the bootstrap's own four, and any encoded file content; + - any secret-looking key (RPC URL, password, token, private key, keystore, + `decdn_extra_env`); + - a `NAME=value` assignment of a secret-looking variable anywhere, including + commands; - a URL with embedded credentials; - - an unknown `bootstrap.env` key. + - an unknown `bootstrap.env` key; + - any mention of the test-only switch that skips hardening. - **Everything is pinned.** - - This repo: by commit SHA (checked after checkout) or tag. + - This repo: by commit SHA (checked after checkout), or by tag, which is weaker + because a tag can be moved. - ansible-core: by version and hash. - The collections: by exact version. - The node: by release version, installed only if `SHA256SUMS` carries a valid - signature from deCDN's release key (`decdn_verify_release_signature` cannot be - turned off in this file). + signature from deCDN's release key. The lint refuses a user-data that turns + `decdn_verify_release_signature` off, swaps `decdn_release_keyring`, or sets either + one as a host var. Nothing is piped from `curl` into a shell. - **No lockout.** Baseline refuses to harden SSH unless `baseline_sudo_users` names a @@ -177,15 +201,19 @@ next step while the bootstrap is unfinished. The full log is in the node's QUIC udp/4433 open. Metrics and the admin RPC stay on loopback. - The files in `/etc/decdn-bootstrap/` are `root` `0600`. `decdn-bootstrap` refuses a `bootstrap.env` that is not `root`-owned `0600`. It reads the file as literal - `KEY=value` lines and never sources it, and it rejects unknown keys. + `KEY=value` lines and never sources it, and it rejects unknown keys. That leaves no + way to pass Ansible arguments (extra-vars or skipped tags) from user-data. +- `runcmd` must be exactly stage 1, so a failure always reaches `cloud-init status`. What CI proves: - `make lint-cloud-init` (CI job `cloud-init`) checks the schema and the invariants above, and `make test-scripts` checks that it rejects broken variants. - The molecule `cloud-init` scenario boots this file with cloud-init in Debian 12 and - Ubuntu 26.04 containers. It covers the secret gate, then `decdn.env`, then a running - node installed from a locally signed mirror. + Ubuntu 26.04 containers. It covers the secret gate first. It then checks that stage 1 + refuses a branch name, the placeholder ref, an unknown key and a loose file mode, + each recording `failed`. Finally, after an upgrade to a tag and `decdn.env`, it + covers a running node installed from a locally signed mirror. - The scenario skips `baseline`, because host hardening means nothing in a container. It is exercised on real hosts, as for the Ansible path. @@ -200,7 +228,8 @@ What CI proves: Keep a pin whose controller Python range covers 3.11 (Debian 12) and one covering 3.12 to 3.14 (Ubuntu 24.04, Debian 13, Ubuntu 26.04). -- **Collections:** run `make deps` in `ansible/`, then copy the resolved versions into - [`collections.lock.yml`](collections.lock.yml). `make lint-cloud-init` checks the +- **Collections:** resolve from scratch (`rm -rf ansible/collections && make -C ansible + deps`), because an existing tree keeps what it has. Then copy the resolved versions + into [`collections.lock.yml`](collections.lock.yml). `make lint-cloud-init` checks the lock against `ansible/requirements.yml`. The molecule scenario checks that a node ends up with exactly the locked set. diff --git a/cloud-init/bootstrap.sh b/cloud-init/bootstrap.sh index 3e8cd4d..58b8bd9 100755 --- a/cloud-init/bootstrap.sh +++ b/cloud-init/bootstrap.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash # Stage 2 of the cloud-init bootstrap (see README.md). Stage 1 is the small -# /usr/local/sbin/decdn-bootstrap that user-data.yaml writes. It clones this repo at the -# pinned ref, verifies the checkout, then execs this script from it. +# /usr/local/sbin/decdn-bootstrap that user-data.yaml writes. It takes a lock, records +# "running", clones this repo at the pinned ref, verifies the checkout, then execs this +# script from it. # # This script turns the host into a node by running ansible/playbooks/site.yml against # localhost: @@ -17,23 +18,28 @@ # - decdn.env present: run the whole playbook (install, keystore, service), then # record "complete". # -# Re-running it is how the operator continues after writing decdn.env, and how a host -# picks up a new pinned ref: `sudo decdn-bootstrap`. -set -Eeuo pipefail +# `sudo decdn-bootstrap` (stage 1, then this) is how the operator continues after +# writing decdn.env, and how a host picks up a new pinned ref. +set -euo pipefail readonly CONF_DIR=/etc/decdn-bootstrap readonly INVENTORY=$CONF_DIR/inventory.yml readonly VENV=/opt/decdn-bootstrap/venv readonly STATE_DIR=/var/lib/decdn-bootstrap readonly STATE_FILE=$STATE_DIR/state -readonly LOGIN_HINT=/etc/profile.d/decdn-bootstrap.sh # The decdn_node role's default decdn_env_file (roles/decdn_node/defaults/main.yml). readonly ENV_FILE=/etc/decdn/decdn.env +# Test hook for the molecule `cloud-init` scenario only: baseline's hardening means +# nothing in a container. bootstrap.env carries no Ansible arguments, so there is no +# route for extra-vars; this marker is the only way to skip baseline, and +# `make lint-cloud-init` rejects a user-data that mentions it. +readonly SKIP_BASELINE_MARKER=$CONF_DIR/TEST-ONLY-skip-baseline repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" readonly repo -# World-readable on purpose: the login hint below runs as the admin account. +# World-readable on purpose: the login hint (/etc/profile.d/decdn-bootstrap.sh, written +# by the user-data) reads it as the admin account. Stage 1 has a copy of this function. set_state() { install -d -m 0755 "$STATE_DIR" printf '%s\n' "$1" >"$STATE_FILE.tmp" @@ -41,42 +47,30 @@ set_state() { mv -f "$STATE_FILE.tmp" "$STATE_FILE" } -die() { - echo "decdn-bootstrap: $*" >&2 - [[ $EUID -ne 0 ]] || set_state failed - exit 1 -} - -trap 'set_state failed; echo "decdn-bootstrap: FAILED (see the output above; re-run: sudo decdn-bootstrap)" >&2' ERR +die() { echo "decdn-bootstrap: $*" >&2; exit 1; } [[ $EUID -eq 0 ]] || die "run as root (sudo decdn-bootstrap)" +# Any non-zero exit records "failed": a failed command under set -e, a die, or a signal +# (an SSH session dropping mid-run). An ERR trap alone would miss the last two. +trap 'rc=$?; if ((rc != 0)); then set_state failed + echo "decdn-bootstrap: FAILED (see the output above; re-run: sudo decdn-bootstrap)" >&2; fi' EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM + [[ -f $INVENTORY ]] || die "$INVENTORY is missing (it is written by the cloud-init user-data)" if grep -n 'CHANGE_ME' "$INVENTORY" >&2; then die "$INVENTORY still has CHANGE_ME placeholders (the lines above); edit them, then re-run" fi -# Extra ansible-playbook arguments from bootstrap.env, for example -# `--skip-tags baseline` in the containerised CI test. Split on whitespace. -read -ra extra_args <<<"${DECDN_BOOTSTRAP_ANSIBLE_ARGS:-}" +extra_args=() +if [[ -e $SKIP_BASELINE_MARKER ]]; then + echo "decdn-bootstrap: WARNING: $SKIP_BASELINE_MARKER exists; skipping host hardening (test use only)" >&2 + extra_args=(--skip-tags baseline) +fi set_state running -# A login hint for the admin account, written before anything can fail so that a failed -# first boot shows it too. It prints nothing once the node is complete. It lives in -# /etc/profile.d rather than a MOTD, because DevSec ssh_hardening disables the PAM motd. -cat >"$LOGIN_HINT" </dev/null)" in - running) - echo "deCDN: the bootstrap is running (cloud-init status --wait; log: /var/log/cloud-init-output.log)" ;; - awaiting-secret) - echo "deCDN: hardened, waiting for its RPC secret. Write 0600 $ENV_FILE, then run: sudo decdn-bootstrap" ;; - failed) - echo "deCDN: the last bootstrap run FAILED. Re-run it: sudo decdn-bootstrap" ;; -esac -EOF -chmod 0644 "$LOGIN_HINT" - # --- 1. Pinned toolchain ------------------------------------------------------ # A venv whose interpreter no longer runs (the distro's python3 changed under it after a # release upgrade) is rebuilt instead of being patched. @@ -103,6 +97,12 @@ ansible-playbook -i "$INVENTORY" playbooks/site.yml --syntax-check members=$(ansible -i "$INVENTORY" decdn_nodes --list-hosts) grep -qE '^\s+localhost$' <<<"$members" \ || die "$INVENTORY does not put localhost in the decdn_nodes group (site.yml would match nothing)" +# Phase 1 below relies on site.yml tagging the baseline role `baseline`. If that tag +# were renamed, --tags baseline would select nothing and exit 0, and the host would be +# reported hardened without being so. +tasks=$(ansible-playbook -i "$INVENTORY" playbooks/site.yml --tags baseline --list-tasks) +grep -qE '^\s+baseline : ' <<<"$tasks" \ + || die "--tags baseline selects no baseline task in playbooks/site.yml (was the role's tag renamed?)" # --- 3. Converge ---------------------------------------------------------------- if [[ -e $ENV_FILE ]]; then diff --git a/cloud-init/collections.lock.yml b/cloud-init/collections.lock.yml index aaa3913..74f23dc 100644 --- a/cloud-init/collections.lock.yml +++ b/cloud-init/collections.lock.yml @@ -7,7 +7,8 @@ # collections the roles use directly, and every collection they depend on transitively. # Pinning only the top two would leave the transitive ones to resolve to latest. # -# To bump, run `make deps` and copy the resolved versions from +# To bump, resolve from scratch (`rm -rf ansible/collections && make -C ansible deps`; +# an existing tree keeps what it has) and copy the resolved versions from # ansible/collections/ansible_collections/*/*/MANIFEST.json. Leave out anything that # neither of the top two needs (community.docker is only vendored for molecule). # `make lint-cloud-init` checks that every collection in ansible/requirements.yml is diff --git a/cloud-init/requirements.in b/cloud-init/requirements.in index 9951db8..8a4c8d8 100644 --- a/cloud-init/requirements.in +++ b/cloud-init/requirements.in @@ -5,7 +5,7 @@ # # Two pins, because no single ansible-core release supports every supported host's # controller Python. The range is Debian 12's 3.11 through Ubuntu 26.04's 3.14. 2.19 is -# the last release that supports 3.11, and 2.20 and later support 3.12 to 3.14. Both +# the last release that supports 3.11, and 2.20 and 2.21 support 3.12 to 3.14. Both # stay at or above the roles' min_ansible_version (2.15). ansible-core==2.19.13 ; python_version < "3.12" ansible-core==2.21.4 ; python_version >= "3.12" diff --git a/cloud-init/tests/lint.py b/cloud-init/tests/lint.py index 38b9525..e344cae 100755 --- a/cloud-init/tests/lint.py +++ b/cloud-init/tests/lint.py @@ -5,20 +5,26 @@ `cloud-init schema` checks the file's shape, including the `#cloud-config` header. This script checks what the schema cannot: - * no secret anywhere: no key whose name looks secret-bearing (RPC URL, password, - token, keystore contents, private key), no credentials embedded in a URL, and only - the known keys in bootstrap.env; - * the node is installed from the signed release, and its wallet is generated on the - host; + * no secret anywhere: + - write_files writes only the bootstrap's own four files, as plain text; + - no key whose name looks secret-bearing (RPC URL, password, token, keystore + contents, private key, decdn_extra_env); + - no `NAME=value` assignment of a secret-looking variable in any string (file + contents, commands); + - no credentials embedded in a URL, and only the known keys in bootstrap.env; + * nothing mentions the test-only switch that skips the host hardening; + * the node is installed from the release, verified against the vendored deCDN key, + and its wallet is generated on the host. The knobs that decide this may be set + only in decdn_nodes.vars, where this checks them; a host var would override them; * the inventory puts localhost in decdn_nodes (site.yml's host pattern) with a local connection, and names a keyed admin account (baseline's lockout guard); - * stage 1 (the embedded decdn-bootstrap script) passes shellcheck and is what - runcmd runs; + * stage 1 and the login hint pass shellcheck, and runcmd runs exactly stage 1; * cloud-init/collections.lock.yml pins every collection in ansible/requirements.yml, at a version inside its range. -Exit 0 when all of that holds, 1 on a violated invariant (the message says which), and -2 when the file cannot be read or parsed. +Exit 0 when all of that holds and 1 on a violated invariant (the message says which). +Exit 2 when the check cannot run: an unreadable or unparseable file (this one, the +embedded inventory, the lock or ansible/requirements.yml), or no yq or shellcheck. """ import json @@ -32,15 +38,40 @@ ENV_PATH = "/etc/decdn-bootstrap/bootstrap.env" INVENTORY_PATH = "/etc/decdn-bootstrap/inventory.yml" STAGE1_PATH = "/usr/local/sbin/decdn-bootstrap" -ENV_KEYS = {"DEVOPS_REPO", "DEVOPS_REF", "DECDN_BOOTSTRAP_ANSIBLE_ARGS"} +HINT_PATH = "/etc/profile.d/decdn-bootstrap.sh" +# The only files a user-data may write. Anything else (/etc/decdn/decdn.env, +# /etc/grafana-alloy.env, …) is where a secret would go, and belongs on the host. +FILE_PATHS = {ENV_PATH, INVENTORY_PATH, STAGE1_PATH, HINT_PATH} +ENV_KEYS = {"DEVOPS_REPO", "DEVOPS_REF"} +# The molecule scenario's switch for skipping baseline in a container (bootstrap.sh). +# A user-data that creates it, by write_files or a command, ships an unhardened host. +TEST_ONLY_MARKER = "TEST-ONLY-skip-baseline" + +# Knobs that decide the install's trust: allowed only in decdn_nodes.vars (checked +# there), because a host var or another group could override the checked value. +PINNED_VARS = {"decdn_node_install_method", "decdn_node_generate_keystore", + "decdn_verify_release_signature"} +# Knobs a user-data may not set at all: a different signing key would make "verified" +# meaningless, and bootstrap.sh's secret gate looks for the default env file path. +FORBIDDEN_VARS = {"decdn_release_keyring", "decdn_env_file"} # A key whose NAME suggests it carries a secret. Public-key material is fine -# (baseline_sudo_users[].keys), so match whole words, not "key" alone. -SECRET_KEY = re.compile(r"passw|secret|token|rpc_url|private|api_?key|keystore|chpasswd", re.I) -# Named knobs that match SECRET_KEY but carry no secret. -SECRET_KEY_ALLOW = {"decdn_node_generate_keystore"} +# (baseline_sudo_users[].keys), so match specific fragments, not "key" alone. extra_env +# is decdn_extra_env, the role's home for AWS credentials and similar. +SECRET_KEY = re.compile(r"passw|secret|token|rpc_url|private|api_?key|keystore|chpasswd|extra_env", re.I) +# Role knobs that match SECRET_KEY but carry no secret: booleans and file paths. +SECRET_KEY_ALLOW = { + "decdn_node_generate_keystore", + "baseline_sudo_passwordless", + "decdn_keystore_file", + "decdn_keystore_password_file", + "grafana_alloy_secret_file", +} # user:password@ (or token@) in any URL. URL_CREDENTIALS = re.compile(r"[a-z][a-z0-9+.-]*://[^/\s@]+@", re.I) +# NAME=value for a secret-looking variable, in file contents or a command. +SECRET_ASSIGNMENT = re.compile( + r"(?m)^\s*(export\s+)?[A-Z0-9_]*(TOKEN|SECRET|PASSW|API_?KEY|ACCESS_KEY|RPC_URL)[A-Z0-9_]*=") errors = [] @@ -77,6 +108,28 @@ def walk_keys(node, path=""): yield from walk_keys(v, f"{path}[{i}]") +def walk_strings(node, path=""): + """Yield (dotted path, value) for every string scalar in a parsed YAML tree.""" + if isinstance(node, dict): + for k, v in node.items(): + yield from walk_strings(v, f"{path}.{k}" if path else str(k)) + elif isinstance(node, list): + for i, v in enumerate(node): + yield from walk_strings(v, f"{path}[{i}]") + elif isinstance(node, str): + yield path, node + + +def shellcheck(name, text, shell): + if not shutil.which("shellcheck"): + print("lint.py: needs shellcheck", file=sys.stderr) + sys.exit(2) + sc = subprocess.run(["shellcheck", "-s", shell, "-"], input=text, text=True, + capture_output=True, check=False) + if sc.returncode != 0: + violation(f"{name} fails shellcheck:\n{sc.stdout}{sc.stderr}") + + def version_tuple(v): return tuple(int(x) for x in re.findall(r"\d+", v)) @@ -133,13 +186,21 @@ def main(): print(f"lint.py: {path} is not a YAML mapping", file=sys.stderr) sys.exit(2) - files = {f.get("path"): f for f in doc.get("write_files") or [] if isinstance(f, dict)} - for p in (ENV_PATH, INVENTORY_PATH, STAGE1_PATH): - if p not in files: - violation(f"write_files has no {p}") + entries = [f for f in doc.get("write_files") or [] if isinstance(f, dict)] + files = {f.get("path"): f for f in entries} + for p in sorted(FILE_PATHS - files.keys()): + violation(f"write_files has no {p}") + for p in sorted(files.keys() - FILE_PATHS, key=str): + violation(f"write_files writes {p}; only the bootstrap's own files belong in user-data " + "(secrets such as decdn.env are written on the host)") + for f in entries: + if f.get("encoding", "text/plain") not in ("text/plain", "text"): + violation(f"write_files {f.get('path')}: encoding {f.get('encoding')} hides its content from this check") for p in (ENV_PATH, INVENTORY_PATH): if p in files and str(files[p].get("permissions")) != "0600": violation(f"{p} must be written 0600") + if TEST_ONLY_MARKER in raw: + violation(f"the user-data mentions {TEST_ONLY_MARKER}, the test-only switch that skips host hardening") # --- No secrets --------------------------------------------------------------- try: @@ -151,6 +212,10 @@ def main(): for p, k in walk_keys(tree): if SECRET_KEY.search(k) and k not in SECRET_KEY_ALLOW: violation(f"{where}: {p} looks secret-bearing; secrets never go in user-data") + for where, tree in (("user-data", doc), ("inventory", inventory)): + for p, v in walk_strings(tree): + if SECRET_ASSIGNMENT.search(v): + violation(f"{where}: {p} assigns a secret-looking variable; secrets never go in user-data") for i, line in enumerate(raw.splitlines(), 1): if URL_CREDENTIALS.search(line): violation(f"line {i}: a URL with embedded credentials") @@ -172,9 +237,17 @@ def main(): host = (group.get("hosts") or {}).get("localhost") if not isinstance(host, dict) or host.get("ansible_connection") != "local": violation("inventory: localhost must be in decdn_nodes with ansible_connection: local") + if set(inventory) - {"decdn_nodes"}: + violation(f"inventory: only the decdn_nodes group belongs here, not {sorted(set(inventory) - {'decdn_nodes'})}") + for p, k in walk_keys(inventory): + if k in PINNED_VARS and p != f"decdn_nodes.vars.{k}": + violation(f"inventory: {p}: set {k} only in decdn_nodes.vars, where it is checked") + if k in FORBIDDEN_VARS: + violation(f"inventory: {p}: {k} may not be overridden here") iv = group.get("vars") or {} if iv.get("decdn_node_install_method") != "release": - violation("inventory: decdn_node_install_method must be release (the only method that needs no control machine)") + violation("inventory: decdn_node_install_method must be release (the only method that verifies " + "the binaries against deCDN's signature)") if iv.get("decdn_node_generate_keystore") is not True: violation("inventory: decdn_node_generate_keystore must be true (the wallet is generated on the host)") if iv.get("decdn_verify_release_signature", True) is not True: @@ -184,18 +257,13 @@ def main(): violation("inventory: baseline_sudo_users needs at least one named account, each with keys (lockout guard)") # --- Stage 1 -------------------------------------------------------------------- - runcmd = [c if isinstance(c, str) else " ".join(map(str, c)) for c in doc.get("runcmd") or []] - if not any(STAGE1_PATH in c for c in runcmd): - violation(f"runcmd does not run {STAGE1_PATH}") - stage1 = files.get(STAGE1_PATH, {}).get("content", "") - if stage1: - if not shutil.which("shellcheck"): - print("lint.py: needs shellcheck", file=sys.stderr) - sys.exit(2) - sc = subprocess.run(["shellcheck", "-s", "bash", "-"], input=stage1, text=True, - capture_output=True, check=False) - if sc.returncode != 0: - violation(f"{STAGE1_PATH} fails shellcheck:\n{sc.stdout}") + # Exactly stage 1, so nothing masks its exit status (`|| true`) from cloud-init. + if doc.get("runcmd") not in ([[STAGE1_PATH]], [STAGE1_PATH]): + violation(f"runcmd must be exactly [[{STAGE1_PATH}]], so a failure shows in cloud-init status") + if files.get(STAGE1_PATH, {}).get("content"): + shellcheck(STAGE1_PATH, files[STAGE1_PATH]["content"], "bash") + if files.get(HINT_PATH, {}).get("content"): + shellcheck(HINT_PATH, files[HINT_PATH]["content"], "sh") check_lock() diff --git a/cloud-init/user-data.yaml b/cloud-init/user-data.yaml index 4035316..7c19161 100644 --- a/cloud-init/user-data.yaml +++ b/cloud-init/user-data.yaml @@ -32,7 +32,6 @@ write_files: content: | DEVOPS_REPO=https://github.com/decdn/devops.git DEVOPS_REF=CHANGE_ME - DECDN_BOOTSTRAP_ANSIBLE_ARGS= # The Ansible inventory for this host. Everything the playbook needs that is not a # secret lives here. More knobs: ansible/roles/*/defaults/main.yml. @@ -71,6 +70,31 @@ write_files: decdn_network: arbitrum-sepolia decdn_region: "CHANGE_ME" # MUST-EDIT: ISO 3166-1 alpha-2 of the VM's location, e.g. "DE" + # What an interactive login prints while the bootstrap is unfinished. It is written + # here, not by the bootstrap, so that it is in place even if stage 1 fails. + - path: /etc/profile.d/decdn-bootstrap.sh + owner: root:root + permissions: "0644" + content: | + # deCDN bootstrap status at login (cloud-init/README.md in decdn/devops). + log=/var/log/cloud-init-output.log + rerun="sudo decdn-bootstrap" + case "$(cat /var/lib/decdn-bootstrap/state 2>/dev/null)" in + complete) ;; + awaiting-secret) + echo "deCDN: hardened, waiting for its RPC secret. Write 0600 /etc/decdn/decdn.env, then run: $rerun" ;; + running) + if pgrep -f 'sbin/decdn-bootstrap|cloud-init/bootstrap[.]sh' >/dev/null 2>&1; then + echo "deCDN: the bootstrap is running (log: $log)" + else + echo "deCDN: the bootstrap was interrupted. Re-run it: $rerun" + fi ;; + failed) + echo "deCDN: the last bootstrap run FAILED (log: $log). Re-run it: $rerun" ;; + *) + echo "deCDN: the bootstrap has not run yet (cloud-init status --wait; log: $log)" ;; + esac + # Stage 1: fetch and verify the pinned checkout, then hand over to its # cloud-init/bootstrap.sh. Re-run it by hand after writing /etc/decdn/decdn.env, or # after changing DEVOPS_REF: `sudo decdn-bootstrap`. @@ -83,26 +107,43 @@ write_files: set -euo pipefail conf=/etc/decdn-bootstrap/bootstrap.env checkout=/opt/decdn-devops + state_dir=/var/lib/decdn-bootstrap die() { echo "decdn-bootstrap: $*" >&2; exit 1; } + # World-readable: the login hint reads it as the admin account. + set_state() { + install -d -m 0755 "$state_dir" + printf '%s\n' "$1" >"$state_dir/state.tmp" + chmod 0644 "$state_dir/state.tmp" + mv -f "$state_dir/state.tmp" "$state_dir/state" + } [[ $EUID -eq 0 ]] || die "run as root (sudo decdn-bootstrap)" + # One run at a time. The lock is held until stage 2 exits (exec keeps fd 9). + exec 9>/run/decdn-bootstrap.lock + flock -n 9 || die "another decdn-bootstrap run is in progress" + # Any non-zero exit before the hand-over, a signal included, records "failed". + # exec replaces this process, so stage 2's own traps take over from there. + trap 'rc=$?; ((rc == 0)) || set_state failed' EXIT + trap 'exit 129' HUP + trap 'exit 130' INT + trap 'exit 143' TERM + set_state running + [[ $(stat -c '%u %a' "$conf") == "0 600" ]] || die "$conf must be owned by root, mode 0600" # KEY=value lines, read literally: never sourced, so a value is data, not shell. - DEVOPS_REPO='' DEVOPS_REF='' DECDN_BOOTSTRAP_ANSIBLE_ARGS='' + DEVOPS_REPO='' DEVOPS_REF='' while IFS= read -r line || [[ -n $line ]]; do [[ $line =~ ^[[:space:]]*(#|$) ]] && continue [[ $line =~ ^([A-Z_]+)=(.*)$ ]] || die "$conf: not a KEY=value line: $line" case ${BASH_REMATCH[1]} in DEVOPS_REPO) DEVOPS_REPO=${BASH_REMATCH[2]} ;; DEVOPS_REF) DEVOPS_REF=${BASH_REMATCH[2]} ;; - DECDN_BOOTSTRAP_ANSIBLE_ARGS) DECDN_BOOTSTRAP_ANSIBLE_ARGS=${BASH_REMATCH[2]} ;; *) die "$conf: unknown key ${BASH_REMATCH[1]}" ;; esac done <"$conf" [[ -n $DEVOPS_REPO ]] || die "DEVOPS_REPO is empty in $conf" [[ -n $DEVOPS_REF && $DEVOPS_REF != CHANGE_ME ]] \ || die "set DEVOPS_REF in $conf to a commit SHA or release tag of $DEVOPS_REPO" - export DECDN_BOOTSTRAP_ANSIBLE_ARGS export HOME="${HOME:-/root}" if [[ ! -d $checkout/.git ]]; then diff --git a/tests/scripts-test.sh b/tests/scripts-test.sh index 1d0864b..15edb5b 100755 --- a/tests/scripts-test.sh +++ b/tests/scripts-test.sh @@ -78,6 +78,8 @@ variant "published port" 's/^(\s*)network_mode: host$/\1ports: ["127.0.0. variant "tag instead of digest" 's#^(\s*)image: .*#\1image: ghcr.io/decdn/decdn-node:latest#' # --- lint-cloud-init negatives: each broken variant must be rejected ------------------- +# Skipped without cloud-init on PATH, except in CI (which installs it), so a broken +# install step cannot quietly drop these cases. if command -v cloud-init >/dev/null; then userdata="$repo/cloud-init/user-data.yaml" expect 0 "lint-cloud-init accepts cloud-init/user-data.yaml" make -s -C "$repo" lint-cloud-init @@ -87,25 +89,54 @@ if command -v cloud-init >/dev/null; then cat "$work/out" >&2; fail "lint-cloud-init did not reject a missing #cloud-config header as invalid" fi pass "lint-cloud-init rejects: no #cloud-config header (schema)" - ci_variant() { # - sed -E "$2" "$userdata" > "$work/ci-$1.yaml" + # : the fragment pins WHICH + # invariant fired, since one edit can trip several. + ci_variant() { + sed -E "$3" "$userdata" > "$work/ci-$1.yaml" cmp -s "$userdata" "$work/ci-$1.yaml" && fail "variant $1 did not change user-data.yaml" if make -s -C "$repo" lint-cloud-init CLOUD_INIT_FILE="$work/ci-$1.yaml" >"$work/out" 2>&1; then fail "lint-cloud-init accepted: $1" fi grep -q 'violates an invariant' "$work/out" || { cat "$work/out" >&2; fail "lint-cloud-init failed for another reason: $1"; } + grep -qF -- "$2" "$work/out" || { cat "$work/out" >&2; fail "lint-cloud-init rejected $1, but not with: $2"; } pass "lint-cloud-init rejects: $1" } - ci_variant "RPC URL in bootstrap.env" 's#^(\s*)DECDN_BOOTSTRAP_ANSIBLE_ARGS=$#\1DECDN_RPC_URL=https://rpc.example/key#' - ci_variant "RPC URL in the inventory" 's#^(\s*)decdn_network: arbitrum-sepolia$#\1decdn_rpc_url: "https://rpc.example/"#' - ci_variant "credentials in a URL" 's#^(\s*)DEVOPS_REPO=https://#\1DEVOPS_REPO=https://user:pw@#' - ci_variant "manual install method" 's/^(\s*)decdn_node_install_method: release$/\1decdn_node_install_method: manual/' - ci_variant "no host-generated wallet" 's/^(\s*)decdn_node_generate_keystore: true(.*)$/\1decdn_node_generate_keystore: false\2/' - ci_variant "localhost outside decdn_nodes" 's/^(\s*)decdn_nodes:$/\1decdn_hosts:/' - ci_variant "admin account without keys" '/^\s*keys:$/,+1d' - ci_variant "stage 1 not run" 's#^ - \[/usr/local/sbin/decdn-bootstrap\]$# - [/bin/true]#' + ref='^(\s*)DEVOPS_REF=CHANGE_ME$' + net='^(\s*)decdn_network: arbitrum-sepolia$' + loc='^(\s*)ansible_connection: local$' + stage1='^(\s*)- path: /usr/local/sbin/decdn-bootstrap$' + # Secrets + ci_variant "RPC URL in bootstrap.env" 'bootstrap.env: unexpected key DECDN_RPC_URL' "s#$ref#&\\n\\1DECDN_RPC_URL=https://rpc.example/key#" + ci_variant "unknown bootstrap.env key" 'bootstrap.env: unexpected key FOO' "s#$ref#&\\n\\1FOO=bar#" + ci_variant "RPC URL in the inventory" 'decdn_nodes.vars.decdn_rpc_url looks secret-bearing' "s#$net#&\\n\\1decdn_rpc_url: https://rpc.example/#" + ci_variant "decdn_extra_env" 'decdn_nodes.vars.decdn_extra_env looks secret-bearing' "s#$net#&\\n\\1decdn_extra_env: {AWS_REGION: eu-west-1}#" + ci_variant "credentials in a URL" 'a URL with embedded credentials' 's#^(\s*)DEVOPS_REPO=https://#\1DEVOPS_REPO=https://user:pw@#' + ci_variant "another write_files path" 'write_files writes /etc/decdn/decdn.env' "s#$stage1#\\1- path: /etc/decdn/decdn.env\\n\\1 content: FOO=bar\\n&#" + ci_variant "b64-encoded content" 'encoding b64 hides its content' "s#$stage1#&\\n\\1 encoding: b64#" + ci_variant "secret assigned in runcmd" 'assigns a secret-looking variable' 's#^ - \[/usr/local/sbin/decdn-bootstrap\]$#&\n - "GC_API_TOKEN=x /bin/true"#' + # Hardening and the signed install + ci_variant "the test-only baseline skip" 'mentions TEST-ONLY-skip-baseline' 's#^ - \[/usr/local/sbin/decdn-bootstrap\]$# - [touch, /etc/decdn-bootstrap/TEST-ONLY-skip-baseline]\n&#' + ci_variant "manual install method" 'decdn_node_install_method must be release' 's/^(\s*)decdn_node_install_method: release$/\1decdn_node_install_method: manual/' + ci_variant "no host-generated wallet" 'decdn_node_generate_keystore must be true' 's/^(\s*)decdn_node_generate_keystore: true(.*)$/\1decdn_node_generate_keystore: false\2/' + ci_variant "signature off as a host var" 'set decdn_verify_release_signature only in decdn_nodes.vars' "s#$loc#&\\n\\1decdn_verify_release_signature: false#" + ci_variant "install method as a host var" 'set decdn_node_install_method only in decdn_nodes.vars' "s#$loc#&\\n\\1decdn_node_install_method: manual#" + ci_variant "another signing key" 'decdn_release_keyring may not be overridden' "s#$net#&\\n\\1decdn_release_keyring: /tmp/KEYS.asc#" + ci_variant "another env file path" 'decdn_env_file may not be overridden' "s#$net#&\\n\\1decdn_env_file: /etc/decdn/other.env#" + ci_variant "another inventory group" 'only the decdn_nodes group belongs here' 's#^(\s*)decdn_nodes:$#\1all: {vars: {decdn_verify_release_signature: false}}\n&#' + # Shape + ci_variant "localhost outside decdn_nodes" 'localhost must be in decdn_nodes' 's/^(\s*)decdn_nodes:$/\1decdn_hosts:/' + ci_variant "admin account without keys" 'baseline_sudo_users needs at least one named account' '/^\s*keys:$/,+1d' + ci_variant "stage 1 not run" 'runcmd must be exactly' 's#^ - \[/usr/local/sbin/decdn-bootstrap\]$# - [/bin/true]#' + ci_variant "stage 1 failure masked" 'runcmd must be exactly' 's#^ - \[/usr/local/sbin/decdn-bootstrap\]$# - "/usr/local/sbin/decdn-bootstrap || true"#' # shellcheck disable=SC2016 # a literal $DEVOPS_REPO: the variant unquotes it in stage 1 - ci_variant "shellcheck-dirty stage 1" 's#git clone --quiet --no-checkout "\$DEVOPS_REPO"#git clone --quiet --no-checkout $DEVOPS_REPO#' + ci_variant "shellcheck-dirty stage 1" 'decdn-bootstrap fails shellcheck' 's#git clone --quiet --no-checkout "\$DEVOPS_REPO"#git clone --quiet --no-checkout $DEVOPS_REPO#' + # Non-secret knobs whose names look secret must still pass. + sed -E "s#$net#&\\n\\1baseline_sudo_passwordless: false\\n\\1decdn_keystore_file: /var/lib/decdn/keystore.json#" \ + "$userdata" > "$work/ci-knobs.yaml" + expect 0 "lint-cloud-init accepts non-secret knobs with secret-looking names" \ + make -s -C "$repo" lint-cloud-init CLOUD_INIT_FILE="$work/ci-knobs.yaml" +elif [[ -n ${CI:-} ]]; then + fail "cloud-init is not on PATH in CI; the lint-cloud-init negatives would be skipped" else skipped+=("lint-cloud-init negatives (needs cloud-init on PATH; CI installs it)") fi