diff --git a/.github/workflows/molecule.yml b/.github/workflows/molecule.yml index b0b0a31..031f1ec 100644 --- a/.github/workflows/molecule.yml +++ b/.github/workflows/molecule.yml @@ -33,14 +33,17 @@ jobs: run: | python -m pip install --upgrade \ molecule "molecule-plugins[docker]" ansible ansible-lint docker - - name: Install Galaxy collections - run: make deps - name: molecule test - # --all runs every scenario under ansible/molecule/: default - # (operator-provisioned keystore + rendered-value assertions), schema - # (config key-set drift against the upstream field list), validation - # (bad knobs must be rejected by the role's own asserts), generate-keystore - # (opt-in host-side wallet generation), host-env (host-provisioned - # /etc/decdn/decdn.env — no secret on the control machine), and - # slow-readiness (advisory /metrics probe timeout). - run: molecule test --all + # `make molecule` runs every scenario under ansible/molecule/ and fans them out + # in parallel; ansible/Makefile documents the set and the fail-loud guards. + # + # There is deliberately no separate `make deps` step: the molecule targets take + # `deps` as a prerequisite, so a standalone one would resolve and install the + # Galaxy requirements a second time every run — a second chance to trip over a + # flaky galaxy.ansible.com, for no added coverage. + # + # JOBS is capped at 3 rather than the default (one job per scenario, currently + # 6): every scenario is a privileged systemd container, and they share this + # runner's cores and cgroup hierarchy. If this job turns flaky, drop to JOBS=1 + # or swap in `make molecule-serial` — the latter also serialises the output. + run: make molecule JOBS=3 diff --git a/AGENTS.md b/AGENTS.md index 0c37784..addec7a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -103,7 +103,9 @@ deploys (its targets must run from `ansible/`). `make help` lists root targets. make hooks # one-time: install pre-commit git hook (pip install pre-commit first) make lint # all pre-commit hooks on all files (hygiene, shellcheck, yamllint, markdown) make lint-ansible # vendor collections + full ansible-lint (production profile) -make molecule # containerised converge/verify of the decdn_node role (needs Docker) +make molecule # containerised converge/verify of the decdn_node role — all six + # scenarios in parallel (needs Docker); cap with JOBS= +make molecule-serial # the same suite one scenario at a time (readable failure output) make lint-helm # chart: helm lint + render tests + kubeconform + schema keys (needs helm, yq, Docker) make security # = security-ansible + security-helm (KICS over the rendered chart; needs helm) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1adfc79..8a0c897 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -37,7 +37,8 @@ Run it on demand with `make lint-ansible`, or `pre-commit run ansible-lint --hoo run via **pre-commit locally only** (`make hooks` / `make lint`), not in CI. - **`molecule.yml`** — containerised converge + idempotence + verify of the `decdn_node` role (privileged systemd Docker container; scoped to `ansible/**`). Run locally with - `make molecule` (needs Docker). + `make molecule` (needs Docker) — it runs all six scenarios in parallel, so reach for + `make molecule-serial` when you need to read a failure in order. ## Supply-chain / pinning rules diff --git a/Makefile b/Makefile index 98e9d5e..c283631 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ # Convenience targets for the deCDN DevOps monorepo. # Run from the repo root. Ansible-specific work is delegated to ansible/Makefile. -.PHONY: help hooks lint lint-ansible lint-helm security security-ansible security-helm molecule galaxy-build galaxy-check +.PHONY: help hooks lint lint-ansible lint-helm security security-ansible security-helm molecule molecule-serial galaxy-build galaxy-check SHELL := /bin/bash # KICS runs straight from the engine image, pinned by digest. This target IS the @@ -59,9 +59,12 @@ security-helm: ## KICS scan of the decdn-node chart's rendered manifests ( lint-helm: ## helm lint + render tests + kubeconform + shared schema-key check (needs helm, yq, python3>=3.11, docker) KUBECONFORM="docker run --rm -i $(KUBECONFORM_IMAGE)" $(CHART)/tests/render-test.sh -molecule: ## containerised converge/verify of the decdn_node role (needs Docker) +molecule: ## containerised converge/verify of the decdn_node role, scenarios in parallel (needs Docker) $(MAKE) -C ansible molecule +molecule-serial: ## same suite, one scenario at a time (readable output on failure) + $(MAKE) -C ansible molecule-serial + galaxy-build: ## stage + build the decdn.node Galaxy collection artifact $(MAKE) -C ansible build diff --git a/ansible/Makefile b/ansible/Makefile index b7dbcc7..4cdcb7b 100644 --- a/ansible/Makefile +++ b/ansible/Makefile @@ -1,6 +1,6 @@ # Convenience targets for the deCDN Ansible project. # Always run from the ansible/ directory. -.PHONY: deps lint check deploy molecule build galaxy-check +.PHONY: deps lint check deploy molecule molecule-serial build galaxy-check SHELL := /bin/bash # Install the required Galaxy collections (>= constraints in requirements.yml) @@ -67,11 +67,57 @@ deploy: # stub daemon (needs Docker; a privileged systemd container). See molecule/. # Scenarios: default (rendered values + idempotence), schema (config key-set drift # against the upstream field list), validation (bad knobs must be rejected by the -# role's own asserts), generate-keystore (opt-in host-side wallet), slow-readiness -# (advisory /metrics probe timeout). -# `--all` runs every scenario: `default` (operator-provisioned keystore) and -# `generate-keystore` (opt-in host-side wallet generation). -molecule: +# role's own asserts), generate-keystore (opt-in host-side wallet), host-env +# (host-provisioned /etc/decdn/decdn.env), slow-readiness (advisory /metrics probe +# timeout). +# +# The scenarios run concurrently: distinct container names, no published host ports, +# per-scenario ephemeral dirs, and what they share on the control machine (the +# vendored collections, roles/, the stub under default/files/) they only read. +# 595s -> 151s on a 16-core box. They DO share one Docker daemon and the host cgroup +# hierarchy — every platform is a privileged systemd container — which is why CI caps +# JOBS. Output interleaves, hence the [scenario] prefix; `molecule-serial` is the +# escape hatch when a failure needs reading in order. Override the fan-out width with +# e.g. `make molecule JOBS=2` on a small machine. +# +# Molecule's own --workers is not usable here: it refuses to run outside collection +# mode ("--workers > 1 is only supported in collection mode (galaxy.yml required)"), +# and this project deliberately has no galaxy.yml at its root — that would make +# ansible-lint reinterpret the deploy project as a collection (see galaxy/README.md). +# +# Fail loud (hard rule 4), in three places, because a test harness that passes when it +# did not run is worse than a slow one: +# - the guard below refuses an empty or truncated scenario set. Discovery moved from +# molecule to a glob, and a glob that matches nothing (wrong cwd, renamed layout) +# would otherwise run zero scenarios and exit 0 — a green build that tested nothing. +# - pipefail stops the [scenario] prefixing pipe from masking molecule's status, and +# each scenario announces its own failure by name (the bare xargs message does not). +# - xargs then exits non-zero. The `|| exit 1` in the wrapper normalises every +# scenario failure to 1, which is what keeps xargs off its own abort-on-255 path — +# that one stops launching queued scenarios, so a single unlucky exit code would +# otherwise cut the suite short. So: 123 when a scenario failed, 125 if one was +# killed by a signal, and every scenario always gets to run. +SCENARIOS := $(notdir $(patsubst %/,%,$(dir $(wildcard molecule/*/molecule.yml)))) +SCENARIO_DIRS := $(notdir $(patsubst %/,%,$(wildcard molecule/*/))) +JOBS ?= $(words $(SCENARIOS)) + +molecule: deps + @test -n "$(strip $(SCENARIOS))" && test "$(strip $(SCENARIOS))" = "$(strip $(SCENARIO_DIRS))" || { \ + echo "scenario discovery failed: molecule.yml in [$(SCENARIOS)] but scenario dirs are [$(SCENARIO_DIRS)]"; \ + echo "refusing to run a silently-truncated suite (this target must run from ansible/)"; exit 1; } + @case '$(JOBS)' in ''|*[!0-9]*) \ + echo "JOBS must be a positive integer (got '$(JOBS)')"; exit 1 ;; esac + @test "$$((10#$(JOBS)))" -gt 0 || { \ + echo "JOBS must be a positive integer (got '$(JOBS)'): any all-zero value — 0, 00, 000 —"; \ + echo "reaches xargs as -P 0, which means UNLIMITED concurrency, not none"; exit 1; } + @printf '%s\n' $(SCENARIOS) | xargs -P $(JOBS) -I{} \ + bash -c 'set -euo pipefail; molecule test -s {} --no-command-borders 2>&1 | sed -u "s/^/[{}] /" \ + || { echo "[{}] SCENARIO FAILED"; exit 1; }' + +# The same suite, one scenario at a time (molecule's own --all). It stops at the FIRST +# failing scenario (--continue-on-failure defaults off), so it reports less than the +# parallel target — but readably, which is the point. +molecule-serial: deps molecule test --all # --- Galaxy collection (decdn.node) ------------------------------------------ diff --git a/ansible/README.md b/ansible/README.md index 94dc86d..0977195 100644 --- a/ansible/README.md +++ b/ansible/README.md @@ -191,11 +191,22 @@ primitives underneath it. All take `--dry-run`. See ```bash make lint # yamllint + ansible-lint (production profile) ansible-playbook playbooks/site.yml --syntax-check -make molecule # containerised converge + idempotence + verify (needs Docker) +make molecule # all six molecule scenarios, in parallel (needs Docker) +make molecule JOBS=2 # …capped to two at a time on a small machine +make molecule-serial # …one at a time, when a failure needs readable output ``` -`make molecule` converges the **`decdn_node`** role in a privileged systemd container -against a stub daemon (`molecule/default/`): it installs via the `manual` method (no +`make molecule` runs every scenario under `molecule/`: **`default`** (described below), +`schema` (config key-set drift against the upstream field list), `validation` (bad knobs +must be rejected by the role's own asserts), `generate-keystore` (opt-in host-side +wallet), `host-env` (host-provisioned `/etc/decdn/decdn.env`) and `slow-readiness` +(advisory `/metrics` probe timeout). They are independent, so they run concurrently — +~151s instead of ~595s — and each line of output is prefixed with its scenario name +because the runs interleave. `make molecule-serial` is the escape hatch when that +interleaving gets in the way of reading a failure. + +The `default` scenario converges the **`decdn_node`** role in a privileged systemd +container against a stub daemon: it installs via the `manual` method (no published release needed), stages a placeholder keystore, renders `node.toml` + the hardened unit, starts the service, and passes the role's own `/metrics` readiness probe; `verify.yml` then asserts the node user, valid TOML, a valid systemd unit, loopback-only diff --git a/ansible/galaxy/CHANGELOG.md b/ansible/galaxy/CHANGELOG.md index 679bf1e..53cf3fe 100644 --- a/ansible/galaxy/CHANGELOG.md +++ b/ansible/galaxy/CHANGELOG.md @@ -6,6 +6,14 @@ collection adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) ## [Unreleased] +### Added + +- `baseline_packages` now includes `acl`. `decdn_node` runs two tasks as the + unprivileged `decdn` user (`decdn key-gen`, and the `decdn config validate` gate), + and on Debian Ansible needs ACL support to hand the temp module file to that user. + Without it both fail without an `rc`, which the role can only report after the fact + ("becoming the unprivileged decdn user needs the `acl` package on this host"). + ### Removed - `decdn_delivery_floor` and `decdn_rate_bounds_poll_interval_sec`: upstream diff --git a/ansible/molecule/default/molecule.yml b/ansible/molecule/default/molecule.yml index 8cb42ae..6261c2e 100644 --- a/ansible/molecule/default/molecule.yml +++ b/ansible/molecule/default/molecule.yml @@ -11,10 +11,12 @@ # ssh_hardening (sysctl, mount, auth, sshd) and fail2ban — none of which behave # meaningfully in a throwaway systemd container. It is validated on real hosts via # `make check` instead. -dependency: - name: galaxy - options: - requirements-file: ../../requirements.yml +# No `dependency` step (and no dependency config): molecule never vendored anything +# here. Its roles invoker reads `role-file`, not `requirements-file`, and its +# collections invoker resolves a relative path against the process CWD — so +# ../../requirements.yml pointed outside the repo and both halves just warned on every +# run. The collections come from `make deps` -> ansible/collections, surfaced by +# ANSIBLE_COLLECTIONS_PATH below. driver: name: docker platforms: @@ -34,11 +36,24 @@ provisioner: env: ANSIBLE_ROLES_PATH: "${MOLECULE_PROJECT_DIRECTORY}/roles" ANSIBLE_COLLECTIONS_PATH: "${MOLECULE_PROJECT_DIRECTORY}/collections" + # The docker connection plugin supports pipelining, which drops a `docker exec` + # round trip per remote task — 105s -> 82s measured on this scenario. Safe here: + # the containers have no sudo requiretty. + # Known coverage boundary: with pipelining on, command/shell modules are fed over + # stdin, so the remote-tmp path behind `become_user` is not exercised. Molecule + # connects as root, where that path is a plain chown, so it never reproduced the + # real-host `acl` failure the role warns about (tasks/main.yml) either way. + ANSIBLE_PIPELINING: "true" verifier: name: ansible scenario: test_sequence: - - dependency + # The leading `destroy` is deliberate. The docker driver creates with + # `recreate: false`, so a container orphaned by an aborted run (Ctrl-C, OOM — + # both likelier now that the scenarios run in parallel) would be REUSED, and + # `converge` + `idempotence` would then pass trivially against a host that was + # already converged by the previous run. + - destroy - create - prepare - converge diff --git a/ansible/molecule/generate-keystore/molecule.yml b/ansible/molecule/generate-keystore/molecule.yml index 0098e7f..aafb9d9 100644 --- a/ansible/molecule/generate-keystore/molecule.yml +++ b/ansible/molecule/generate-keystore/molecule.yml @@ -8,10 +8,12 @@ # (host-level hardening is meaningless in a throwaway systemd container) — see # ../default/molecule.yml. This scenario reuses the shared stub daemon under # ../default/files/decdn-node-stub (which implements a `key-gen` subcommand). -dependency: - name: galaxy - options: - requirements-file: ../../requirements.yml +# No `dependency` step (and no dependency config): molecule never vendored anything +# here. Its roles invoker reads `role-file`, not `requirements-file`, and its +# collections invoker resolves a relative path against the process CWD — so +# ../../requirements.yml pointed outside the repo and both halves just warned on every +# run. The collections come from `make deps` -> ansible/collections, surfaced by +# ANSIBLE_COLLECTIONS_PATH below. driver: name: docker platforms: @@ -30,13 +32,19 @@ provisioner: env: ANSIBLE_ROLES_PATH: "${MOLECULE_PROJECT_DIRECTORY}/roles" ANSIBLE_COLLECTIONS_PATH: "${MOLECULE_PROJECT_DIRECTORY}/collections" + # The docker connection plugin supports pipelining, which drops a `docker exec` + # round trip per remote task (measured at -22% on the `default` scenario, which + # also carries the coverage note). Safe here: the containers have no sudo requiretty. + ANSIBLE_PIPELINING: "true" verifier: name: ansible scenario: # No `prepare` — the role must generate the wallet itself. `idempotence` proves # the stat/`creates:` gating makes a second converge a no-op (nothing re-minted). test_sequence: - - dependency + # Leading `destroy` so an orphaned container is never reused — see + # ../default/molecule.yml. + - destroy - create - converge - idempotence diff --git a/ansible/molecule/host-env/molecule.yml b/ansible/molecule/host-env/molecule.yml index 858d2c6..26c19d9 100644 --- a/ansible/molecule/host-env/molecule.yml +++ b/ansible/molecule/host-env/molecule.yml @@ -10,10 +10,12 @@ # (host-level hardening is meaningless in a throwaway systemd container) — see # ../default/molecule.yml. This scenario reuses the shared stub daemon under # ../default/files/decdn-node-stub. -dependency: - name: galaxy - options: - requirements-file: ../../requirements.yml +# No `dependency` step (and no dependency config): molecule never vendored anything +# here. Its roles invoker reads `role-file`, not `requirements-file`, and its +# collections invoker resolves a relative path against the process CWD — so +# ../../requirements.yml pointed outside the repo and both halves just warned on every +# run. The collections come from `make deps` -> ansible/collections, surfaced by +# ANSIBLE_COLLECTIONS_PATH below. driver: name: docker platforms: @@ -32,6 +34,10 @@ provisioner: env: ANSIBLE_ROLES_PATH: "${MOLECULE_PROJECT_DIRECTORY}/roles" ANSIBLE_COLLECTIONS_PATH: "${MOLECULE_PROJECT_DIRECTORY}/collections" + # The docker connection plugin supports pipelining, which drops a `docker exec` + # round trip per remote task (measured at -22% on the `default` scenario, which + # also carries the coverage note). Safe here: the containers have no sudo requiretty. + ANSIBLE_PIPELINING: "true" verifier: name: ansible scenario: @@ -40,7 +46,9 @@ scenario: # converge must restart the daemon (verify.yml compares MainPID across it). The # order matters: idempotence has to run while the file is still untouched. test_sequence: - - dependency + # Leading `destroy` so an orphaned container is never reused — see + # ../default/molecule.yml. + - destroy - create - prepare - converge diff --git a/ansible/molecule/schema/molecule.yml b/ansible/molecule/schema/molecule.yml index 1c2afe2..8434e5d 100644 --- a/ansible/molecule/schema/molecule.yml +++ b/ansible/molecule/schema/molecule.yml @@ -12,10 +12,12 @@ # `baseline` is NOT exercised here, for the same reasons as the `default` scenario. # There is no idempotence step: this scenario asserts on rendered content, and # `default` already covers idempotence. -dependency: - name: galaxy - options: - requirements-file: ../../requirements.yml +# No `dependency` step (and no dependency config): molecule never vendored anything +# here. Its roles invoker reads `role-file`, not `requirements-file`, and its +# collections invoker resolves a relative path against the process CWD — so +# ../../requirements.yml pointed outside the repo and both halves just warned on every +# run. The collections come from `make deps` -> ansible/collections, surfaced by +# ANSIBLE_COLLECTIONS_PATH below. driver: name: docker platforms: @@ -35,11 +37,17 @@ provisioner: env: ANSIBLE_ROLES_PATH: "${MOLECULE_PROJECT_DIRECTORY}/roles" ANSIBLE_COLLECTIONS_PATH: "${MOLECULE_PROJECT_DIRECTORY}/collections" + # The docker connection plugin supports pipelining, which drops a `docker exec` + # round trip per remote task (measured at -22% on the `default` scenario, which + # also carries the coverage note). Safe here: the containers have no sudo requiretty. + ANSIBLE_PIPELINING: "true" verifier: name: ansible scenario: test_sequence: - - dependency + # Leading `destroy` so an orphaned container is never reused — see + # ../default/molecule.yml. + - destroy - create - prepare - converge diff --git a/ansible/molecule/slow-readiness/molecule.yml b/ansible/molecule/slow-readiness/molecule.yml index cb20924..1a2cc52 100644 --- a/ansible/molecule/slow-readiness/molecule.yml +++ b/ansible/molecule/slow-readiness/molecule.yml @@ -9,10 +9,12 @@ # # `baseline` is NOT exercised here (same rationale as the default scenario: # host-level hardening is real-host-only — see molecule/default/molecule.yml). -dependency: - name: galaxy - options: - requirements-file: ../../requirements.yml +# No `dependency` step (and no dependency config): molecule never vendored anything +# here. Its roles invoker reads `role-file`, not `requirements-file`, and its +# collections invoker resolves a relative path against the process CWD — so +# ../../requirements.yml pointed outside the repo and both halves just warned on every +# run. The collections come from `make deps` -> ansible/collections, surfaced by +# ANSIBLE_COLLECTIONS_PATH below. driver: name: docker platforms: @@ -31,11 +33,17 @@ provisioner: env: ANSIBLE_ROLES_PATH: "${MOLECULE_PROJECT_DIRECTORY}/roles" ANSIBLE_COLLECTIONS_PATH: "${MOLECULE_PROJECT_DIRECTORY}/collections" + # The docker connection plugin supports pipelining, which drops a `docker exec` + # round trip per remote task (measured at -22% on the `default` scenario, which + # also carries the coverage note). Safe here: the containers have no sudo requiretty. + ANSIBLE_PIPELINING: "true" verifier: name: ansible scenario: test_sequence: - - dependency + # Leading `destroy` so an orphaned container is never reused — see + # ../default/molecule.yml. + - destroy - create - prepare - converge diff --git a/ansible/molecule/validation/molecule.yml b/ansible/molecule/validation/molecule.yml index 2d702e4..ba36a4a 100644 --- a/ansible/molecule/validation/molecule.yml +++ b/ansible/molecule/validation/molecule.yml @@ -7,10 +7,12 @@ # here; there is no verify/idempotence step — the converge IS the assertion. Reuses # the default scenario's stub binary. `baseline` is not exercised (host-only — see # molecule/default/molecule.yml). -dependency: - name: galaxy - options: - requirements-file: ../../requirements.yml +# No `dependency` step (and no dependency config): molecule never vendored anything +# here. Its roles invoker reads `role-file`, not `requirements-file`, and its +# collections invoker resolves a relative path against the process CWD — so +# ../../requirements.yml pointed outside the repo and both halves just warned on every +# run. The collections come from `make deps` -> ansible/collections, surfaced by +# ANSIBLE_COLLECTIONS_PATH below. driver: name: docker platforms: @@ -29,9 +31,15 @@ provisioner: env: ANSIBLE_ROLES_PATH: "${MOLECULE_PROJECT_DIRECTORY}/roles" ANSIBLE_COLLECTIONS_PATH: "${MOLECULE_PROJECT_DIRECTORY}/collections" + # The docker connection plugin supports pipelining, which drops a `docker exec` + # round trip per remote task (measured at -22% on the `default` scenario, which + # also carries the coverage note). Safe here: the containers have no sudo requiretty. + ANSIBLE_PIPELINING: "true" scenario: test_sequence: - - dependency + # Leading `destroy` so an orphaned container is never reused — see + # ../default/molecule.yml. + - destroy - create - prepare - converge diff --git a/ansible/roles/baseline/defaults/main.yml b/ansible/roles/baseline/defaults/main.yml index d3cb7f7..d629ab9 100644 --- a/ansible/roles/baseline/defaults/main.yml +++ b/ansible/roles/baseline/defaults/main.yml @@ -65,3 +65,10 @@ baseline_packages: - fail2ban - unattended-upgrades - chrony + # `acl` is what lets Ansible hand a temp module file to an unprivileged become_user. + # decdn_node runs two tasks as the unprivileged decdn user (`decdn key-gen` and the + # `decdn config validate` gate); without it those fail on Debian with no rc at all, + # and the role can only report the failure after the fact ("needs the `acl` package + # on this host"). Molecule never caught this: it connects as root, where the same + # hand-off is a plain chown. + - acl