perf(molecule): run scenarios in parallel, enable pipelining

.github/workflows/molecule.yml

@@ -33,14 +33,17 @@ jobs:
         run: |
           python -m pip install --upgrade \
             molecule "molecule-plugins[docker]" ansible ansible-lint docker
-      - name: Install Galaxy collections
-        run: make deps
       - name: molecule test
-        # --all runs every scenario under ansible/molecule/: default
-        # (operator-provisioned keystore + rendered-value assertions), schema
-        # (config key-set drift against the upstream field list), validation
-        # (bad knobs must be rejected by the role's own asserts), generate-keystore
-        # (opt-in host-side wallet generation), host-env (host-provisioned
-        # /etc/decdn/decdn.env — no secret on the control machine), and
-        # slow-readiness (advisory /metrics probe timeout).
-        run: molecule test --all
+        # `make molecule` runs every scenario under ansible/molecule/ and fans them out
+        # in parallel; ansible/Makefile documents the set and the fail-loud guards.
+        #
+        # There is deliberately no separate `make deps` step: the molecule targets take
+        # `deps` as a prerequisite, so a standalone one would resolve and install the
+        # Galaxy requirements a second time every run — a second chance to trip over a
+        # flaky galaxy.ansible.com, for no added coverage.
+        #
+        # JOBS is capped at 3 rather than the default (one job per scenario, currently
+        # 6): every scenario is a privileged systemd container, and they share this
+        # runner's cores and cgroup hierarchy. If this job turns flaky, drop to JOBS=1
+        # or swap in `make molecule-serial` — the latter also serialises the output.
+        run: make molecule JOBS=3

AGENTS.md

@@ -103,7 +103,9 @@ deploys (its targets must run from `ansible/`). `make help` lists root targets.
 make hooks            # one-time: install pre-commit git hook (pip install pre-commit first)
 make lint             # all pre-commit hooks on all files (hygiene, shellcheck, yamllint, markdown)
 make lint-ansible     # vendor collections + full ansible-lint (production profile)
-make molecule         # containerised converge/verify of the decdn_node role (needs Docker)
+make molecule         # containerised converge/verify of the decdn_node role — all six
+                      # scenarios in parallel (needs Docker); cap with JOBS=<n>
+make molecule-serial  # the same suite one scenario at a time (readable failure output)
 make lint-helm        # chart: helm lint + render tests + kubeconform + schema keys (needs helm, yq, Docker)
 make security         # = security-ansible + security-helm (KICS over the rendered chart; needs helm)
 

CONTRIBUTING.md

@@ -37,7 +37,8 @@ Run it on demand with `make lint-ansible`, or `pre-commit run ansible-lint --hoo
   run via **pre-commit locally only** (`make hooks` / `make lint`), not in CI.
 - **`molecule.yml`** — containerised converge + idempotence + verify of the `decdn_node`
   role (privileged systemd Docker container; scoped to `ansible/**`). Run locally with
-  `make molecule` (needs Docker).
+  `make molecule` (needs Docker) — it runs all six scenarios in parallel, so reach for
+  `make molecule-serial` when you need to read a failure in order.
 
 ## Supply-chain / pinning rules
 

Makefile

@@ -1,6 +1,6 @@
 # Convenience targets for the deCDN DevOps monorepo.
 # Run from the repo root. Ansible-specific work is delegated to ansible/Makefile.
-.PHONY: help hooks lint lint-ansible lint-helm security security-ansible security-helm molecule galaxy-build galaxy-check
+.PHONY: help hooks lint lint-ansible lint-helm security security-ansible security-helm molecule molecule-serial galaxy-build galaxy-check
 SHELL := /bin/bash
 
 # KICS runs straight from the engine image, pinned by digest. This target IS the
@@ -59,9 +59,12 @@ security-helm:       ## KICS scan of the decdn-node chart's rendered manifests (
 lint-helm:           ## helm lint + render tests + kubeconform + shared schema-key check (needs helm, yq, python3>=3.11, docker)
 	KUBECONFORM="docker run --rm -i $(KUBECONFORM_IMAGE)" $(CHART)/tests/render-test.sh
 
-molecule:            ## containerised converge/verify of the decdn_node role (needs Docker)
+molecule:            ## containerised converge/verify of the decdn_node role, scenarios in parallel (needs Docker)
 	$(MAKE) -C ansible molecule
 
+molecule-serial:     ## same suite, one scenario at a time (readable output on failure)
+	$(MAKE) -C ansible molecule-serial
+
 galaxy-build:        ## stage + build the decdn.node Galaxy collection artifact
 	$(MAKE) -C ansible build
 

ansible/Makefile

@@ -1,6 +1,6 @@
 # Convenience targets for the deCDN Ansible project.
 # Always run from the ansible/ directory.
-.PHONY: deps lint check deploy molecule build galaxy-check
+.PHONY: deps lint check deploy molecule molecule-serial build galaxy-check
 SHELL := /bin/bash
 
 # Install the required Galaxy collections (>= constraints in requirements.yml)
@@ -67,11 +67,57 @@ deploy:
 # stub daemon (needs Docker; a privileged systemd container). See molecule/.
 # Scenarios: default (rendered values + idempotence), schema (config key-set drift
 # against the upstream field list), validation (bad knobs must be rejected by the
-# role's own asserts), generate-keystore (opt-in host-side wallet), slow-readiness
-# (advisory /metrics probe timeout).
-# `--all` runs every scenario: `default` (operator-provisioned keystore) and
-# `generate-keystore` (opt-in host-side wallet generation).
-molecule:
+# role's own asserts), generate-keystore (opt-in host-side wallet), host-env
+# (host-provisioned /etc/decdn/decdn.env), slow-readiness (advisory /metrics probe
+# timeout).
+#
+# The scenarios run concurrently: distinct container names, no published host ports,
+# per-scenario ephemeral dirs, and what they share on the control machine (the
+# vendored collections, roles/, the stub under default/files/) they only read.
+# 595s -> 151s on a 16-core box. They DO share one Docker daemon and the host cgroup
+# hierarchy — every platform is a privileged systemd container — which is why CI caps
+# JOBS. Output interleaves, hence the [scenario] prefix; `molecule-serial` is the
+# escape hatch when a failure needs reading in order. Override the fan-out width with
+# e.g. `make molecule JOBS=2` on a small machine.
+#
+# Molecule's own --workers is not usable here: it refuses to run outside collection
+# mode ("--workers > 1 is only supported in collection mode (galaxy.yml required)"),
+# and this project deliberately has no galaxy.yml at its root — that would make
+# ansible-lint reinterpret the deploy project as a collection (see galaxy/README.md).
+#
+# Fail loud (hard rule 4), in three places, because a test harness that passes when it
+# did not run is worse than a slow one:
+#   - the guard below refuses an empty or truncated scenario set. Discovery moved from
+#     molecule to a glob, and a glob that matches nothing (wrong cwd, renamed layout)
+#     would otherwise run zero scenarios and exit 0 — a green build that tested nothing.
+#   - pipefail stops the [scenario] prefixing pipe from masking molecule's status, and
+#     each scenario announces its own failure by name (the bare xargs message does not).
+#   - xargs then exits non-zero. The `|| exit 1` in the wrapper normalises every
+#     scenario failure to 1, which is what keeps xargs off its own abort-on-255 path —
+#     that one stops launching queued scenarios, so a single unlucky exit code would
+#     otherwise cut the suite short. So: 123 when a scenario failed, 125 if one was
+#     killed by a signal, and every scenario always gets to run.
+SCENARIOS := $(notdir $(patsubst %/,%,$(dir $(wildcard molecule/*/molecule.yml))))
+SCENARIO_DIRS := $(notdir $(patsubst %/,%,$(wildcard molecule/*/)))
+JOBS ?= $(words $(SCENARIOS))
+
+molecule: deps
+	@test -n "$(strip $(SCENARIOS))" && test "$(strip $(SCENARIOS))" = "$(strip $(SCENARIO_DIRS))" || { \
+	  echo "scenario discovery failed: molecule.yml in [$(SCENARIOS)] but scenario dirs are [$(SCENARIO_DIRS)]"; \
+	  echo "refusing to run a silently-truncated suite (this target must run from ansible/)"; exit 1; }
+	@case '$(JOBS)' in ''|*[!0-9]*) \
+	  echo "JOBS must be a positive integer (got '$(JOBS)')"; exit 1 ;; esac
+	@test "$$((10#$(JOBS)))" -gt 0 || { \
+	  echo "JOBS must be a positive integer (got '$(JOBS)'): any all-zero value — 0, 00, 000 —"; \
+	  echo "reaches xargs as -P 0, which means UNLIMITED concurrency, not none"; exit 1; }
+	@printf '%s\n' $(SCENARIOS) | xargs -P $(JOBS) -I{} \
+		bash -c 'set -euo pipefail; molecule test -s {} --no-command-borders 2>&1 | sed -u "s/^/[{}] /" \
+		  || { echo "[{}] SCENARIO FAILED"; exit 1; }'
+
+# The same suite, one scenario at a time (molecule's own --all). It stops at the FIRST
+# failing scenario (--continue-on-failure defaults off), so it reports less than the
+# parallel target — but readably, which is the point.
+molecule-serial: deps
 	molecule test --all
 
 # --- Galaxy collection (decdn.node) ------------------------------------------

ansible/README.md

@@ -191,11 +191,22 @@ primitives underneath it. All take `--dry-run`. See
 ```bash
 make lint           # yamllint + ansible-lint (production profile)
 ansible-playbook playbooks/site.yml --syntax-check
-make molecule       # containerised converge + idempotence + verify (needs Docker)
+make molecule       # all six molecule scenarios, in parallel (needs Docker)
+make molecule JOBS=2   # …capped to two at a time on a small machine
+make molecule-serial   # …one at a time, when a failure needs readable output
 ```
 
-`make molecule` converges the **`decdn_node`** role in a privileged systemd container
-against a stub daemon (`molecule/default/`): it installs via the `manual` method (no
+`make molecule` runs every scenario under `molecule/`: **`default`** (described below),
+`schema` (config key-set drift against the upstream field list), `validation` (bad knobs
+must be rejected by the role's own asserts), `generate-keystore` (opt-in host-side
+wallet), `host-env` (host-provisioned `/etc/decdn/decdn.env`) and `slow-readiness`
+(advisory `/metrics` probe timeout). They are independent, so they run concurrently —
+~151s instead of ~595s — and each line of output is prefixed with its scenario name
+because the runs interleave. `make molecule-serial` is the escape hatch when that
+interleaving gets in the way of reading a failure.
+
+The `default` scenario converges the **`decdn_node`** role in a privileged systemd
+container against a stub daemon: it installs via the `manual` method (no
 published release needed), stages a placeholder keystore, renders `node.toml` + the
 hardened unit, starts the service, and passes the role's own `/metrics` readiness probe;
 `verify.yml` then asserts the node user, valid TOML, a valid systemd unit, loopback-only

ansible/galaxy/CHANGELOG.md

@@ -6,6 +6,14 @@ collection adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html)
 
 ## [Unreleased]
 
+### Added
+
+- `baseline_packages` now includes `acl`. `decdn_node` runs two tasks as the
+  unprivileged `decdn` user (`decdn key-gen`, and the `decdn config validate` gate),
+  and on Debian Ansible needs ACL support to hand the temp module file to that user.
+  Without it both fail without an `rc`, which the role can only report after the fact
+  ("becoming the unprivileged decdn user needs the `acl` package on this host").
+
 ### Removed
 
 - `decdn_delivery_floor` and `decdn_rate_bounds_poll_interval_sec`: upstream

ansible/molecule/default/molecule.yml

@@ -11,10 +11,12 @@
 # ssh_hardening (sysctl, mount, auth, sshd) and fail2ban — none of which behave
 # meaningfully in a throwaway systemd container. It is validated on real hosts via
 # `make check` instead.
-dependency:
-  name: galaxy
-  options:
-    requirements-file: ../../requirements.yml
+# No `dependency` step (and no dependency config): molecule never vendored anything
+# here. Its roles invoker reads `role-file`, not `requirements-file`, and its
+# collections invoker resolves a relative path against the process CWD — so
+# ../../requirements.yml pointed outside the repo and both halves just warned on every
+# run. The collections come from `make deps` -> ansible/collections, surfaced by
+# ANSIBLE_COLLECTIONS_PATH below.
 driver:
   name: docker
 platforms:
@@ -34,11 +36,24 @@ provisioner:
   env:
     ANSIBLE_ROLES_PATH: "${MOLECULE_PROJECT_DIRECTORY}/roles"
     ANSIBLE_COLLECTIONS_PATH: "${MOLECULE_PROJECT_DIRECTORY}/collections"
+    # The docker connection plugin supports pipelining, which drops a `docker exec`
+    # round trip per remote task — 105s -> 82s measured on this scenario. Safe here:
+    # the containers have no sudo requiretty.
+    # Known coverage boundary: with pipelining on, command/shell modules are fed over
+    # stdin, so the remote-tmp path behind `become_user` is not exercised. Molecule
+    # connects as root, where that path is a plain chown, so it never reproduced the
+    # real-host `acl` failure the role warns about (tasks/main.yml) either way.
+    ANSIBLE_PIPELINING: "true"
 verifier:
   name: ansible
 scenario:
   test_sequence:
-    - dependency
+    # The leading `destroy` is deliberate. The docker driver creates with
+    # `recreate: false`, so a container orphaned by an aborted run (Ctrl-C, OOM —
+    # both likelier now that the scenarios run in parallel) would be REUSED, and
+    # `converge` + `idempotence` would then pass trivially against a host that was
+    # already converged by the previous run.
+    - destroy
     - create
     - prepare
     - converge

ansible/molecule/generate-keystore/molecule.yml

@@ -8,10 +8,12 @@
 # (host-level hardening is meaningless in a throwaway systemd container) — see
 # ../default/molecule.yml. This scenario reuses the shared stub daemon under
 # ../default/files/decdn-node-stub (which implements a `key-gen` subcommand).
-dependency:
-  name: galaxy
-  options:
-    requirements-file: ../../requirements.yml
+# No `dependency` step (and no dependency config): molecule never vendored anything
+# here. Its roles invoker reads `role-file`, not `requirements-file`, and its
+# collections invoker resolves a relative path against the process CWD — so
+# ../../requirements.yml pointed outside the repo and both halves just warned on every
+# run. The collections come from `make deps` -> ansible/collections, surfaced by
+# ANSIBLE_COLLECTIONS_PATH below.
 driver:
   name: docker
 platforms:
@@ -30,13 +32,19 @@ provisioner:
   env:
     ANSIBLE_ROLES_PATH: "${MOLECULE_PROJECT_DIRECTORY}/roles"
     ANSIBLE_COLLECTIONS_PATH: "${MOLECULE_PROJECT_DIRECTORY}/collections"
+    # The docker connection plugin supports pipelining, which drops a `docker exec`
+    # round trip per remote task (measured at -22% on the `default` scenario, which
+    # also carries the coverage note). Safe here: the containers have no sudo requiretty.
+    ANSIBLE_PIPELINING: "true"
 verifier:
   name: ansible
 scenario:
   # No `prepare` — the role must generate the wallet itself. `idempotence` proves
   # the stat/`creates:` gating makes a second converge a no-op (nothing re-minted).
   test_sequence:
-    - dependency
+    # Leading `destroy` so an orphaned container is never reused — see
+    # ../default/molecule.yml.
+    - destroy
     - create
     - converge
     - idempotence

ansible/molecule/host-env/molecule.yml

@@ -10,10 +10,12 @@
 # (host-level hardening is meaningless in a throwaway systemd container) — see
 # ../default/molecule.yml. This scenario reuses the shared stub daemon under
 # ../default/files/decdn-node-stub.
-dependency:
-  name: galaxy
-  options:
-    requirements-file: ../../requirements.yml
+# No `dependency` step (and no dependency config): molecule never vendored anything
+# here. Its roles invoker reads `role-file`, not `requirements-file`, and its
+# collections invoker resolves a relative path against the process CWD — so
+# ../../requirements.yml pointed outside the repo and both halves just warned on every
+# run. The collections come from `make deps` -> ansible/collections, surfaced by
+# ANSIBLE_COLLECTIONS_PATH below.
 driver:
   name: docker
 platforms:
@@ -32,6 +34,10 @@ provisioner:
   env:
     ANSIBLE_ROLES_PATH: "${MOLECULE_PROJECT_DIRECTORY}/roles"
     ANSIBLE_COLLECTIONS_PATH: "${MOLECULE_PROJECT_DIRECTORY}/collections"
+    # The docker connection plugin supports pipelining, which drops a `docker exec`
+    # round trip per remote task (measured at -22% on the `default` scenario, which
+    # also carries the coverage note). Safe here: the containers have no sudo requiretty.
+    ANSIBLE_PIPELINING: "true"
 verifier:
   name: ansible
 scenario:
@@ -40,7 +46,9 @@ scenario:
   # converge must restart the daemon (verify.yml compares MainPID across it). The
   # order matters: idempotence has to run while the file is still untouched.
   test_sequence:
-    - dependency
+    # Leading `destroy` so an orphaned container is never reused — see
+    # ../default/molecule.yml.
+    - destroy
     - create
     - prepare
     - converge

ansible/molecule/schema/molecule.yml

@@ -12,10 +12,12 @@
 # `baseline` is NOT exercised here, for the same reasons as the `default` scenario.
 # There is no idempotence step: this scenario asserts on rendered content, and
 # `default` already covers idempotence.
-dependency:
-  name: galaxy
-  options:
-    requirements-file: ../../requirements.yml
+# No `dependency` step (and no dependency config): molecule never vendored anything
+# here. Its roles invoker reads `role-file`, not `requirements-file`, and its
+# collections invoker resolves a relative path against the process CWD — so
+# ../../requirements.yml pointed outside the repo and both halves just warned on every
+# run. The collections come from `make deps` -> ansible/collections, surfaced by
+# ANSIBLE_COLLECTIONS_PATH below.
 driver:
   name: docker
 platforms:
@@ -35,11 +37,17 @@ provisioner:
   env:
     ANSIBLE_ROLES_PATH: "${MOLECULE_PROJECT_DIRECTORY}/roles"
     ANSIBLE_COLLECTIONS_PATH: "${MOLECULE_PROJECT_DIRECTORY}/collections"
+    # The docker connection plugin supports pipelining, which drops a `docker exec`
+    # round trip per remote task (measured at -22% on the `default` scenario, which
+    # also carries the coverage note). Safe here: the containers have no sudo requiretty.
+    ANSIBLE_PIPELINING: "true"
 verifier:
   name: ansible
 scenario:
   test_sequence:
-    - dependency
+    # Leading `destroy` so an orphaned container is never reused — see
+    # ../default/molecule.yml.
+    - destroy
     - create
     - prepare
     - converge

ansible/molecule/slow-readiness/molecule.yml

@@ -9,10 +9,12 @@
 #
 # `baseline` is NOT exercised here (same rationale as the default scenario:
 # host-level hardening is real-host-only — see molecule/default/molecule.yml).
-dependency:
-  name: galaxy
-  options:
-    requirements-file: ../../requirements.yml
+# No `dependency` step (and no dependency config): molecule never vendored anything
+# here. Its roles invoker reads `role-file`, not `requirements-file`, and its
+# collections invoker resolves a relative path against the process CWD — so
+# ../../requirements.yml pointed outside the repo and both halves just warned on every
+# run. The collections come from `make deps` -> ansible/collections, surfaced by
+# ANSIBLE_COLLECTIONS_PATH below.
 driver:
   name: docker
 platforms:
@@ -31,11 +33,17 @@ provisioner:
   env:
     ANSIBLE_ROLES_PATH: "${MOLECULE_PROJECT_DIRECTORY}/roles"
     ANSIBLE_COLLECTIONS_PATH: "${MOLECULE_PROJECT_DIRECTORY}/collections"
+    # The docker connection plugin supports pipelining, which drops a `docker exec`
+    # round trip per remote task (measured at -22% on the `default` scenario, which
+    # also carries the coverage note). Safe here: the containers have no sudo requiretty.
+    ANSIBLE_PIPELINING: "true"
 verifier:
   name: ansible
 scenario:
   test_sequence:
-    - dependency
+    # Leading `destroy` so an orphaned container is never reused — see
+    # ../default/molecule.yml.
+    - destroy
     - create
     - prepare
     - converge