ci: cache Galaxy collections + pip, and bound every job (#53)

.github/workflows/ci.yml

@@ -21,6 +21,7 @@ jobs:
   # Detect whether ansible/ or the Helm chart changed so heavy jobs skip unrelated PRs.
   changes:
     runs-on: ubuntu-latest
+    timeout-minutes: 5          # ~5s in practice
     permissions:
       contents: read
       pull-requests: read
@@ -33,8 +34,11 @@ jobs:
         id: filter
         with:
           filters: |
+            # ci.yml is listed because the ansible jobs' cache wiring lives here, so a
+            # change to it must re-run them — same reason the helm filter claims it.
             ansible:
               - 'ansible/**'
+              - '.github/workflows/ci.yml'
             # The chart shares the schema-key inventory and checker with molecule.
             helm:
               - 'charts/**'
@@ -48,18 +52,60 @@ jobs:
     needs: changes
     if: needs.changes.outputs.ansible == 'true'
     runs-on: ubuntu-latest
+    timeout-minutes: 20         # ~1m15s in practice
     defaults:
       run:
         working-directory: ansible
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      # `cache: pip` needs a file to key on and this repo ships no pip manifest, so
+      # the workflow is the dependency declaration — it is where the package list
+      # literally lives. The installs stay unpinned `--upgrade`, so pip still queries
+      # the PyPI index to resolve; the cache saves the download, not the round trip.
+      # That is the right trade: PyPI is not the flaky dependency here, Galaxy is
+      # (see the collections cache below). setup-python derives its own key and takes
+      # no custom suffix, so this job and galaxy-build share one pip cache; their
+      # package sets overlap heavily (ansible-core, ansible-lint) so whichever
+      # populates it first covers most of both. To split them, drop `cache:` and use
+      # actions/cache on ~/.cache/pip with explicit per-job keys.
       - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
         with:
           python-version: '3.12'
+          cache: pip
+          cache-dependency-path: .github/workflows/ci.yml
       - name: Install Ansible tooling
         run: python -m pip install --upgrade ansible ansible-lint yamllint
+      # A warm collections tree removes the Galaxy round trip outright: with the
+      # requirements already satisfied on disk, `ansible-galaxy collection install`
+      # prints "Nothing to do" and never contacts a server. So a hit is IMMUNE to the
+      # transient `Connection reset by peer` that failed #50 mid-resolve on the
+      # transitive community.crypto — not merely faster than it.
+      #
+      # restore/save are split rather than the all-in-one action on purpose: the
+      # failure being defended against is Galaxy dying part way through an install,
+      # and the combined action's post step would persist that half-written tree.
+      # Saving only after `make deps` returns 0 keeps a poisoned cache out.
+      #
+      # No restore-keys — an older cache still satisfies the `>=` ranges in
+      # requirements.yml, so `make deps` would find nothing to do and the cache-epoch
+      # bump documented in that file would silently be a no-op.
+      #
+      # `path:` and `hashFiles()` resolve against the workspace root, NOT this job's
+      # `working-directory: ansible`. The key is shared with galaxy-build and with
+      # molecule.yml; when they miss together they race to save it and the losers log
+      # a harmless "Cache already exists".
+      - uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0
+        id: galaxy-cache
+        with:
+          path: ansible/collections
+          key: galaxy-${{ runner.os }}-${{ hashFiles('ansible/requirements.yml') }}
       - name: Install Galaxy collections
         run: make deps          # must precede lint/syntax-check
+      - uses: actions/cache/save@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0
+        if: steps.galaxy-cache.outputs.cache-hit != 'true'
+        with:
+          path: ansible/collections
+          key: galaxy-${{ runner.os }}-${{ hashFiles('ansible/requirements.yml') }}
       - name: Lint (yamllint + ansible-lint)
         run: make lint
       - name: Syntax-check playbook
@@ -75,22 +121,38 @@ jobs:
     needs: changes
     if: needs.changes.outputs.ansible == 'true'
     runs-on: ubuntu-latest
+    timeout-minutes: 20         # ~1m45s in practice
     defaults:
       run:
         working-directory: ansible
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      # Shares the pip cache with ansible-lint — see that job for why the key is
+      # the workflow file.
       - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
         with:
           python-version: '3.12'
+          cache: pip
+          cache-dependency-path: .github/workflows/ci.yml
       - name: Install build + import tooling
         run: python -m pip install --upgrade ansible-core ansible-lint galaxy-importer
+      # Same cache as ansible-lint (same key, same rationale — see that job).
+      - uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0
+        id: galaxy-cache
+        with:
+          path: ansible/collections
+          key: galaxy-${{ runner.os }}-${{ hashFiles('ansible/requirements.yml') }}
       - name: Vendor collection dependencies
         # So galaxy-importer's embedded ansible-lint can resolve the roles' FQCNs
         # (devsec.hardening, ansible.posix). Installed under ansible/collections.
         run: make deps
         env:
           ANSIBLE_COLLECTIONS_PATH: collections
+      - uses: actions/cache/save@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0
+        if: steps.galaxy-cache.outputs.cache-hit != 'true'
+        with:
+          path: ansible/collections
+          key: galaxy-${{ runner.os }}-${{ hashFiles('ansible/requirements.yml') }}
       - name: Build + validate the decdn.node collection
         run: make galaxy-check
         env:
@@ -110,6 +172,7 @@ jobs:
     needs: changes
     if: needs.changes.outputs.helm == 'true'
     runs-on: ubuntu-latest
+    timeout-minutes: 20         # pulls the kubeconform image; ~1m in practice
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
       # Helm version is pinned here AND in the kics job below; bump both.
@@ -146,6 +209,7 @@ jobs:
     needs: changes
     if: needs.changes.outputs.ansible == 'true' || needs.changes.outputs.helm == 'true'
     runs-on: ubuntu-latest
+    timeout-minutes: 20         # pulls the KICS engine image; ~30s in practice
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
       - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1
@@ -198,6 +262,7 @@ jobs:
   # Lint the workflow files themselves.
   actionlint:
     runs-on: ubuntu-latest
+    timeout-minutes: 10         # ~15s in practice
     permissions:
       contents: read
       checks: write

.github/workflows/molecule.yml

@@ -4,12 +4,14 @@ name: Molecule
 # Containerised converge + idempotence + verify for the decdn_node role.
 # Heavy (privileged systemd Docker container) — scoped to ansible/ changes and
 # blocking. Mark it a required status check in branch protection once proven.
+# This file is in `paths` alongside ansible/ so a change to the job itself (its cache
+# wiring, JOBS, the timeout) is exercised by the PR that makes it.
 on:
   pull_request:
-    paths: ['ansible/**']
+    paths: ['ansible/**', '.github/workflows/molecule.yml']
   push:
     branches: [main]
-    paths: ['ansible/**']
+    paths: ['ansible/**', '.github/workflows/molecule.yml']
 
 permissions:
   contents: read
@@ -21,29 +23,60 @@ concurrency:
 jobs:
   molecule:
     runs-on: ubuntu-latest      # Docker is preinstalled
+    # ~4m41s at JOBS=3. A bound, not a target: without one a wedged privileged
+    # systemd container burns the 360-minute default, and with cancel-in-progress
+    # above, some branch-protection setups read the resulting cancelled check as
+    # "not failed" rather than as a failure.
+    timeout-minutes: 45
     defaults:
       run:
         working-directory: ansible
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      # Keyed on this workflow file: no pip manifest exists, and the workflow is
+      # where the package list lives. ci.yml's ansible-lint job has the full note.
       - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
         with:
           python-version: '3.12'
+          cache: pip
+          cache-dependency-path: .github/workflows/molecule.yml
       - name: Install molecule + Ansible
         run: |
           python -m pip install --upgrade \
             molecule "molecule-plugins[docker]" ansible ansible-lint docker
+      # Same key and same rationale as ci.yml's jobs (see ansible-lint there): a warm
+      # tree makes the install a no-op that never contacts galaxy.ansible.com, and the
+      # save is gated so a part-way Galaxy failure cannot poison the cache. The key is
+      # derived wholly from ansible/requirements.yml, so it cannot drift from ci.yml's
+      # copy the way a duplicated env var would. It wraps
+      # `make molecule` because the deps install is owned by the Make target here, not
+      # by a step of its own.
+      - uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0
+        id: galaxy-cache
+        with:
+          path: ansible/collections
+          key: galaxy-${{ runner.os }}-${{ hashFiles('ansible/requirements.yml') }}
       - name: molecule test
         # `make molecule` runs every scenario under ansible/molecule/ and fans them out
         # in parallel; ansible/Makefile documents the set and the fail-loud guards.
         #
         # There is deliberately no separate `make deps` step: the molecule targets take
         # `deps` as a prerequisite, so a standalone one would resolve and install the
         # Galaxy requirements a second time every run — a second chance to trip over a
-        # flaky galaxy.ansible.com, for no added coverage.
+        # flaky galaxy.ansible.com, for no added coverage. The cache above wraps that
+        # Make-owned invocation instead.
         #
         # JOBS is capped at 3 rather than the default (one job per scenario, currently
         # 6): every scenario is a privileged systemd container, and they share this
         # runner's cores and cgroup hierarchy. If this job turns flaky, drop to JOBS=1
         # or swap in `make molecule-serial` — the latter also serialises the output.
         run: make molecule JOBS=3
+      # A step `if:` without a status-check function implies success(), so this is
+      # skipped when anything above failed — including a genuinely failing scenario,
+      # which leaves the cache cold for that run. Conservative on purpose: it is the
+      # same guard that keeps a part-way Galaxy install out of the cache.
+      - uses: actions/cache/save@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0
+        if: steps.galaxy-cache.outputs.cache-hit != 'true'
+        with:
+          path: ansible/collections
+          key: galaxy-${{ runner.os }}-${{ hashFiles('ansible/requirements.yml') }}

CONTRIBUTING.md

@@ -39,6 +39,19 @@ Run it on demand with `make lint-ansible`, or `pre-commit run ansible-lint --hoo
   role (privileged systemd Docker container; scoped to `ansible/**`). Run locally with
   `make molecule` (needs Docker) — it runs all six scenarios in parallel, so reach for
   `make molecule-serial` when you need to read a failure in order.
+- **Every job is bounded** by `timeout-minutes`. The values are bounds sized off
+  observed runtimes, not targets — without one a hung job burns the 360-minute
+  default, and combined with `cancel-in-progress` some branch-protection setups read
+  the resulting *cancelled* check as "not failed" rather than as a failure.
+- **Two caches.** `ansible/collections` is cached across `ansible-lint`,
+  `galaxy-build` and `molecule` under one shared key; a hit makes `make deps` a no-op
+  that never contacts `galaxy.ansible.com`, which is what keeps a transient Galaxy
+  error from failing an unrelated PR. pip is cached via `setup-python`, keyed on the
+  workflow file (the repo has no pip manifest, so the workflow *is* the package list);
+  the installs stay unpinned, so that saves the download but not the PyPI round trip.
+  Only the Galaxy cache is wired by hand — it uses `actions/cache`'s split
+  `restore`/`save` with `save` gated on success, so a part-way Galaxy failure can't
+  poison it. The pip cache is `setup-python`'s built-in one and manages itself.
 
 ## Supply-chain / pinning rules
 
@@ -57,8 +70,15 @@ Run it on demand with `make lint-ansible`, or `pre-commit run ansible-lint --hoo
 - **Dependabot** (`.github/dependabot.yml`) bumps the other action SHAs weekly.
 - **Bump manually** (Dependabot can't): the `KICS_IMAGE` and `KUBECONFORM_IMAGE` digests
   in the `Makefile`, both `setup-helm` `version:` inputs in `ci.yml` (`helm` and `kics`
-  jobs),
-  and the pre-commit hook revs via `pre-commit autoupdate`.
+  jobs), the collection versions in `ansible/requirements.yml`, and the pre-commit hook
+  revs via `pre-commit autoupdate`.
+- **Bump the `cache-epoch:` counter in `ansible/requirements.yml` to make CI
+  re-resolve the collections.** Those are `>=` ranges, so a warm cache pins the
+  resolved set — transitive collections like `community.crypto` included — until the
+  file changes; the counter forces a fresh resolve without editing the requirements
+  themselves. It is a comment, but a load-bearing one: the cache key is that file's
+  hash. Keeping it *in* the hashed file is deliberate — an epoch duplicated across
+  both workflows could drift, since neither workflow runs on a change to the other.
 
 ## Solidity
 

ansible/requirements.yml

@@ -1,6 +1,20 @@
 ---
 # Galaxy collections this project composes. Install with `make deps`
 # (ansible-galaxy collection install -r requirements.yml -p collections).
+#
+# CI caches the resolved tree under a key derived from THIS FILE's hash
+# (.github/workflows/{ci,molecule}.yml). The versions below are `>=` ranges, not
+# pins, so a warm cache freezes whatever was resolved — transitive collections like
+# community.crypto included — until this file changes.
+#
+# The counter below is the lever to force a fresh resolve without otherwise editing
+# the requirements. It is LOAD-BEARING despite being a comment: bumping it changes
+# this file's hash, and therefore the cache key. It lives here, rather than as an env
+# var in each workflow, so the key has exactly one source — two copies could drift
+# apart silently, since neither workflow runs on a change to the other. Editing it
+# also matches both workflows' `ansible/**` trigger, so a bump is exercised at once.
+#
+#   cache-epoch: 1
 collections:
   - name: devsec.hardening      # os_hardening + ssh_hardening (CIS-style baseline)
     version: ">=10.0.0"