diff --git a/.github/workflows/supply-chain.yml b/.github/workflows/supply-chain.yml index aeffe02161..5cf10e2339 100644 --- a/.github/workflows/supply-chain.yml +++ b/.github/workflows/supply-chain.yml @@ -175,6 +175,304 @@ jobs: GITHUB_TOKEN: ${{ github.token }} run: python3 scripts/check_action_refs.py + actions-security: + name: Security audit (workflows, package-lock, Python source) + runs-on: ubuntu-latest + # CI is part of the supply chain: a workflow step runs with this repo's + # token and secrets, and its output is packaged into the wheel PyPI + # serves. Both private repos in this family (clawmetry-cloud, + # clawmetry-pro) have run zizmor on every PR for weeks; this repo, with + # the most workflow definitions of the three and the only public attack + # surface, was the one running no Actions-security scanner. Scorecard's + # `Dangerous-Workflow` and `Token-Permissions` probes below cover part + # of the same ground, but they run only on push (`if:` above) and score + # the repository rather than naming the file and line. + # + # Three scanning responsibilities, matching the SecurityAuditScanner the + # private repos already run: + # 1. workflow definitions -> zizmor + # 2. dependency manifests -> pip-audit (the `python-deps` job + # above, on requirements.txt) + `npm audit` here, on every committed + # package-lock.json. The npm half is the one that had no scanner at + # all: this repo's largest dependency tree is `frontend/`, and it + # ships into `clawmetry/static/`. + # 3. Python source -> bandit + # pip-audit stays in `python-deps` rather than moving here: it has its own + # SBOM artifact and its own red condition, and splitting a passing job to + # tidy the layout is churn, not hardening. + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.11" + # The npm-audit step below needs a Node of its own. Without this it runs + # against whatever the runner image happens to ship, which is not a + # hypothetical: clawmetry-pro#234 hit exactly that, where an unpinned old + # npm returned a 400 from the retired `audits/quick` endpoint npm 6 uses. + # That matters more here than it did there, because this job's gate + # treats an npm-audit outage as RED — an image bump could turn `main` + # red with nothing in the diff to explain it. + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: "20" + # A GitHub Actions cache is not trusted input, and this job reads + # lockfiles to decide whether the tree is vulnerable. Nothing is + # installed here anyway (`npm audit --package-lock-only`), so the + # cache would buy nothing and could only muddy the result. Same + # reasoning as publish.yml. + package-manager-cache: false + - run: python3 -m pip install --quiet zizmor bandit + + - name: Audit every workflow definition + env: + # zizmor resolves action tags through the GitHub API. Unauthenticated, + # those calls share the runner's IP rate limit and fail intermittently + # — which, because a reachability failure aborts the WHOLE audit, is + # the difference between a scan and no scan. Read-only; the job's + # `contents: read` default is enough to read public action repos. + GH_TOKEN: ${{ github.token }} + run: | + mkdir -p audit + # Workflow definitions only, which is the scope the SecurityAuditScanner + # component describes. + # + # The composite action under .github/actions/ is a real gap and is NOT + # covered here: its steps run inline in the calling job, with that job's + # token and secrets, so it carries the same rule families a workflow + # does, and nothing scans it today. Widening this input set to include + # it is a one-line change, but it is a scope the blueprint does not + # describe, so it needs the product record first rather than arriving + # as a side effect of turning the scanner on. Tracked as follow-up. + inputs=".github/workflows" + # Record the file set the scanner was handed. A finding list is + # identical whether it audited thirty-seven files or none, so + # coverage is the one thing the JSON cannot tell you afterwards. + # shellcheck disable=SC2086 + find $inputs -type f \( -name '*.yml' -o -name '*.yaml' \) \ + | sort > audit/zizmor-inputs.txt + n_inputs=$(wc -l < audit/zizmor-inputs.txt | tr -d '[:space:]') + echo "$n_inputs" > audit/zizmor-input-count.txt + echo "zizmor inputs ($n_inputs file(s)):" + sed 's/^/ /' audit/zizmor-inputs.txt + + # `--no-exit-codes` makes FINDINGS exit 0, so a non-zero exit here + # means only "the tool failed". Without it the two cases are + # indistinguishable, and the usual `|| true` launders a crash — which + # writes NO json — into an empty file that reads as a clean scan. + rc=0 + # shellcheck disable=SC2086 + zizmor --no-exit-codes --format json --persona regular $inputs \ + > audit/zizmor.json 2> audit/zizmor.err || rc=$? + # An empty input set would let zizmor exit 0 having audited nothing: + # the same "clean scan" impostor as a crash. + if [ "$n_inputs" -eq 0 ]; then + echo "zizmor was handed no files to audit" >> audit/zizmor.err + if [ "$rc" -eq 0 ]; then rc=1; fi + fi + # A crash leaves no json at all, so reject anything that does not + # parse as a list. + if ! python3 -c "import json; d = json.load(open('audit/zizmor.json')); assert isinstance(d, list); print(f'zizmor: {len(d)} finding(s)')" 2>/dev/null; then + if [ "$rc" -eq 0 ]; then rc=1; fi + fi + if [ "$rc" -ne 0 ]; then + # Recorded, not raised: the artifact still has to upload, and the + # gate at the end of the job is what turns this red — once there + # is something to diagnose it from. + echo "$rc" > audit/zizmor.failed + echo "::error::zizmor did not complete (exit $rc); this is a scanner outage, not a clean scan" + sed 's/^/ /' audit/zizmor.err || true + fi + + # Responsibility 3: static analysis on this repo's Python source. + # Scorecard's SAST probe with teeth — it scores the repository, this + # names the file and line. + - name: Scan Python source (bandit) + run: | + # -ll: medium severity and above. Low-severity bandit over a tree + # this size is thousands of `assert` and `subprocess` notes, which + # buries the findings that matter. + # + # bandit exits 1 on FINDINGS and on failure alike, so — same + # reasoning as the zizmor step above — the exit code cannot tell a + # finding from an outage. The report is what separates them: a real + # run writes a dict with a `results` list, a crash writes nothing. + bandit -r . -f json -o audit/bandit.json -ll \ + -x ./node_modules,./.venv,./venv,./tests,./frontend,./.git || true + if ! python3 -c "import json; d = json.load(open('audit/bandit.json')); print(f\"bandit: {len(d['results'])} finding(s)\")" 2>/dev/null; then + echo 1 > audit/bandit.failed + echo "::error::bandit did not produce a usable report; this is a scanner outage, not a clean scan" + fi + + # Responsibility 2 (the package-lock half). Every committed lockfile, + # not just frontend/: /tests/e2e, /.github/scripts and the rest are all + # installed by CI jobs that run with this repo's token, and auditing one + # lockfile would report "clean" for the ones nobody scanned. + - name: Scan npm dependencies (package-lock) + run: | + locks=$(find . -name package-lock.json -not -path '*/node_modules/*' -not -path './.git/*' | sort) + if [ -z "$locks" ]; then + echo "no package-lock.json — nothing to audit" + exit 0 + fi + while IFS= read -r lock; do + dir=$(dirname "$lock") + # `.github/scripts` -> `github-scripts`: one report per lockfile, + # named after the directory it audited. + slug=$(printf '%s' "${dir#./}" | tr '/' '-' | tr -d '.') + if [ "$dir" = "." ]; then slug="root"; fi + out="audit/npm-audit-$slug.json" + err="audit/npm-audit-$slug.err" + echo "auditing $lock -> $out" + # `npm audit` exits non-zero BOTH when it finds vulnerabilities and + # when it fails outright, so `|| true` on its own launders a + # registry error into what reads as a clean scan. The shape of the + # output is what separates them: a real result carries + # .metadata.vulnerabilities.total, an error response carries + # .message instead. + ( cd "$dir" && npm audit --package-lock-only --json ) > "$out" 2> "$err" || true + if ! python3 -c "import json; d=json.load(open('$out')); t=d['metadata']['vulnerabilities']['total']; assert isinstance(t, int); print(f' npm audit ($lock): {t} vulnerability(ies)')" 2>/dev/null; then + echo 1 > audit/npm-audit.failed + echo "::error::npm audit did not produce a result for $lock; this is a scanner outage, not a clean scan" + python3 -c "import json; print(' registry said:', json.load(open('$out')).get('message', ''))" 2>/dev/null || true + sed 's/^/ /' "$err" 2>/dev/null || true + fi + done <<< "$locks" + + # Findings are reported, not gated. Today's tree has findings that are + # load-bearing by design — the release, quarantine and cloud-pin jobs + # persist their checkout credential BECAUSE they push with it — and a + # gate that fails on those would be turned off within a week. The + # summary is what makes a NEW finding visible in the PR that adds it. + - name: Summarise findings + if: always() + run: | + python3 - <<'PY' >> "$GITHUB_STEP_SUMMARY" + import collections, json, pathlib + + print("### Security audit\n") + print("| Scanner | Scope | Findings |") + print("| --- | --- | ---: |") + + def bandit_row(): + if pathlib.Path("audit/bandit.failed").exists(): + return "**SCAN FAILED**" + try: + return str(len(json.loads(pathlib.Path("audit/bandit.json").read_text())["results"])) + except Exception: + return "**SCAN FAILED**" + + def npm_rows(): + if pathlib.Path("audit/npm-audit.failed").exists(): + yield "npm audit", "package-lock", "**SCAN FAILED**" + for f in sorted(pathlib.Path("audit").glob("npm-audit-*.json")): + try: + t = json.loads(f.read_text())["metadata"]["vulnerabilities"]["total"] + except Exception: + t = "**SCAN FAILED**" + yield "npm audit", f"`{f.stem[len('npm-audit-'):]}`", str(t) + + p = pathlib.Path("audit/zizmor.json") + n = pathlib.Path("audit/zizmor-input-count.txt") + covered = n.read_text().strip() if n.exists() else "?" + zizmor_failed = pathlib.Path("audit/zizmor.failed").exists() or not p.exists() + findings = [] + if not zizmor_failed: + try: + findings = json.loads(p.read_text()) + except Exception: + zizmor_failed = True + print( + f"| zizmor | workflow definitions ({covered} file(s)) | " + + ("**SCAN FAILED**" if zizmor_failed else str(len(findings))) + + " |" + ) + for name, scope, count in npm_rows(): + print(f"| {name} | {scope} | {count} |") + print(f"| bandit | Python source (medium+) | {bandit_row()} |") + print("\nFull reports: the `actions-security-audit` artifact.\n") + + # bandit's total is a baseline number on a tree this size; the + # severity/confidence split is what says whether anything in it + # needs reading today. + try: + b = json.loads(pathlib.Path("audit/bandit.json").read_text())["results"] + except Exception: + b = [] + if b: + print("#### Python source (bandit)\n") + brank = {"HIGH": 0, "MEDIUM": 1, "LOW": 2} + by_test = collections.Counter( + (r.get("issue_severity") or "?", r.get("issue_confidence") or "?", + r.get("test_id") or "?", r.get("test_name") or "?") + for r in b + ) + print("| Severity | Confidence | Rule | Findings |") + print("| --- | --- | --- | ---: |") + for (sev, conf, tid, tname), c in sorted( + by_test.items(), + key=lambda kv: (brank.get(kv[0][0], 9), brank.get(kv[0][1], 9), kv[0][2]), + ): + print(f"| {sev} | {conf} | `{tid}` {tname} | {c} |") + print() + + # The zizmor breakdown below is the part a reviewer reads to spot a + # NEW workflow finding in the PR that introduced it. + print("#### GitHub Actions security (zizmor)\n") + if zizmor_failed: + print("**SCAN FAILED** — see the `actions-security-audit` artifact.") + raise SystemExit(0) + print(f"{len(findings)} finding(s) across {covered} audited file(s).\n") + if not findings: + raise SystemExit(0) + rank = {"High": 0, "Medium": 1, "Low": 2, "Informational": 3} + counts = collections.Counter( + (f.get("determinations", {}).get("severity") or "?") for f in findings + ) + print("| Severity | Findings |") + print("| --- | ---: |") + for sev, c in sorted(counts.items(), key=lambda kv: rank.get(kv[0], 9)): + print(f"| {sev} | {c} |") + print() + by_rule = collections.Counter( + ( + (f.get("determinations", {}).get("severity") or "?"), + (f.get("ident") or "?"), + ) + for f in findings + ) + print("| Severity | Rule | Findings |") + print("| --- | --- | ---: |") + for (sev, rule), c in sorted( + by_rule.items(), key=lambda kv: (rank.get(kv[0][0], 9), kv[0][1]) + ): + print(f"| {sev} | `{rule}` | {c} |") + PY + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: always() + with: + name: actions-security-audit + path: audit/ + retention-days: 30 + + # The only red condition: the scanner did not run. A silent outage is + # the failure mode this whole job exists to avoid, because it looks + # exactly like a clean result. + - name: Fail if a scanner did not run + if: always() + run: | + rc=0 + for s in zizmor bandit npm-audit; do + if [ -f "audit/$s.failed" ]; then + echo "$s did not produce a usable report; treating as a failed scan." + rc=1 + fi + done + if [ "$rc" -eq 0 ]; then echo "every scanner completed."; fi + exit "$rc" + scorecard: name: OpenSSF Scorecard runs-on: ubuntu-latest