From ff5db0c4892d3967de4ea2cf6047a3685cebff5d Mon Sep 17 00:00:00 2001 From: John Osumi <931193+sumitake@users.noreply.github.com> Date: Mon, 3 Aug 2026 23:12:30 -0700 Subject: [PATCH] tests: property-based action-pin assertions (unbreak Dependabot bumps) Replace the four frozen action-pin fixtures (exact gitleaks SHA; three '# v4' major-version regexes for codeql init/analyze and upload-artifact) with anchored property assertions: a real 'uses:' line, a full 40-hex commit SHA, and a version comment (dotted forms accepted). Every Dependabot actions-group PR (#9/#29/#67) failed CI by construction because Dependabot cannot update fixtures that mirror the workflow it bumps. Full-SHA pinning stays enforced repo-wide by test_every_external_action_is_pinned_to_a_full_commit; WHICH SHA runs is governed by CODEOWNERS review of workflow diffs. Anchoring to uses: lines is a strictness increase over the old substring assertions (Codex cross-family review, DISAGREE-MINOR H, both concerns integrated; its proposed regex was corrected empirically to accept dotted version comments, which Dependabot writes). Co-Authored-By: Claude Fable 5 --- .../2026-08-04-pin-assertion-property-tests.md | 15 +++++++++++++++ tests/test_ci_security_contract.py | 11 ++++++++++- tests/test_provider_plugin_retirement.py | 17 ++++++++++++++--- 3 files changed, 39 insertions(+), 4 deletions(-) create mode 100644 changelog.d/2026-08-04-pin-assertion-property-tests.md diff --git a/changelog.d/2026-08-04-pin-assertion-property-tests.md b/changelog.d/2026-08-04-pin-assertion-property-tests.md new file mode 100644 index 0000000..51bdc8b --- /dev/null +++ b/changelog.d/2026-08-04-pin-assertion-property-tests.md @@ -0,0 +1,15 @@ +### Changed + +- CI security-contract tests now assert action-pin *properties* instead of + exact pin fixtures: the Gitleaks, CodeQL init/analyze, and upload-artifact + assertions require a real `uses:` line pinned to a full 40-hex commit SHA + with a version comment, but no longer freeze WHICH SHA or major version. + The frozen fixtures broke every Dependabot actions-group PR by construction + (#9, #29, #67) while adding no enforcement beyond the repo-wide + full-SHA pin test and CODEOWNERS review of workflow diffs — Dependabot + cannot update test fixtures that mirror the workflow it bumps. The + replacement regexes are anchored to `uses:` lines (a comment or string + mentioning an action no longer satisfies them — a strictness *increase* + over the old substring assertions, per Codex cross-family review) and + accept dotted version comments (`# v3.0.0`) as Dependabot writes them. + Test infrastructure only — no version bump. diff --git a/tests/test_ci_security_contract.py b/tests/test_ci_security_contract.py index c66fce8..2f8409e 100644 --- a/tests/test_ci_security_contract.py +++ b/tests/test_ci_security_contract.py @@ -126,9 +126,18 @@ def test_secret_scan_combines_local_and_full_history_scanners(self) -> None: "python scripts/secret_scan.py", "fetch-depth: 0", "name: Gitleaks", - "gitleaks/gitleaks-action@ff98106e4c7b2bc287b24eaf42907196329070c7", ): self.assertIn(token, text) + # Property, not fixture: the Gitleaks step must exist as a real + # `uses:` line pinned to a full commit SHA with a version comment. + # Freezing the exact SHA here only mirrored the workflow and broke + # every Dependabot bump; WHICH SHA runs is governed by the workflow + # diff review (CODEOWNERS) and the repo-wide pin test above. + self.assertRegex( + text, + r"(?m)^\s*(?:-\s*)?uses:\s*" + r"gitleaks/gitleaks-action@[0-9a-f]{40}\s+# v\d+(?:\.\d+)*\s*$", + ) def test_release_assets_use_download_stable_filenames(self) -> None: text = (WORKFLOWS / "release.yml").read_text(encoding="utf-8") diff --git a/tests/test_provider_plugin_retirement.py b/tests/test_provider_plugin_retirement.py index dfd21eb..d000c8f 100644 --- a/tests/test_provider_plugin_retirement.py +++ b/tests/test_provider_plugin_retirement.py @@ -142,11 +142,20 @@ def test_public_security_workflows_are_present(self) -> None: secret_scan = ( ROOT / ".github" / "workflows" / "secret-scan.yml" ).read_text(encoding="utf-8") + # Property, not fixture: each required action must appear as a real + # `uses:` line pinned to a full commit SHA with a version comment. + # The exact SHA/major is deliberately NOT frozen here — freezing it + # broke every Dependabot bump while adding no enforcement beyond the + # repo-wide pin test and CODEOWNERS review of workflow diffs. self.assertRegex( - codeql, r"github/codeql-action/init@[0-9a-f]{40} # v4" + codeql, + r"(?m)^\s*(?:-\s*)?uses:\s*" + r"github/codeql-action/init@[0-9a-f]{40}\s+# v\d+(?:\.\d+)*\s*$", ) self.assertRegex( - codeql, r"github/codeql-action/analyze@[0-9a-f]{40} # v4" + codeql, + r"(?m)^\s*(?:-\s*)?uses:\s*" + r"github/codeql-action/analyze@[0-9a-f]{40}\s+# v\d+(?:\.\d+)*\s*$", ) self.assertRegex(codeql, r"(?m)^ actions: read$") self.assertIn("id: analyze", codeql) @@ -156,7 +165,9 @@ def test_public_security_workflows_are_present(self) -> None: ) self.assertIn("if: github.event.repository.private", codeql) self.assertRegex( - codeql, r"actions/upload-artifact@[0-9a-f]{40} # v4" + codeql, + r"(?m)^\s*(?:-\s*)?uses:\s*" + r"actions/upload-artifact@[0-9a-f]{40}\s+# v\d+(?:\.\d+)*\s*$", ) self.assertIn("path: ${{ steps.analyze.outputs.sarif-output }}", codeql) self.assertIn("scripts/secret_scan.py", secret_scan)