From ba2605302424327eb8bc936952a74fc5af2c48e5 Mon Sep 17 00:00:00 2001 From: chaoz23 Date: Thu, 20 Aug 2026 12:51:55 -0700 Subject: [PATCH] ci: run the conformance gate on every PR, ratcheted by a baseline Closes the remaining half of #81. The gate landed in #82 but nothing ran it, so it could only catch defects for whoever remembered to invoke it by hand. Blocking rather than advisory. An advisory job that is permanently red is noise people learn to scroll past, and the gate currently reports real findings in three of four members. So known findings are waived in family-conformance-baseline.json and the job fails only on a NEW divergence. Two rules make the waiver list a ratchet instead of a place defects go to die: - every waiver must name the issue tracking it. A waiver without a ticket is just a hidden defect with extra steps. - a waiver that no longer fires FAILS the gate. Fixing a defect and leaving its waiver behind would silently pre-accept the next regression, so fixes have to shrink the file. Verified in all three directions, because a gate that cannot fail is worthless: known findings waived -> exit 0 a waived finding un-waived -> exit 1, reported as not in the baseline a waiver that no longer fires -> exit 1, reported as STALE BASELINE srdcheck itself passes with no waivers. The baseline's sibling entries are consulted only when those repos are audited, so this job reports nothing for them; the sibling CI jobs come next and will read the same canonical file rather than copying it, per FAMILY.md's pin-by-link rule. Co-Authored-By: Claude Opus 5 --- .github/workflows/ci.yml | 25 ++++++++++++++++++++ family-conformance-baseline.json | 16 +++++++++++++ scripts/family_conformance.py | 39 +++++++++++++++++++++++++++++--- 3 files changed, 77 insertions(+), 3 deletions(-) create mode 100644 family-conformance-baseline.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 063a44a..13c1117 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,6 +26,31 @@ jobs: echo "::error::pack exceeds 20 MiB — a large binary was committed" exit 1; } + family-conformance: + needs: hygiene + # Executes this repo's documented exit contract against its real CLI and + # fails if SKILL.md's claims and the tool's behaviour disagree. The SKILL.md + # acceptance test checks that an agent can *invoke* the tool; this checks + # that what the file promises is *true*. Every family member's SKILL.md + # passed the former while failing the latter. + # + # Blocking, not advisory: known findings are waived in + # family-conformance-baseline.json, each naming the issue that tracks it, so + # this fails only on a NEW divergence. A waiver that stops firing also fails, + # so fixes shrink the baseline rather than leaving stale permission behind. + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + - name: Install so the gate probes the real console script + run: pip install -e . + - name: SKILL.md claims match the CLI + run: | + python scripts/family_conformance.py . \ + --baseline family-conformance-baseline.json + test: needs: hygiene # Ubuntu carries the full correctness/provenance suite on every supported diff --git a/family-conformance-baseline.json b/family-conformance-baseline.json new file mode 100644 index 0000000..8b07fdc --- /dev/null +++ b/family-conformance-baseline.json @@ -0,0 +1,16 @@ +{ + "_comment": "Findings accepted for now. Each MUST name the issue tracking it — a waiver without a ticket is just a hidden defect. A waiver that stops firing fails the gate, so fixes shrink this file.", + "accepted": { + "charactercheck": { + "HONEST_LANE_OVERLOAD": "chaoz23/charactercheck#18" + }, + "dmcheck": { + "MISSING_PIPE": "chaoz23/dmcheck#14", + "HONEST_LANE_OVERLOAD": "chaoz23/dmcheck#15" + }, + "tablekit": { + "SCHEMA_NOT_FLAG": "chaoz23/table-kit#23", + "HONEST_LANE_OVERLOAD": "chaoz23/table-kit#1" + } + } +} diff --git a/scripts/family_conformance.py b/scripts/family_conformance.py index 1caaa41..9e1e500 100755 --- a/scripts/family_conformance.py +++ b/scripts/family_conformance.py @@ -410,6 +410,10 @@ def main(argv: list[str] | None = None) -> int: help="read repo paths from stdin, one per line; one JSON result per line") ap.add_argument("--schema", action="store_true", help="print the I/O contract and exit 0") ap.add_argument("--json", action="store_true", help="machine-readable output") + ap.add_argument("--baseline", type=Path, metavar="PATH", + help="accept the findings listed in PATH; fail only on new ones. " + "A baseline entry that no longer fires is itself a failure, " + "so fixes must shrink the file.") args = ap.parse_args(argv) if args.schema: @@ -420,14 +424,37 @@ def main(argv: list[str] | None = None) -> int: if not targets: ap.error("no repos given (pass paths, or --pipe with paths on stdin)") + accepted = {} + if args.baseline: + if not args.baseline.is_file(): + die(f"{args.baseline}: baseline file not found") + accepted = json.loads(args.baseline.read_text()).get("accepted", {}) + results, worst = [], 0 + stale = [] for repo in targets: r = audit(repo) + allowed = accepted.get(r["tool"], {}) + fired = {f["rule"] for f in r["findings"]} + for f in r["findings"]: + f["accepted"] = f["rule"] in allowed + if f["accepted"]: + f["tracked_by"] = allowed[f["rule"]] + # A baseline entry that stopped firing must be removed, or the ratchet + # only ever loosens. Fixing a defect and leaving its waiver behind means + # the next regression is silently pre-accepted. + for rule, issue in allowed.items(): + if rule not in fired: + stale.append((r["tool"], rule, issue)) + r["new_findings"] = [f for f in r["findings"] if not f["accepted"]] results.append(r) - worst = max(worst, 1 if r["findings"] else 0) + worst = max(worst, 1 if r["new_findings"] else 0) if args.pipe: print(json.dumps(r), flush=True) + if stale: + worst = 1 + if args.json: print(json.dumps({"results": results}, indent=2)) elif not args.pipe: @@ -438,11 +465,17 @@ def main(argv: list[str] | None = None) -> int: f"documented {r['documented_codes']} · observed {r['observed_codes']}" f" · honest-lane {r['honest_lane_codes']}") for f in r["findings"]: - print(f" [{f['severity']}] {f['rule']}") + tag = f" (accepted · {f['tracked_by']})" if f.get("accepted") else "" + print(f" [{f['severity']}] {f['rule']}{tag}") print(f" {f['message']}") print(f" evidence: {f['evidence']}") total = sum(len(r["findings"]) for r in results) - print(f"\n{total} finding(s) across {len(results)} repo(s)") + fresh = sum(len(r.get("new_findings", r["findings"])) for r in results) + print(f"\n{total} finding(s) across {len(results)} repo(s); {fresh} not in the baseline") + for tool, rule, issue in stale: + print(f"\nSTALE BASELINE: {tool}/{rule} no longer fires but is still " + f"waived (tracked by {issue}).\n Remove it from the baseline — " + f"a waiver left behind pre-accepts the next regression.") return worst