diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 063a44a..13c1117 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,6 +26,31 @@ jobs: echo "::error::pack exceeds 20 MiB — a large binary was committed" exit 1; } + family-conformance: + needs: hygiene + # Executes this repo's documented exit contract against its real CLI and + # fails if SKILL.md's claims and the tool's behaviour disagree. The SKILL.md + # acceptance test checks that an agent can *invoke* the tool; this checks + # that what the file promises is *true*. Every family member's SKILL.md + # passed the former while failing the latter. + # + # Blocking, not advisory: known findings are waived in + # family-conformance-baseline.json, each naming the issue that tracks it, so + # this fails only on a NEW divergence. A waiver that stops firing also fails, + # so fixes shrink the baseline rather than leaving stale permission behind. + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + - name: Install so the gate probes the real console script + run: pip install -e . + - name: SKILL.md claims match the CLI + run: | + python scripts/family_conformance.py . \ + --baseline family-conformance-baseline.json + test: needs: hygiene # Ubuntu carries the full correctness/provenance suite on every supported diff --git a/family-conformance-baseline.json b/family-conformance-baseline.json new file mode 100644 index 0000000..8b07fdc --- /dev/null +++ b/family-conformance-baseline.json @@ -0,0 +1,16 @@ +{ + "_comment": "Findings accepted for now. Each MUST name the issue tracking it — a waiver without a ticket is just a hidden defect. A waiver that stops firing fails the gate, so fixes shrink this file.", + "accepted": { + "charactercheck": { + "HONEST_LANE_OVERLOAD": "chaoz23/charactercheck#18" + }, + "dmcheck": { + "MISSING_PIPE": "chaoz23/dmcheck#14", + "HONEST_LANE_OVERLOAD": "chaoz23/dmcheck#15" + }, + "tablekit": { + "SCHEMA_NOT_FLAG": "chaoz23/table-kit#23", + "HONEST_LANE_OVERLOAD": "chaoz23/table-kit#1" + } + } +} diff --git a/scripts/family_conformance.py b/scripts/family_conformance.py index 1caaa41..9e1e500 100755 --- a/scripts/family_conformance.py +++ b/scripts/family_conformance.py @@ -410,6 +410,10 @@ def main(argv: list[str] | None = None) -> int: help="read repo paths from stdin, one per line; one JSON result per line") ap.add_argument("--schema", action="store_true", help="print the I/O contract and exit 0") ap.add_argument("--json", action="store_true", help="machine-readable output") + ap.add_argument("--baseline", type=Path, metavar="PATH", + help="accept the findings listed in PATH; fail only on new ones. " + "A baseline entry that no longer fires is itself a failure, " + "so fixes must shrink the file.") args = ap.parse_args(argv) if args.schema: @@ -420,14 +424,37 @@ def main(argv: list[str] | None = None) -> int: if not targets: ap.error("no repos given (pass paths, or --pipe with paths on stdin)") + accepted = {} + if args.baseline: + if not args.baseline.is_file(): + die(f"{args.baseline}: baseline file not found") + accepted = json.loads(args.baseline.read_text()).get("accepted", {}) + results, worst = [], 0 + stale = [] for repo in targets: r = audit(repo) + allowed = accepted.get(r["tool"], {}) + fired = {f["rule"] for f in r["findings"]} + for f in r["findings"]: + f["accepted"] = f["rule"] in allowed + if f["accepted"]: + f["tracked_by"] = allowed[f["rule"]] + # A baseline entry that stopped firing must be removed, or the ratchet + # only ever loosens. Fixing a defect and leaving its waiver behind means + # the next regression is silently pre-accepted. + for rule, issue in allowed.items(): + if rule not in fired: + stale.append((r["tool"], rule, issue)) + r["new_findings"] = [f for f in r["findings"] if not f["accepted"]] results.append(r) - worst = max(worst, 1 if r["findings"] else 0) + worst = max(worst, 1 if r["new_findings"] else 0) if args.pipe: print(json.dumps(r), flush=True) + if stale: + worst = 1 + if args.json: print(json.dumps({"results": results}, indent=2)) elif not args.pipe: @@ -438,11 +465,17 @@ def main(argv: list[str] | None = None) -> int: f"documented {r['documented_codes']} · observed {r['observed_codes']}" f" · honest-lane {r['honest_lane_codes']}") for f in r["findings"]: - print(f" [{f['severity']}] {f['rule']}") + tag = f" (accepted · {f['tracked_by']})" if f.get("accepted") else "" + print(f" [{f['severity']}] {f['rule']}{tag}") print(f" {f['message']}") print(f" evidence: {f['evidence']}") total = sum(len(r["findings"]) for r in results) - print(f"\n{total} finding(s) across {len(results)} repo(s)") + fresh = sum(len(r.get("new_findings", r["findings"])) for r in results) + print(f"\n{total} finding(s) across {len(results)} repo(s); {fresh} not in the baseline") + for tool, rule, issue in stale: + print(f"\nSTALE BASELINE: {tool}/{rule} no longer fires but is still " + f"waived (tracked by {issue}).\n Remove it from the baseline — " + f"a waiver left behind pre-accepts the next regression.") return worst