From 49c8d4270a05b74ca87c5761b71056b2a3c696d0 Mon Sep 17 00:00:00 2001 From: sairenchristianbuerano Date: Mon, 12 Oct 2026 22:29:33 +0800 Subject: [PATCH] feat: keep a history of the rule count, not just the current number The badge carries one number, so it answers how many rules there are and never how fast the pack is growing. Reporting growth meant reading the badge file's commit messages, which worked but is not a source anyone should have to use. Two paths now record it, both taking the count from the validator's own output so they cannot drift: validate.yml records a point whenever rules change on main, which captures every change but only when there is one. rule-count-history.yml runs weekly and fills the quiet weeks. Without it a week with no rule changes produces no push and therefore no point, and the series cannot then distinguish no rules were added from nothing was recorded. Counting rule ids directly in the new workflow would have avoided building the engine, and is exactly how the number went stale in three places before. There is one derivation on purpose. The history is seeded from the badge file's own commits, so it starts with real data from 4 September rather than from today: 208 rules then, 333 now. record_rule_count.py upserts by day, so several pushes in a day collapse to that day's final count, and a rerun after a merge is safe. It exits non-zero on an empty or unparsed count, because a silently skipped point leaves a flat line that reads as a quiet week rather than a failure. --- .github/stats/rule-count-history.json | 43 ++++++++++ .github/workflows/rule-count-history.yml | 102 +++++++++++++++++++++++ .github/workflows/validate.yml | 21 ++++- scripts/record_rule_count.py | 54 ++++++++++++ 4 files changed, 218 insertions(+), 2 deletions(-) create mode 100644 .github/stats/rule-count-history.json create mode 100644 .github/workflows/rule-count-history.yml create mode 100644 scripts/record_rule_count.py diff --git a/.github/stats/rule-count-history.json b/.github/stats/rule-count-history.json new file mode 100644 index 0000000..4911c9a --- /dev/null +++ b/.github/stats/rule-count-history.json @@ -0,0 +1,43 @@ +{ + "repo": "trustabl/agent-reliability-rules", + "unit": "rules_in_pack", + "source": "trustabl rules validate", + "history": [ + { + "date": "2026-09-04", + "rules": 208 + }, + { + "date": "2026-09-09", + "rules": 209 + }, + { + "date": "2026-09-10", + "rules": 210 + }, + { + "date": "2026-09-15", + "rules": 217 + }, + { + "date": "2026-09-21", + "rules": 276 + }, + { + "date": "2026-09-22", + "rules": 277 + }, + { + "date": "2026-09-29", + "rules": 298 + }, + { + "date": "2026-10-05", + "rules": 330 + }, + { + "date": "2026-10-12", + "rules": 333 + } + ] +} diff --git a/.github/workflows/rule-count-history.yml b/.github/workflows/rule-count-history.yml new file mode 100644 index 0000000..7496dd9 --- /dev/null +++ b/.github/workflows/rule-count-history.yml @@ -0,0 +1,102 @@ +name: Rule count history + +# Weekly snapshot of how many rules the pack contains. +# +# Why this exists alongside the hook in validate.yml. That hook records a point +# whenever rules change on main, which captures every change but only when there +# is one. A quiet week produces no push, so no point, and the series then cannot +# distinguish "no rules were added" from "nothing was recorded". This fills the +# quiet weeks so a flat line means flat rather than missing. +# +# Both paths call the same script and take the count from the same place, the +# validator's own output, so the two cannot drift apart. Counting rule ids +# directly here would be cheaper than building the engine, and is exactly how +# the number went stale in three places before: there must be one derivation. +# +# Same-day reruns overwrite that day's point rather than appending, so running +# this by hand after a merge is safe. + +on: + schedule: + - cron: '0 6 * * 1' # weekly, Mondays 06:00 UTC, an hour after the download snapshot + workflow_dispatch: # run on demand to add a point now + +permissions: + contents: write # push the branch the PR is built from + pull-requests: write # open or update the snapshot PR + +jobs: + snapshot: + runs-on: ubuntu-latest + steps: + - name: Checkout rules + uses: actions/checkout@v4 + with: + path: rules + + # The engine is checked out and built rather than downloaded so the count + # comes from the same validator validate.yml uses. + - name: Resolve engine ref + id: engineref + run: echo "ref=main" >> "$GITHUB_OUTPUT" + + - name: Checkout engine + uses: actions/checkout@v4 + with: + repository: trustabl/agent-reliability-analyzer + ref: ${{ steps.engineref.outputs.ref }} + path: engine + + - uses: actions/setup-go@v5 + with: + go-version-file: engine/go.mod + cache: true + cache-dependency-path: engine/go.sum + + - name: Build the validator + working-directory: engine + run: go build -o "$RUNNER_TEMP/trustabl" ./cmd/trustabl + + - name: Count the rules + id: validate + run: | + out="$("$RUNNER_TEMP/trustabl" rules validate ./rules)" + echo "$out" + count=$(printf '%s' "$out" | sed -n 's/.*, \([0-9][0-9]*\) rule(s) valid.*/\1/p') + echo "count=$count" >> "$GITHUB_OUTPUT" + + # The script exits non-zero on an empty or unparsed count, so a change in + # the validator's output shape fails the run instead of silently writing + # nothing and leaving a gap that reads as a quiet week. + - name: Record the point + working-directory: rules + run: python3 scripts/record_rule_count.py '${{ steps.validate.outputs.count }}' + + # main is protection-gated, so this lands as a PR. create-pull-request + # reuses one rolling branch and no-ops when the file is unchanged, so a + # week with no new rules opens nothing. + - name: Open or update the snapshot PR + uses: peter-evans/create-pull-request@v7 + with: + token: ${{ secrets.GITHUB_TOKEN }} + path: rules + add-paths: .github/stats/rule-count-history.json + branch: chore/rule-count-history + base: main + commit-message: "chore: record weekly rule-count snapshot" + title: "chore: weekly rule-count snapshot" + body: | + Automated weekly snapshot of how many rules the pack contains, + reported by `trustabl rules validate` on this commit: + **${{ steps.validate.outputs.count }}** rules. + + Updates `.github/stats/rule-count-history.json`. One point per day, + upserted, so this never duplicates a day already recorded. Safe to + merge. + + The badge in `badges/rules.json` is refreshed separately by + `validate.yml` when rules actually change; this workflow only keeps + the history continuous through weeks where nothing changed. + author: "github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>" + committer: "github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>" + delete-branch: false diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 0846a87..c104ee1 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -93,13 +93,27 @@ jobs: printf '{"schemaVersion":1,"label":"rules","message":"%s","color":"brightgreen"}\n' "$n" \ > badges/rules.json + # The badge holds only the current number, so it answers how many rules + # there are but never how fast the pack is growing. This keeps the series + # alongside it, one dated point per day. + # + # Recorded here rather than in a weekly cron because the count already + # exists in this job; a separate schedule would rebuild the engine purely + # to recompute a number this workflow is already holding. + - name: Record the rule count in the history + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + working-directory: rules + run: python3 scripts/record_rule_count.py '${{ steps.validate.outputs.count }}' + - name: Open or update the badge PR if: github.event_name == 'push' && github.ref == 'refs/heads/main' uses: peter-evans/create-pull-request@v7 with: token: ${{ secrets.GITHUB_TOKEN }} path: rules - add-paths: badges/rules.json + add-paths: | + badges/rules.json + .github/stats/rule-count-history.json branch: chore/rule-count-badge base: main commit-message: "chore(badges): rule count -> ${{ steps.validate.outputs.count }}" @@ -109,7 +123,10 @@ jobs: **${{ steps.validate.outputs.count }}** rules on this commit. Updates `badges/rules.json`, the shields endpoint the README badge - reads. No-ops when the count is unchanged. Safe to merge. + reads, and appends today's point to + `.github/stats/rule-count-history.json` so growth over time is kept + rather than only the current number. No-ops when both are unchanged. + Safe to merge. author: "github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>" committer: "github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>" delete-branch: false diff --git a/scripts/record_rule_count.py b/scripts/record_rule_count.py new file mode 100644 index 0000000..2cc6a19 --- /dev/null +++ b/scripts/record_rule_count.py @@ -0,0 +1,54 @@ +"""Record one dated point of the pack's rule count. + + python3 scripts/record_rule_count.py + +The badge carries only the current number, so it answers "how many rules are +there" but never "how fast is the pack growing". This keeps the series. + +One point per day, upserted, so several pushes in a day collapse to that day's +final count rather than nine entries. The count is passed in rather than +recomputed here: it comes from `trustabl rules validate`, which the validate +workflow has already run, and deriving it a second way would let the two +disagree. +""" + +import datetime +import json +import pathlib +import sys + +HISTORY = pathlib.Path(".github/stats/rule-count-history.json") + +EMPTY = { + "repo": "trustabl/agent-reliability-rules", + "unit": "rules_in_pack", + "source": "trustabl rules validate", + "history": [], +} + + +def main(): + if len(sys.argv) < 2: + sys.exit("usage: record_rule_count.py ") + raw = sys.argv[1].strip() + if not raw.isdigit(): + # An empty or unparsed count means the validator's output changed shape. + # Fail loudly: a silently skipped point leaves a flat line that reads as + # "no rules were added" rather than "nothing was recorded". + sys.exit(f"rule count {raw!r} is not a number; the validator output may have changed") + count = int(raw) + + doc = json.loads(HISTORY.read_text(encoding="utf-8")) if HISTORY.exists() else dict(EMPTY) + + day = datetime.date.today().isoformat() # UTC on the runner + points = {p["date"]: p for p in doc.get("history", [])} + points[day] = {"date": day, "rules": count} + doc["history"] = [points[d] for d in sorted(points)] + + HISTORY.parent.mkdir(parents=True, exist_ok=True) + HISTORY.write_text(json.dumps(doc, indent=2) + "\n", encoding="utf-8") + print(f"{day}: {count} rules ({len(doc['history'])} points)") + + +if __name__ == "__main__": + main()