diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index d487afa..70293a1 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -9,19 +9,19 @@ { "name": "qe", "source": "./qe", - "version": "0.2.0", + "version": "0.2.1", "description": "QuantEcon's author-facing base skills — style checking, lecture editing support, and working through a PR's review feedback" }, { "name": "benchmark", "source": "./benchmark", - "version": "0.3.0", + "version": "0.3.1", "description": "Benchmarking and acceleration-evaluation tools for QuantEcon lecture code" }, { "name": "audit", "source": "./audit", - "version": "0.1.2", + "version": "0.1.3", "description": "Bulk, read-only audits of a QuantEcon repository — issue triage, PR review, technical debt, translation parity — each producing an evidence-cited report bundle" } ] diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 16e4c59..b4495d8 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -10,12 +10,41 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + # The version-bump guard diffs the PR against its merge base and reads + # plugin.json as of that commit and as of the base tip. The default + # depth-1, single-ref checkout has neither the base ref nor shared + # history, and the guard exits 2 rather than passing blind — so full + # history is load-bearing for that step. It applies to the whole job + # rather than just the step that needs it; harmless for the others, + # since neither reads git history. + fetch-depth: 0 - uses: actions/setup-python@v5 with: python-version: "3.12" - name: Validate manifests and skill frontmatter run: python scripts/validate.py + # A plugin's install cache is keyed by version string, and + # `claude plugin update` compares version strings only — so shipping an + # edited SKILL.md under an unchanged version publishes nothing: every + # consuming repo is told "already at the latest version" and keeps the old + # files. The bump is the delivery mechanism, which makes a missing one a + # build failure rather than a review nit. Three of this repo's first eleven + # merges would have been caught by it. + # + # Only a PR into the default branch publishes anything, so only those are + # checked: a stacked PR targeting another feature branch would otherwise be + # told to mint a second version for content that ships once. It gets + # checked the moment it is retargeted to main. + - name: Plugin changes carry a version bump + if: >- + github.event_name == 'pull_request' && + github.base_ref == github.event.repository.default_branch + env: + BASE_REF: ${{ github.base_ref }} + run: python scripts/check-version-bump.py --base "origin/$BASE_REF" + # The benchmark plugin's claim is that no score is ever written by hand: # every scorecard is a deterministic function of its evidence.json. That # only stays true if it is checked. A non-empty diff here means either a @@ -31,3 +60,44 @@ jobs: python scripts/scoring/score.py references/fixtures/rubric_v2 git diff --exit-code -- 'references/examples/*/results/scorecard.json' \ 'references/fixtures/*/results/scorecard.json' + + # A separate job on purpose: this one installs an ~85 MB npm toolchain, and + # running it beside `validate` rather than inside it keeps that job four fast + # stdlib steps. The two also report independently, so a CLI-install failure + # cannot mask a manifest error. + # + # It earns its place by parsing what scripts/validate.py only pattern-matches. + # The repo's validator reads frontmatter with a regex, so it cannot see invalid + # YAML: `description: Audit every issue … Read-only: it recommends …` is a + # parse error — a `: ` inside an unquoted plain scalar — that shipped in + # audit/skills/issues/SKILL.md and passed CI until this PR. `--strict` also + # flags unknown plugin.json keys with a did-you-mean, so a typo'd `versoin` is + # caught rather than silently ignored. + # + # The CLI is pinned. Under --strict a new upstream warning becomes an error, so + # an unpinned install would let an upstream release redden an unrelated PR. + strict-validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: "22" + - name: Install Claude Code CLI + run: npm install -g @anthropic-ai/claude-code@2.1.220 + + # Needs no credentials — a local manifest and frontmatter check that runs + # with no API key, no login and no network. Four targets because the scopes + # differ: only a plugin directory walks skills/, and only the marketplace + # root can compare an entry's version against the plugin.json it points at. + - name: Validate against the runtime's own parser + env: + DISABLE_AUTOUPDATER: "1" + run: | + rc=0 + for target in qe benchmark audit .; do + echo "::group::claude plugin validate --strict $target" + claude plugin validate --strict "$target" || rc=1 + echo "::endgroup::" + done + exit $rc diff --git a/AGENTS.md b/AGENTS.md index 119880a..d922e2b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -33,7 +33,8 @@ Before adding a paragraph, check whether it already exists. If it does, link to |---|---| | What the marketplace is, installation (local, lecture repos, CI) | [README.md](README.md) | | Using the skills: setup, invocation, what to expect | [docs/using-skills.md](docs/using-skills.md) | -| Contributing: layout, conventions, dev loop, local testing, versioning, PR flow | [docs/developing-skills.md](docs/developing-skills.md) | +| Contributing: layout, conventions, dev loop, local testing, versioning, releases and tagging, PR flow | [docs/developing-skills.md](docs/developing-skills.md) | +| What changed in a plugin, release by release | `/CHANGELOG.md` — one per plugin, since the plugin is the released artifact and ships only its own directory | | Running an evaluation by hand, end to end | [docs/tutorial-run-an-evaluation.md](docs/tutorial-run-an-evaluation.md) | | Running a whole-tracker audit, and reviewing what it produces | [docs/tutorial-run-an-audit.md](docs/tutorial-run-an-audit.md) | | The benchmark skill: modes, report format, manual pipeline | [benchmark/README.md](benchmark/README.md) | @@ -46,6 +47,7 @@ Before adding a paragraph, check whether it already exists. If it does, link to ## Working in this repo - **Validate before committing**: `python scripts/validate.py`. A malformed manifest breaks installation silently in every consuming repo, so CI runs the same check. +- **Changing a file under `/` means a version bump and a changelog entry, in the same PR.** The install cache is keyed by version string, so content merged without a bump reaches nobody who already has the plugin installed. CI enforces it; see [developing-skills § Versioning and releases](docs/developing-skills.md#versioning-and-releases). - **Test from a real consuming project**, not from inside this repo — path-resolution bugs only surface when a plugin runs from an install location. Both tiers are in [developing-skills § Testing locally](docs/developing-skills.md#testing-locally). - **The product principles** — report first, fix on request; deterministic before LLM; cited claims and computed scores; scaffolding as advice rather than instruction — are stated once in [CATALOG.md § Principles](CATALOG.md#principles) and elaborated in [developing-skills § Conventions](docs/developing-skills.md#conventions). Follow them; don't restate them in new files. - **A new skill starts as an issue, not a doc entry.** CATALOG.md lists what has merged *and* is operational, so it stays true; the plan for something unbuilt — and the scaffolding for something merged but not yet operational — belongs in its plugin's tracking issue, where it can change without anyone mistaking it for a description of the repo. diff --git a/README.md b/README.md index 020d668..bf4bf16 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ Each plugin bundles one area of work — a skill (the instructions Claude follow |---|---| | [AGENTS.md](AGENTS.md) | AI agents and contributors: canonical repo instructions — the single-source-of-truth principle, doc map, working conventions | | [docs/using-skills.md](docs/using-skills.md) | Authors/reviewers: setup, invoking skills, what to expect | -| [docs/developing-skills.md](docs/developing-skills.md) | Contributors: layout, conventions, dev loop, testing locally, versioning, PR flow | +| [docs/developing-skills.md](docs/developing-skills.md) | Contributors: layout, conventions, dev loop, testing locally, versioning and releases, PR flow | | [docs/tutorial-run-an-evaluation.md](docs/tutorial-run-an-evaluation.md) | Tutorial: the evaluation procedure by hand, with the ge_arrow validation run as the checkable example | | [benchmark/README.md](benchmark/README.md) | The evaluation skill: review mode, triage mode, report format, manual pipeline | | [audit/README.md](audit/README.md) | The audit family: what belongs in it, the shared method, running one | @@ -103,4 +103,6 @@ Open a PR adding or modifying a plugin directory and registering it in `.claude- Run `python scripts/validate.py` before pushing — it checks that every plugin resolves, that `plugin.json` agrees with `marketplace.json` on name, version, and description, and that each `SKILL.md` has frontmatter whose `name` matches its directory. CI runs the same script on every PR; a malformed manifest otherwise breaks installation silently in every consuming lecture repository. +CI also fails a PR that changes files under a plugin directory without bumping that plugin's version and adding its changelog entry — the install cache is keyed by version string, so an unbumped change reaches nobody ([developing-skills § Versioning and releases](docs/developing-skills.md#versioning-and-releases)). + Broader context for this repository: [QuantEcon/meta#304](https://github.com/QuantEcon/meta/issues/304) (toolkit proposal) and [QuantEcon/meta#335](https://github.com/QuantEcon/meta/issues/335) (benchmarking programme). diff --git a/audit/.claude-plugin/plugin.json b/audit/.claude-plugin/plugin.json index 6b3eaba..0cb6d84 100644 --- a/audit/.claude-plugin/plugin.json +++ b/audit/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "audit", "description": "Bulk, read-only audits of a QuantEcon repository — issue triage, PR review, technical debt, translation parity — each producing an evidence-cited report bundle", - "version": "0.1.2", + "version": "0.1.3", "author": { "name": "QuantEcon" } } diff --git a/audit/CHANGELOG.md b/audit/CHANGELOG.md new file mode 100644 index 0000000..5b9fb3d --- /dev/null +++ b/audit/CHANGELOG.md @@ -0,0 +1,70 @@ +# Changelog — `audit` + +Every released version of this plugin, newest first. A version exists because the content below shipped in it: the plugin cache is keyed by version string, so what you have installed is exactly the entries down to the version `claude plugin list` reports for `audit`. + +Versions are [semver](https://semver.org) as a user of this plugin experiences it — a new skill, or a procedure that now does something materially different, is a minor bump; a correction that leaves the procedure as it was is a patch. Nothing below 1.0.0 promises stability. + +Repository: [QuantEcon/skills](https://github.com/QuantEcon/skills) ([every commit that touched this plugin](https://github.com/QuantEcon/skills/commits/main/audit)). How a release is made: [developing-skills § Versioning and releases](https://github.com/QuantEcon/skills/blob/main/docs/developing-skills.md#versioning-and-releases). + +## 0.1.3 — 2026-08-03 + +**Added** + +- This changelog. + +**Fixed** + +- `/audit:issues`'s frontmatter `description` was an unquoted YAML plain scalar containing `Read-only: it recommends…`. A `: ` inside a plain scalar is a parse error, so a strict loader drops the skill's metadata rather than reading it, and `claude plugin validate` rejects the file outright. The value is now quoted. Nothing about the procedure changed. + +## 0.1.2 — 2026-07-28 + +Resolves the contradiction that told an audit to write its bundle into the repo it promised not to touch: the boundary is now mutation, not writing, and the skill says exactly where to put its working directory so a run leaves `git status` clean. + +**Added** + +- Discovery-ordered working-directory selection, taken from contact with a real repo: prefer a location the repo already ignores (`.dev/scratch/audit-/` in QuantEcon repos, where `.dev/scratch/*` is already gitignored), fall back to an untracked `.audit/-/` at the checkout root, then to somewhere outside the checkout entirely. Which one was used goes in the report's method section. +- Doctrine §3 now says explicitly that a run may write its own working directory, including inside the audited checkout — provided the directory stays untracked and nothing is added to `.gitignore`, since that would itself be an edit to a tracked file. + +**Changed** + +- Doctrine §3 narrowed from "no branch or file changes in the audited repo" to what it was always protecting — content and history: no commits, no pushes, no branches, no edits to tracked files. Mutation, not writing, is the boundary. +- `deliverables.md` states the split: writing the bundle is the audit's job, committing or publishing it is a human step taken after reading it. + +**Fixed** + +- The read-only/working-directory contradiction the plugin carried since 0.1.0 — §3 forbade file changes in the audited repo while `deliverables.md` made that repo's own notes system the bundle's first-choice destination, and 0.1.1's default `--out` wrote there too. A run following the docs literally could not satisfy both. + +## 0.1.1 — 2026-07-28 + +An interrupted run can actually be resumed: the intermediate artifacts now have names and locations, phase 2 appends per item instead of writing at the end, and the bundle shrinks to fit a small tracker. + +**Added** + +- A stated working-directory layout under `--out` (`.audit/-/` by convention): `snapshot/` from phase 1, `findings.md` from phase 2, `links.md` from phase 3, and the delivered `01-…`/`02-…`/`03-…`/`README.md` bundle from phase 4. Previously phases 2 and 3 produced "per-item findings" and "the cluster map" with no filename and no location, so resuming worked only if two sessions independently invented the same file. +- A stated resume rule: on restart, read `findings.md` and resume at the lowest number in `issues.json` with no entry, re-verifying the last entry rather than trusting a possibly truncated write. +- `meta.json` records `fetched_by`, the account the snapshot was taken as — which matters because visibility on the org's private repos is per-account. + +**Changed** + +- Phase 2 appends each item's finding to `findings.md` as it is verified, in the catalog entry format, so phase 4 assembles the catalog instead of re-deriving it. +- The bundle scales to the tracker: below roughly 30 open issues, fold the catalog and the link graph into the report, keep the `README.md` index, and say which shape was used in the coverage statement. Four unconditional documents forced three files of padding on a small tracker, and padding makes a report less checkable. +- Doctrine §4 now states the general rule: a checkpoint owes a findable name and incremental writes, or it is a claim about resumability rather than the property itself. + +**Fixed** + +- `meta["authenticated"]` is removed, not deprecated. It could only ever be `true` (preflight exits on every unauthenticated path), so it was a provenance field carrying no evidence — in the plugin whose doctrine is that every claim carries its evidence class. **Anything reading that field must switch to `fetched_by`.** +- An interrupted phase 2 now loses one item rather than the whole phase — it was the phase specified to write on completion, and the phase a hundred-item run dies inside rather than between. + +## 0.1.0 — 2026-07-27 + +First release. `/audit:issues` sweeps an entire GitHub tracker — open and closed — verifies each item against the code rather than the thread, tiers the open set into the repo's existing plan, and delivers an evidence-cited report bundle, without ever touching the tracker. + +- `/audit:issues ` — a whole-tracker audit in five phases (snapshot, per-item verification, cross-link graph, tiered report, coverage self-audit). The four runbook fields (plan anchor, tier scheme, repo type, notes system) are optional arguments with documented discovery, so the usual invocation is just the repo. +- A deterministic snapshot step, `scripts/fetch_tracker.py OWNER/REPO --out `: every issue and PR in any state with full comment threads (and PR reviews, and `closingIssuesReferences`) in two `gh` round trips, written as `meta.json`, `issues.json`, `prs.json`, `coverage.json`. Closed threads cost nothing extra to read, and the snapshot freezes the audit's point in time so "events after the snapshot" is a stated property of the report instead of an unnoticed gap. +- `coverage.json` reconciliation: captured items against the number sequence `1..max`, discussion counts split open/closed, and an explicit truncation flag when a stream returns exactly at `--limit` (default 1000) — a case indistinguishable from truncation, so it is surfaced rather than swallowed. PR review bodies count toward captured discussion, not just comments: on the example repo, closed PRs carried 374 reviews against 28 comments. +- Snapshot files are written in issue/PR number order, so two runs over an unchanged tracker are byte-identical and a re-fetch diffs down to what actually changed. +- Thread payloads are shape-asserted at capture, so a `gh` build returning counts instead of lists fails by name at the point of capture rather than crashing later or silently under-reporting threads while the report still claims thread-completeness. +- Preflight that refuses to start without `gh` and an authenticated account, because the anonymous API is 60 req/h per IP and returns nothing at all for the org's private repos. +- Plugin-level method shared by every future audit skill: `references/doctrine.md` (trust rules, evidence classes `[verified]`/`[stated]`/`[inferred]`, the read-only boundary, checkpointing, the coverage self-audit), `references/quantecon-context.md` (repo types, label ownership, the cross-repo graph, access, and the caveat that an HTML-reconstructed thread may start mid-conversation), and `references/deliverables.md` (what an audit owes its reader, and where a bundle may land). +- QuantEcon-specific triage judgement: tier by repo type (a build break in a lecture repo and a consumer-visible change in an action repo outrank thread activity), check sibling repos before concluding, leave label application to `qe`, and keep GitHub closing keywords out of drafted cross-repo references so drafted text cannot close an upstream item when someone posts it. +- The four-document bundle, the five phases and "produces a bundle" are stated as a worked example rather than a requirement, after a single execution. What an audit owes its reader — coverage statement, evidence tag per claim, recommendations marked as proposals, drafted comments marked unsent, a date and a named snapshot — stays mandatory and presumes no file count. diff --git a/audit/skills/issues/SKILL.md b/audit/skills/issues/SKILL.md index f2a276b..435201b 100644 --- a/audit/skills/issues/SKILL.md +++ b/audit/skills/issues/SKILL.md @@ -1,6 +1,6 @@ --- name: issues -description: Audit every issue in a GitHub repository, open and closed — verify each status against the code rather than the thread, hunt fixed-but-open and never-landed-fix candidates, tier the open set into the repo's existing plan, and deliver a report bundle with a cross-link map. Read-only: it recommends tracker changes but never makes them. Use for a whole-tracker review, not a single issue. +description: "Audit every issue in a GitHub repository, open and closed — verify each status against the code rather than the thread, hunt fixed-but-open and never-landed-fix candidates, tier the open set into the repo's existing plan, and deliver a report bundle with a cross-link map. Read-only: it recommends tracker changes but never makes them. Use for a whole-tracker review, not a single issue." --- # audit:issues diff --git a/benchmark/.claude-plugin/plugin.json b/benchmark/.claude-plugin/plugin.json index 7691505..fa8be63 100644 --- a/benchmark/.claude-plugin/plugin.json +++ b/benchmark/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "benchmark", "description": "Benchmarking and acceleration-evaluation tools for QuantEcon lecture code", - "version": "0.3.0", + "version": "0.3.1", "author": { "name": "QuantEcon" } } diff --git a/benchmark/CHANGELOG.md b/benchmark/CHANGELOG.md new file mode 100644 index 0000000..c0e73b7 --- /dev/null +++ b/benchmark/CHANGELOG.md @@ -0,0 +1,73 @@ +# Changelog — `benchmark` + +Every released version of this plugin, newest first. A version exists because the content below shipped in it: the plugin cache is keyed by version string, so what you have installed is exactly the entries down to the version `claude plugin list` reports for `benchmark`. + +Versions are [semver](https://semver.org) as a user of this plugin experiences it — a new skill, or a procedure that now does something materially different, is a minor bump; a correction that leaves the procedure as it was is a patch. Nothing below 1.0.0 promises stability. + +Repository: [QuantEcon/skills](https://github.com/QuantEcon/skills) ([every commit that touched this plugin](https://github.com/QuantEcon/skills/commits/main/benchmark)). How a release is made: [developing-skills § Versioning and releases](https://github.com/QuantEcon/skills/blob/main/docs/developing-skills.md#versioning-and-releases). + +## 0.3.1 — 2026-08-03 + +**Added** + +- This changelog. + +**Fixed** + +- The plugin README's status line named `v0.2.0` — a version that was never released (see the note at the foot of this file). It now names `v0.3.0`, the release in which the evaluation system actually became runnable. That is a historical fact rather than a restatement of the current version, so it will not go stale again on the next bump. + +## 0.3.0 — 2026-07-27 + +The evaluation system became runnable: a deterministic scoring engine, rubric v2 with verdict gates, two complete worked evaluations to copy from, a triage mode, and the install fix that made the plugin installable at all. + +**Added** + +- A runnable scoring engine: `python scripts/scoring/score.py ` turns an evidence file into a scorecard. No score is ever typed by hand; the session shows the derivation table — every dimension score with the measured number and threshold band that produced it. +- `references/EVALUATION_FRAMEWORK.md` — the rubric in prose: seven weighted dimensions, numeric scoring anchors, structural checklists, verdict bands, worked HIGH/LOW examples. `SKILL.md` points here instead of restating weights, so recalibration cannot drift the copies. +- `scripts/scoring/EVIDENCE_TEMPLATE.json` — the judgement contract you fill in: measured numbers plus cited yes/no answers. +- Two complete worked evaluations in `references/examples/` (ge_arrow 2.85/5, markov_asset 2.25/5) with measurement scripts, results, evidence and reports — usable as per-lecture templates and as regression anchors, plus a README documenting where every evidence number came from. +- `scripts/calibration/bellman_bench.py` — the shared aiyagari Bellman benchmark that pins the "25x as-used = score 5" efficiency anchor. +- Rubric v2 verdict gates: the logic-and-design bug cap is derived from the correctness evidence (does it build, does it diverge under x64) rather than trusting a hand-set boolean, and the correctness score caps the verdict — a float32 catastrophe with no logic bug can no longer come out as "merge". +- A no-conversion verdict: a lecture whose baseline as-used total is under the 1 s materiality floor, with a slower candidate, now gets "don't convert" instead of a polished score of the rewrite. +- A sensitivity stamp on every scorecard: each scored input is perturbed one at a time (bools flipped, counts ±1, floats ±10%) and the verdict is stamped robust / fragile / robust-at-floor with the deciding flips listed. +- K-repeat as-used measurement: `run_all.py` repeats each side three times in fresh processes, the headline speedup is the median, and per-run spread feeds a contested-band annotation. +- Triage mode — "is this lecture worth converting at all?", answered from the existing lecture alone: baseline as-used total, workload-pattern match against the two calibrated poles, crossover check, readability-cost forecast, and the weight algebra that follows. Validated blind against the three known cases before being documented, including the documented limit that it cannot predict conversion-quality defects. +- `benchmark/README.md` — the plugin's user guide: review vs triage mode, the report format, the manual pipeline quickstart, and the one rule to remember (warm-only speedups are never the headline). +- Skill wiring for installed runs: evaluations are scaffolded under `/benchmark-eval//` with the plugin read-only at `${CLAUDE_PLUGIN_ROOT}`, preconditions stated up front, and an extraction/replay diff check so the replay provably matches the lecture. +- A provenance stamp written to `results/env.json` (python/platform/numpy/jax/quantecon versions), including the titles of any failed pipeline step so a partial run cannot claim full provenance. +- `references/fixtures/rubric_v2` — synthetic evidence whose only job is to execute five v2 code paths the worked examples never touch; every source string is prefixed `SYNTHETIC:` so the numbers cannot be cited as evidence about a lecture. + +**Changed** + +- The skill is now `/benchmark:review-acceleration`, renamed from `/benchmark:eval-py-acceleration`. The rename was authored on 2026-07-21 in [#1](https://github.com/QuantEcon/skills/pull/1) but reached installed users only with this version bump. +- `score.py` takes a lecture directory path and works from any working directory, instead of resolving a lecture name against a package root. +- Correction of record on markov_asset: the lecture does build in notebook order — a stale global `err` masks a stray `err.throw()`, silently disabling the checkify stability validation. Worse than a crash, but not the build failure the original report claimed; erratum prepended to the report and the wording fixed in the examples README, `SKILL.md` and the plugin README. +- Two earlier certifications withdrawn as overstated: the reference replays deviate from the lectures' construction patterns (not "mirrors the lecture exactly"), and the as-used totals were single-pass, not medians over repeats (v2 restores repeats explicitly). +- The plugin README's triage baselines are labelled as triage-time (2026-07-21) measurements, and the framework and `SKILL.md` stop restating them — the gate reads each lecture's own `baseline_as_used_seconds`. +- `SKILL.md` forbids reporting robust-at-floor as plain robust: a verdict already in the bottom band cannot be perturbed downward, so zero deciding flips there is band geometry, not evidence strength. + +**Fixed** + +- Install was broken for every user. The repo-level `.claude-plugin/marketplace.json` omitted the required top-level `owner`, and every plugin entry — this one included — used a remote source `{"source": "github", "repo": "QuantEcon/skills", "path": "benchmark"}` that forced an install-time SSH re-clone of this repo. All three entries switched to the co-located relative-path form (`"./benchmark"`), so install uses the marketplace copy already on disk: no SSH, no auth prerequisite. Surfaced by [@xuanguang-li](https://github.com/xuanguang-li) testing this plugin, [#10](https://github.com/QuantEcon/skills/issues/10). +- The verdict band is computed from the rounded total, so the band always agrees with the number shown — raw floating-point sums could land at 2.4999999999999996 for combinations that are exactly 2.50 (797 of 78125 score combinations affected). +- `matches_under_x64` now caps correctness on its own. The extra `max_delta_shipped > 1e-8` conjunct made the guard structurally unable to fire in exactly the "wrong economics masked by low precision" case it exists to catch — such a candidate scored correctness 5 / total 3.25; it now scores correctness 1 / total 2.30, gated to net regression. +- `score.py` validates evidence before scoring and refuses evidence that omits a scored input the gates read, or that marks a structural criterion met without a citation. A missing `baseline_as_used_seconds` silently disarmed the no-conversion verdict, and stripping every citation left the score unchanged. +- The headline metrics (as-used total, cold start) are persisted to `results/as_used.json` and `results/cold_start.json` with the derived speedup, instead of existing only on the console while the docstrings claimed aggregation. +- The sensitivity stamp's denominator is honest: perturbations that raise are recorded in `perturbations_skipped` rather than silently counted as tested. +- `run_all.py` hardened — JSON scalar stdout lines no longer abort the pipeline, per-step return codes are tracked, the as-used speedup derivation guards both sides, and duplicate mode keys warn instead of silently overwriting. +- ge_arrow's `check_equivalence.py` writes `equivalence_x64.json` under `JAX_ENABLE_X64` instead of clobbering the as-shipped results. +- ge_arrow static metrics double-counted concept-token hits via a duplicated pattern (informational metric; 110 → 105). +- markov_asset's `statements_for_one_asset` renamed to `statements_for_one_result` to match the evidence-template vocabulary (values unchanged). +- Two files that were CRLF (`references/EVALUATION_FRAMEWORK.md`, the ge_arrow report) are normalized to LF, so a future one-line edit no longer renders as a whole-file diff. + +## 0.1.0 — 2026-07-07 + +First release: the plugin appears in the marketplace with a documented but not yet runnable evaluation procedure — a v0 outline skill, no executable scripts. + +- `/benchmark:eval-py-acceleration` — a v0 outline of the acceleration-review procedure: the five steps (equivalence check, static metrics, as-used benchmark, seven-dimension scoring, report), the seven weights (readability 0.25 deliberately above efficiency 0.15), the verdict bands, and the two calibration anchors (aiyagari Bellman ~25x faster as-used = HIGH; ge_arrow ~45x slower as-used = LOW). +- The guiding principle a user is meant to apply: lectures are teaching materials first, so "uses JAX" is never a goal in itself. +- `scripts/README.md` listing the eight measurement scripts still to be collected from [lecture-python.myst#717](https://github.com/QuantEcon/lecture-python.myst/pull/717). + +--- + +**There is no 0.2.0.** It existed on a branch inside [#5](https://github.com/QuantEcon/skills/pull/5) and was superseded within the same pull request; because the repo squash-merges, `main` went 0.1.0 → 0.3.0 in one commit and 0.2.0 was never published. Nothing is missing from this file. diff --git a/benchmark/README.md b/benchmark/README.md index 040b5ba..7d05184 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -9,7 +9,7 @@ One skill, two modes: | **Review** | Did this conversion PR improve the lecture? | baseline + candidate implementations | A scored report with a merge recommendation | | **Triage** | Is this lecture worth converting at all? | the existing lecture only | A predicted verdict band with the binding constraint named | -Status: evaluation system landed (v0.2.0); skill wiring tracked in [skills#4](https://github.com/QuantEcon/skills/issues/4). The system was developed and validated by [@xuanguang-li](https://github.com/xuanguang-li) on [lecture-python.myst#717](https://github.com/QuantEcon/lecture-python.myst/pull/717) and [#654](https://github.com/QuantEcon/lecture-python.myst/pull/654). +Status: evaluation system landed (v0.3.0); skill wiring tracked in [skills#4](https://github.com/QuantEcon/skills/issues/4). The system was developed and validated by [@xuanguang-li](https://github.com/xuanguang-li) on [lecture-python.myst#717](https://github.com/QuantEcon/lecture-python.myst/pull/717) and [#654](https://github.com/QuantEcon/lecture-python.myst/pull/654). ## Using the skill diff --git a/docs/developing-skills.md b/docs/developing-skills.md index f6f9e48..e2365ee 100644 --- a/docs/developing-skills.md +++ b/docs/developing-skills.md @@ -90,7 +90,36 @@ Two things follow from how it is wired. **Pages are rendered from the files wher ## Versioning and releases -Bump the version in **both** `plugin.json` and the plugin's `marketplace.json` entry — the validator enforces they match. Scaffolding → first usable content is a minor bump (the benchmark plugin's evaluation-system landing was 0.1.0 → 0.2.0). +The plugin is the released artifact and its version string is how a release is delivered, so a bump is a shipping decision rather than bookkeeping. Bump it in **both** `/.claude-plugin/plugin.json` and the plugin's `marketplace.json` entry — `validate.py` enforces that the two agree, and `plugin.json` is the one that wins at install time. + +- **A version bump is the delivery mechanism** (load-bearing — this is how the install cache works, not a convention we chose). An installed plugin lives at `~/.claude/plugins/cache////`, and `claude plugin update` compares version strings only: if the content changed and the version did not, it reports "already at the latest version" and refreshes nothing. Merged content with an unchanged version reaches nobody who has the plugin installed, and nothing warns you. This has already happened here three times — most recently [#27](https://github.com/QuantEcon/skills/pull/27), which edited a shipped `SKILL.md` under `qe/` without bumping `qe`. +- **Therefore any change to any file under `/` bumps that plugin's version.** Every file in the directory ships; there is no non-shipping edit inside it, and an exception list is where a rule like this rots. A file *moved out* of a plugin counts too — what it ships changed either way. Repo-level files — `docs/`, `README.md`, `CATALOG.md`, CI, the marketplace manifest — ship to nobody and bump nothing. +- **Choose the number for what a user of the plugin experiences.** A new skill, or a procedure that now does something materially different: minor (benchmark's scaffolding → working evaluation system was 0.1.0 → 0.3.0 — there was never a published 0.2.0). A correction that leaves the procedure as it was: patch. Nothing below 1.0.0 promises stability. +- **The catalogue's own top-level `version` in `marketplace.json`** moves only when a plugin is added or removed, which is the existing precedent. It is not a cache key and delivers nothing, so it gets no changelog entry. + +### The changelog + +Every plugin keeps its own `/CHANGELOG.md`, and the version bump and its entry land in the same PR. [`qe/CHANGELOG.md`](../qe/CHANGELOG.md) is the worked example; the format and its reasons are stated in the file itself. + +Why the file exists is worth stating honestly, because inside this repo it adds little: squash-merge already makes `git log --oneline -- qe/` one clean line per PR, and the commit-subject convention already reads as a changelog. It earns its place on two grounds. **The installed user has no git log** — the cache is an extracted directory with no `.git`, no remote to query, and for someone in the VS Code extension no practical path to one — so a file shipped inside the plugin directory is the only way to tell them what changed between the version they have and the one they would get. That is also why it is per-plugin rather than at the repo root: [self-contained plugins](#conventions) means an installed plugin ships only its own directory, so a root changelog is unreachable from the thing it describes, and a plugin's changelog links out with absolute GitHub URLs only. And **the entry is what makes the bump happen**: a reviewer looking at a diff that touches `qe/skills/…` and carries no new version heading can see the omission, which is what #27 walked into. + +Keep entries short and user-facing — what someone can now do, or what behaves differently. An entry that only restates the PR title is the right length for a small change; an entry that recapitulates the diff is not useful to anyone. Where the changelog and a PR title are word for word identical, that is fine: a changelog entry is frozen at release, so the two copies cannot drift, and they are aimed at different readers. + +There is no `Unreleased` section. It would park the description of a change away from the bump that delivers it, which is the one coupling that has to hold, and squash-merge leaves nothing for it to hold anyway: one PR is one commit, one version, one entry. Dates are the date you open the PR — you cannot know the merge date while writing, and a day's drift does not matter. Two PRs against the same plugin will conflict at the top of the file; **that conflict is the mechanism**, not a cost, since it is what stops both branches claiming the same version. Resolve it the ordinary way: rebase onto `main` and take the next version, in `plugin.json`, the `marketplace.json` entry, and the heading. + +### The CI guard + +`validate.yml` runs [`scripts/check-version-bump.py`](../scripts/check-version-bump.py) on every pull request into `main`. If any file under `/` differs from the merge base, that plugin's `plugin.json` version must differ too — and must not be one already published on `main` — and `/CHANGELOG.md` must carry a heading naming it. Any Markdown heading containing the version satisfies the last check; the file's format is not a CI contract. + +It compares against the merge base *and* the base tip, so a branch that is behind `main` cannot land a version somebody else already shipped. It reads committed history only, so `python scripts/check-version-bump.py` run locally before you commit will say so rather than pretend to have checked your working tree. It needs full history (`fetch-depth: 0`) and exits **2** rather than 0 when it cannot see the base — a guard that silently passes when it cannot run manufactures confidence. + +There is no override label. The escape hatch, on the rare occasion a bump feels disproportionate, is a patch bump: one line in two files, semantically honest, and unlike an exemption it actually delivers the change. + +A second job runs `claude plugin validate --strict` against each plugin and the marketplace, using the runtime's own parser. It is separate because it installs an npm toolchain and `validate.py` should stay fast, and it catches what a regex frontmatter reader cannot — real YAML errors, and unknown `plugin.json` keys. The CLI is pinned there deliberately: under `--strict` an upstream warning becomes an error, and an unpinned install would let someone else's release fail an unrelated PR. + +### Tags + +The repo has no git tags today, so nothing outside each `CHANGELOG.md` maps content to a version number. `claude plugin tag ./` creates `{name}--v{version}` from `plugin.json`, refuses unless the marketplace entry agrees, and refuses on a dirty working tree so the tag points at the version you meant to release. Add `--push` to publish it. Adopting it is a separate decision; nothing installs from a tag — the marketplace serves `main` — so the merge is still the release. ## PR flow diff --git a/qe/.claude-plugin/plugin.json b/qe/.claude-plugin/plugin.json index f4d50b1..a2e3314 100644 --- a/qe/.claude-plugin/plugin.json +++ b/qe/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "qe", "description": "QuantEcon's author-facing base skills — style checking, lecture editing support, and working through a PR's review feedback", - "version": "0.2.0", + "version": "0.2.1", "author": { "name": "QuantEcon" } } diff --git a/qe/CHANGELOG.md b/qe/CHANGELOG.md new file mode 100644 index 0000000..4bef95c --- /dev/null +++ b/qe/CHANGELOG.md @@ -0,0 +1,64 @@ +# Changelog — `qe` + +Every released version of this plugin, newest first. A version exists because the content below shipped in it: the plugin cache is keyed by version string, so what you have installed is exactly the entries down to the version `claude plugin list` reports for `qe`. + +Versions are [semver](https://semver.org) as a user of this plugin experiences it — a new skill, or a procedure that now does something materially different, is a minor bump; a correction that leaves the procedure as it was is a patch. Nothing below 1.0.0 promises stability. + +Repository: [QuantEcon/skills](https://github.com/QuantEcon/skills) ([every commit that touched this plugin](https://github.com/QuantEcon/skills/commits/main/qe)). How a release is made: [developing-skills § Versioning and releases](https://github.com/QuantEcon/skills/blob/main/docs/developing-skills.md#versioning-and-releases). + +## 0.2.1 — 2026-08-03 + +**Fixed** + +- `/qe:copilot-review`'s status banner said the skill "has never run from an installed plugin", which was false while being read from one. It now reads **operational**, validated on 2026-08-03 from an installed `qe@quantecon`: plugin-root path resolution, cross-repo mode, and running from outside a working tree. The correction was authored in [#27](https://github.com/QuantEcon/skills/pull/27), which did not bump `qe` — so until this release no installed user had received it. +- The same banner no longer writes `${CLAUDE_PLUGIN_ROOT}` in prose. The harness interpolates the variable, so on an installed plugin the sentence rendered with the reader's own cache path spliced into it. It now appears only inside code blocks. + +**Added** + +- This changelog. + +## 0.2.0 — 2026-08-03 + +Adds `/qe:copilot-review`, the first `qe` skill that does real work end to end. + +**Added** + +- `/qe:copilot-review [PR] [owner/repo]` — runs the loop fetch → advise → fix → reply over a pull request's Copilot review. Both arguments are optional; with neither it works on the current branch's PR in the repo you are standing in. Five argument forms are accepted: none, `42`, `owner/repo 42`, `owner/repo#42`, and a full PR URL. Naming both repo and PR lets the skill run from anywhere, including outside a git working tree. +- Threaded replies via `pulls//comments//replies`, which is what makes a comment resolvable from the GitHub UI — a top-level `gh pr comment` does not thread and leaves the conversations open. +- `scripts/fetch-copilot.sh` — a read-only dump of a PR's Copilot review: the resolved repo and where it came from, the PR's state and title, the review overview, then every inline comment as `== ID [REPLIED by ] path:line`. Invoked through `bash` so it works whether or not the executable bit survived install; `--help` prints the argument forms. +- Safe re-runs: the `[REPLIED by ]` marker names who answered each thread, so a second run replies only to comments still unanswered instead of double-posting. +- An explicit "what this skill writes" table — every mutating call (`/replies` POST, `gh pr comment`, `git commit`/`push`), the step it happens in, and the go-ahead that gates it — plus a warning not to run the skill headlessly, since a posted reply cannot be unposted and CI has nobody to confirm. +- A stop-on-ambiguity rule in the advise step: judgement calls surface the competing interpretations and ask before any code is touched, rather than being silently patched. +- Guidance that Copilot is sometimes wrong — an invalid comment gets a reasoned push-back reply, not a silent skip — and that reply bodies follow the repo's rules for writing to GitHub. + +**Changed** + +- The plugin description widens from "style checking and lecture editing support" to "style checking, lecture editing support, and working through a PR's review feedback", so `qe` now covers a lecture from drafting through to merging its PR. + +**Fixed** + +These are defects found while promoting the skill from a personal one, so none of them ever shipped in a `qe` release. They are listed because they describe what the shipped script does and does not do. + +- Full pagination of `pulls//comments` in both passes, so PRs with more than 30 review comments are reported completely. Previously only the first page was read — on a 104-comment PR it showed 19 and reported none as answered, so a re-run would have posted 19 duplicate replies. +- The overview shows the most detailed Copilot review rather than the newest, and says how many there were. Copilot re-reviews after every push and each re-review is a ~120-character stub, so on a PR with 33 reviews the summary previously rendered as nothing. +- `gh` failures surface `gh`'s own stderr instead of being reported uniformly as "no such PR", so missing `gh`, unauthenticated, offline and rate-limited are distinguishable. +- Transposed arguments are rejected. `fetch-copilot.sh 42 owner/repo` used to discard the second argument and produce a complete, plausible report about a different repository. +- Every line quoted from GitHub is prefixed with `| `, so a comment body cannot forge the `== ID` record header the reply step keys on, and third-party content is visibly marked as data to assess rather than instruction to obey. +- A PR URL no longer half-succeeds into a broken copy-paste command; a null comment body no longer prints the literal `null`; and the ANSI colour in the error path no longer leaks into non-TTY logs. +- Two documented claims corrected: Copilot is one GitHub App under three login strings (`copilot-pull-request-reviewer`, the same with `[bot]`, and `Copilot`), which is why reviews and comments filter on different values; and `line` is null on *outdated* comments, not multi-line ones. +- The working-tree requirement is stated correctly: the tree is what an omitted repo *or* PR number is inferred from, so naming a repo alone is not enough when the PR number is left out. + +## 0.1.0 — 2026-07-21 + +First release. The author-facing style-check surface appears in the slash menu as scaffolding: the skills register and report that they are not yet operational when run. + +- `/qe:check-style [categories...]` — the umbrella style check for one lecture, with an optional category filter (`/qe:check-style lectures/aiyagari.md figures math`). On a PR branch the lecture argument can be omitted to mean "the lectures changed on this branch". +- Six per-category entry points running the same shared rules restricted to one category: `/qe:check-writing`, `/qe:check-math`, `/qe:check-code`, `/qe:check-figures`, `/qe:check-jax`, `/qe:check-refs`. +- The contract every check will follow: deterministic preflight first (build-breaking rules ahead of mechanical ones), then per-category passes, then one report table per category — rule ID, severity, `file:line`, finding, proposed fix — with counts by severity. +- Report first, fix only on request: nothing is edited without confirmation, and rules marked `auto_fix: false` or `build_risk: true` (RNG-stream changes, for instance, which alter published figures) are presented for the author to apply rather than applied. +- `references/rules/README.md` — the rule schema the vendored snapshot will use (`id`, `category`, `mode`, `severity`, `build_risk`, `auto_fix`, `detection`, `exclusions`), and the statement that rule text is authored only in `QuantEcon/style-guide` and rendered here. +- `scripts/README.md` — the pending MyST-context-aware `preflight.py` and `sync-rules.py` drift check. + +--- + +**Before this file existed**, two changes to `qe/` shipped without a version bump, so two different trees have been distributed under one version string each. Under 0.1.0, [#5](https://github.com/QuantEcon/skills/pull/5) rewrote the status banner in all seven `check-*` skills to point at [issue #3](https://github.com/QuantEcon/skills/issues/3) instead of `CATALOG.md`. Under 0.2.0, [#27](https://github.com/QuantEcon/skills/pull/27) made the correction now released as 0.2.1. If your install predates those dates, `claude plugin update` will not have reconciled it — reinstalling at 0.2.1 gets you the current tree. The [CI guard](https://github.com/QuantEcon/skills/blob/main/scripts/check-version-bump.py) landed alongside this release is what stops it happening again. diff --git a/scripts/check-version-bump.py b/scripts/check-version-bump.py new file mode 100755 index 0000000..40ce805 --- /dev/null +++ b/scripts/check-version-bump.py @@ -0,0 +1,594 @@ +#!/usr/bin/env python3 +"""Fail a pull request that changes a plugin's shipped files without bumping +that plugin's version. + +Run from anywhere; paths are resolved from this file's location: + + python scripts/check-version-bump.py [--base origin/main] + +It reads committed history only. Uncommitted work in the tree is not part of +the check — the script says so when it sees any. + +Why this exists +--------------- +The plugin install cache is keyed by *version string* — +`~/.claude/plugins/cache////` — and +`claude plugin update` compares version strings only. Change a SKILL.md +without changing the version and the update reports "already at the latest +version" and refreshes nothing: the edit is merged, published, and invisible +to every consuming repository. **The version bump is the delivery mechanism, +not bookkeeping**, which is why an unbumped change is a CI failure rather than +a review nit. It has happened here already — PR #27 edited a shipped SKILL.md +under `qe/` with `qe` left at 0.2.0. + +What counts as a change +----------------------- +Every tracked file under a plugin's directory, with no exemptions — +`SKILL.md`, `scripts/`, `references/`, the plugin's `README.md`, and its +`CHANGELOG.md`. The plugin directory *is* the shipped artefact; if a file is +inside it, a user gets it from the cache, and the only way to give them the new +copy is a new version. A file *leaving* a plugin directory counts too, which is +why rename detection is switched off below: what a plugin ships changed either +way. + +What this does not check +------------------------ +That `plugin.json` and the `marketplace.json` entry carry the same version — +`scripts/validate.py` enforces that, and duplicating it here would mean two +places to fix when the rule changes. This guard reads `plugin.json` as the +authority (it is what wins at install time) and, if the two disagree, prints a +pointer to the validator rather than raising its own error. + +It does *not* delegate the question of whether a version exists at all. +`validate.py` only compares the two manifests to each other, so a missing +`version` key in both, or an unquoted `"version": 0.2`, passes there — and +would read here as "some other value" if it were trusted. A version that is +not a non-empty string is not a usable cache-key path segment, so this guard +treats it as a failure in its own right. + +And the release note +-------------------- +A plugin that legitimately bumps must also carry a heading for the new version +in `/CHANGELOG.md`, in the same diff. That is not tidiness: the +changelog is the only record of what changed that reaches someone whose copy is +an extracted install cache with no `.git` and no way to reach one. The check is +deliberately loose — any Markdown heading containing the version as a whole +token satisfies it — so the file's format is not a CI contract. + +Exit codes +---------- +0 every touched plugin was bumped (or needed no bump) +1 at least one plugin changed without a usable bump — the version did not + move, it moved backwards, it landed on a version string already published + on the base branch, there is no usable version string at all, or the new + version has no changelog entry +2 the check could not be performed (usually a shallow clone — see below); + deliberately *not* 0, because a guard that silently passes when it cannot + see the base is worse than no guard + +CHANGELOG.md: why a bump is required for changelog-only edits +------------------------------------------------------------- +Against requiring one: a changelog is documentation, not behaviour; no skill +reads it; a typo fix would mint a release with no substance in it. + +For requiring one (what this script does): the changelog ships *inside* the +plugin directory, so it is copied into the install cache like everything else. +If it can be edited without a bump, the changelog a user reads at +`.../cache/quantecon/qe/0.2.0/CHANGELOG.md` is permanently a different document +from the one on `main` — a drifted copy that nobody can tell is stale, which is +the exact failure the repo's single-source-of-truth principle exists to +prevent. An exemption also has to define "only", so a mixed PR would need +per-file bookkeeping and its own tests. And in the intended workflow the rule +costs nothing: a changelog entry is written *in* the release commit that bumps +the version, so the requirement only bites the retroactive edit — the one case +where what a user reads really is changing, and where a patch bump is the +honest description of what happened. +""" + +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +MARKETPLACE_PATH = ".claude-plugin/marketplace.json" + +EXIT_OK = 0 +EXIT_UNBUMPED = 1 +EXIT_CANNOT_CHECK = 2 + +# How many changed files to list per plugin before truncating the report. +MAX_LISTED = 8 + +# Sentinels for the two ways a version can be unreadable. Neither is a string, +# so neither can ever compare equal to a real version and be mistaken for one. +NO_MANIFEST = object() # no plugin.json at that revision +NOT_A_VERSION = object() # plugin.json exists, "version" is missing or not a string + + +# -------------------------------------------------------------------------- +# git plumbing +# -------------------------------------------------------------------------- + +def git(*args): + return subprocess.run( + ["git", "-C", str(ROOT), *args], capture_output=True, text=True + ) + + +def git_out(*args): + """Stdout of a git command, or None if it failed.""" + proc = git(*args) + return None if proc.returncode != 0 else proc.stdout + + +def cannot_check(headline, *lines): + # Flush stdout first: CI merges the two streams, and a report that arrives + # out of order reads as though the guard passed and then complained. + sys.stdout.flush() + print(f"version-bump guard: CANNOT CHECK — {headline}", file=sys.stderr) + for line in lines: + print(f" {line}" if line else "", file=sys.stderr) + sys.stderr.flush() + sys.exit(EXIT_CANNOT_CHECK) + + +CHECKOUT_FIX = ( + " - uses: actions/checkout@v4", + " with:", + " fetch-depth: 0 # the guard needs the base branch and the merge base", +) + + +def resolve_merge_base(base_ref, head_ref): + """Merge base of base_ref and head_ref, or a loud exit explaining the fix. + + Every failure here is a *setup* failure, and setup failures exit 2 rather + than 0. actions/checkout defaults to a single ref at depth 1, which leaves + no base branch and no shared history — under that default this guard can + see nothing, and a green tick would be a lie. + """ + if git_out("rev-parse", "--verify", "--quiet", f"{base_ref}^{{commit}}") is None: + cannot_check( + f"`{base_ref}` does not exist in this clone, so there is no base to diff against.", + "actions/checkout@v4 fetches one ref at depth 1 by default, which does not", + "create remote-tracking refs for other branches. Fix the workflow:", + "", + *CHECKOUT_FIX, + "", + "Refusing to report success on a check that never ran.", + ) + + if (git_out("rev-parse", "--is-shallow-repository") or "").strip() == "true": + cannot_check( + "this is a shallow clone, so the merge base cannot be trusted.", + f"`{base_ref}` resolves, but grafted history makes `git merge-base` either fail", + "or answer from a truncated graph — a wrong base silently changes which files", + "the guard thinks the PR touched. Fix the workflow:", + "", + *CHECKOUT_FIX, + "", + "Or, if the checkout must stay shallow, deepen it before this step:", + "", + " git fetch --unshallow origin", + f" git fetch origin {base_ref.split('/')[-1]}:refs/remotes/{base_ref}", + ) + + merge_base = git_out("merge-base", base_ref, head_ref) + if merge_base is None or not merge_base.strip(): + cannot_check( + f"`{base_ref}` and `{head_ref}` share no common ancestor.", + "The clone is missing history, or the branch was force-pushed onto an", + "unrelated root. Fix the workflow:", + "", + *CHECKOUT_FIX, + ) + return merge_base.strip() + + +def changed_files(base_ref, head_ref): + """Paths the PR touched: the three-dot diff, i.e. base..merge-base..head. + + Three dots, not two: a two-dot diff against a moving `origin/main` reports + every file that main changed since the branch point as though the PR had + changed it. + + `--no-renames`, because a rename is otherwise reported as its destination + only, which hides the deletion side. Moving a file *out* of a plugin + changes what that plugin ships exactly as much as editing it does, and this + repo moves content out of plugins routinely — the single-source-of-truth + principle is what makes that a normal PR rather than an exotic one. + """ + proc = git("diff", "--name-only", "--no-renames", "-z", f"{base_ref}...{head_ref}") + if proc.returncode != 0: + cannot_check( + f"`git diff {base_ref}...{head_ref}` failed.", + proc.stderr.strip() or "(no stderr)", + ) + return [p for p in proc.stdout.split("\0") if p] + + +def load_json_at(rev, path): + """Parse a JSON file as of `rev`; None if it does not exist there.""" + out = git_out("show", f"{rev}:{path}") + if out is None: + return None + try: + return json.loads(out) + except json.JSONDecodeError as exc: + cannot_check(f"{path} at {rev[:12]} is not valid JSON — {exc}") + + +# -------------------------------------------------------------------------- +# plugin discovery +# -------------------------------------------------------------------------- + +def plugin_index(marketplace): + """Map plugin directory prefix -> {name, entry_version}, from the catalogue. + + The set of plugins is whatever `marketplace.json` says it is at that + revision — never a hardcoded list, so adding or renaming a plugin needs no + change here. `source` is normally the relative string form (`"./qe"`); the + object form is accepted for completeness, taking `path` and falling back to + the plugin name. + """ + index = {} + if not isinstance(marketplace, dict): + return index + for entry in marketplace.get("plugins") or []: + if not isinstance(entry, dict): + continue + name = entry.get("name") + source = entry.get("source") + if isinstance(source, str): + prefix = source + elif isinstance(source, dict): + prefix = source.get("path") or name + else: + prefix = None + if not name or not prefix: + continue + if prefix.startswith("./"): + prefix = prefix[2:] + prefix = prefix.strip("/") + # A non-local source ships from somewhere else; nothing in this repo's + # diff can be part of it, and `..`/absolute paths are not ours to police. + if not prefix or prefix.startswith("/") or ".." in Path(prefix).parts: + continue + index[prefix] = {"name": name, "entry_version": entry.get("version")} + return index + + +def manifest_version(rev, prefix): + """The plugin's version at `rev`, or a sentinel saying why there isn't one. + + The version IS the cache-key path segment, so it must be a non-empty + string. A missing key, a null, or an unquoted number (`"version": 0.2`) is + not one. `validate.py` does not catch those — it only compares the two + manifests to each other, and `None == None` and `0.2 == 0.2` both pass — + so they are caught here instead of being trusted as "some other version". + """ + if not prefix: + return NO_MANIFEST + manifest = load_json_at(rev, f"{prefix}/.claude-plugin/plugin.json") + if not isinstance(manifest, dict): + return NO_MANIFEST + version = manifest.get("version") + if not isinstance(version, str) or not version.strip(): + return NOT_A_VERSION + return version + + +def changelog_mentions(rev, prefix, version): + """Does `/CHANGELOG.md` at `rev` carry a heading for `version`? + + Deliberately loose: any Markdown heading line containing the version as a + whole token counts, so the file's exact heading style is not a CI contract. + The check exists so that the release note lands in the same diff as the + bump a reviewer is looking at, not to police formatting. + + Returns True, False, or None when there is no CHANGELOG.md at all. + """ + text = git_out("show", f"{rev}:{prefix}/CHANGELOG.md") + if text is None: + return None + pattern = re.compile(r"(? len(shown): + lines.append(f" … and {len(paths) - len(shown)} more") + return lines + + +CACHE_NOTE = ( + " This is delivery, not bookkeeping. The install cache is keyed by the", + " version string — ~/.claude/plugins/cache//{name}//", + " — and `claude plugin update` compares version strings only. Without a new", + " one, every consuming repo is told \"already at the latest version\" and", + " goes on running the old files.", +) + + +def failure_report(failure): + name, prefix, paths = failure["name"], failure["prefix"], failure["paths"] + kind = failure["kind"] + manifest = f"{prefix}/.claude-plugin/plugin.json" if prefix else "the plugin manifest" + + if kind == "downgrade": + headline = (f" {name} — {files_phrase(paths)} changed and the version went " + f"BACKWARDS, {failure['base']!r} → {failure['head']!r}") + remedy = [ + f" Fix: choose a version above {failure['base']!r}. A version string is a", + " cache key, so reusing an old one leaves anybody who installed that", + " version with their stale copy and no way to be told about it.", + ] + elif kind == "reused": + headline = (f" {name} — {files_phrase(paths)} changed and {failure['head']!r} is " + f"already published on the base branch") + remedy = [ + f" Fix: choose a version above {failure['base']!r} — the branch is behind", + " the base and is reusing a version string that is already serving as a", + " cache key, so the new content would never reach anyone who has it.", + " Rebase onto the base branch first, then pick the next version.", + ] + elif kind == "no_version": + headline = (f" {name} — {files_phrase(paths)} changed and there is no usable " + f"version string in {manifest}") + remedy = [ + " Fix: set \"version\" to a quoted semver string, e.g. \"0.2.1\". An unquoted", + " number (0.2), a null, or a missing key is not a cache-key path segment.", + " scripts/validate.py does not catch this: it only compares plugin.json to", + " the marketplace entry, and two identical non-strings compare equal.", + ] + elif kind in ("no_changelog", "no_entry"): + missing = ("CHANGELOG.md does not exist" if kind == "no_changelog" + else f"CHANGELOG.md has no heading for {failure['head']!r}") + headline = (f" {name} — bumped to {failure['head']!r} but " + f"{prefix}/{missing}") + remedy = [ + f" Fix: add a heading for {failure['head']!r} at the top of", + f" {prefix}/CHANGELOG.md, above the previous version, saying what a user", + " of this plugin can now do or what behaves differently. One line is the", + " right length for a small change.", + "", + " The entry belongs in this diff because the changelog ships inside the", + " plugin directory: it is the only record of what changed that reaches", + " someone whose copy is an extracted install cache with no git history.", + " Any Markdown heading containing the version satisfies this check — the", + " format is not a CI contract.", + ] + return [headline, "", *remedy] + else: # "unbumped" + headline = (f" {name} — {files_phrase(paths)} of shipped content changed, " + f"version still {failure['head']!r}") + remedy = [ + f" Fix: raise \"version\" in {manifest} and in", + f" the `{name}` entry of {MARKETPLACE_PATH} (scripts/validate.py", + " enforces that the two agree), and add the matching entry to", + f" {prefix}/CHANGELOG.md. Semver as this repo already uses it:", + " patch for a fix, minor for new or reworked capability.", + ] + + return [ + headline, + *describe(paths), + "", + *remedy, + "", + *(line.format(name=name) for line in CACHE_NOTE), + ] + + +# -------------------------------------------------------------------------- + +def main(argv=None): + parser = argparse.ArgumentParser( + description="Fail a PR that changes plugin files without bumping the plugin version." + ) + parser.add_argument( + "--base", default="origin/main", + help="base ref the PR targets (default: origin/main)", + ) + parser.add_argument( + "--head", default="HEAD", + help="head ref under test (default: HEAD)", + ) + args = parser.parse_args(argv) + + merge_base = resolve_merge_base(args.base, args.head) + files = changed_files(args.base, args.head) + + # Three views of the catalogue. The merge base is where the branch forked; + # the base *tip* is what is published right now (a branch that is behind can + # otherwise land a version already in use as a cache key); HEAD is what the + # PR proposes. + base_index = plugin_index(load_json_at(merge_base, MARKETPLACE_PATH) or {}) + tip_index = plugin_index(load_json_at(args.base, MARKETPLACE_PATH) or {}) + head_market = load_json_at(args.head, MARKETPLACE_PATH) + if head_market is None: + cannot_check( + f"{MARKETPLACE_PATH} does not exist at {args.head}.", + "Without the catalogue there is no way to discover which directories are plugins.", + ) + head_index = plugin_index(head_market) + + touched = group_by_plugin(files, set(base_index) | set(tip_index) | set(head_index)) + + # Everything below is keyed by plugin NAME, not by directory prefix. `source` + # is only where a plugin happens to live in one revision: keying on the + # prefix makes relocating a plugin look like a delete plus a brand-new + # plugin, and both halves of that pair are exempt from a bump. + prefixes_of = {} + for index, rev_key in ((base_index, "base"), (tip_index, "tip"), (head_index, "head")): + for prefix, meta in index.items(): + prefixes_of.setdefault(meta["name"], {})[rev_key] = prefix + + print(f"version-bump guard: base {args.base} (merge base {merge_base[:12]}), " + f"{len(files)} file(s) changed") + + if args.head == "HEAD": + dirty = [ln for ln in (git_out("status", "--porcelain") or "").splitlines() if ln] + if dirty: + print(f" note {len(dirty)} uncommitted change(s) in the working tree are NOT " + f"part of this check — it reads committed history only.") + + failures = [] + reported = False + + for name in sorted(prefixes_of): + slots = prefixes_of[name] + paths = sorted({p for slot in slots.values() for p in touched.get(slot, [])}) + if not paths: + continue + reported = True + + head_prefix = slots.get("head") + if head_prefix is None: + print(f" ok {name} — removed from {MARKETPLACE_PATH} at {args.head}; " + f"no bump required ({files_phrase(paths)})") + continue + + # Every version this plugin is already published under, on either the + # fork point or the current base tip. + published = {v for v in (manifest_version(merge_base, slots.get("base")), + manifest_version(args.base, slots.get("tip"))) + if isinstance(v, str)} + + head_version = manifest_version(args.head, head_prefix) + if not isinstance(head_version, str): + why = ("no plugin.json" if head_version is NO_MANIFEST + else "\"version\" is missing or not a string") + failures.append({"name": name, "prefix": head_prefix, "paths": paths, + "base": None, "head": None, "kind": "no_version"}) + print(f" !! {name} — {files_phrase(paths)} changed and " + f"{head_prefix}/.claude-plugin/plugin.json has no usable version " + f"({why})") + continue + + entry_version = head_index[head_prefix].get("entry_version") + if entry_version not in (None, head_version): + print(f" note {name} — {MARKETPLACE_PATH} says {entry_version!r}, " + f"plugin.json says {head_version!r}; using plugin.json " + f"(scripts/validate.py reports the mismatch)") + + # `published` is a set, and every unorderable version shares the key (). + # max() over tied keys returns whichever element iteration reached first, + # and set order for strings varies with hash randomisation — so two + # pre-release versions would make `highest` differ between runs. Sorting + # first makes the tie-break the version string itself: still arbitrary, + # but the same arbitrary answer every time. A guard that reports + # different things on different runs cannot be trusted with either. + highest = (max(sorted(published), key=lambda v: version_key(v) or ()) + if published else None) + unorderable = sorted(v for v in published if version_key(v) is None) + if unorderable: + print(f" note {name} — {', '.join(repr(v) for v in unorderable)} " + f"{'is' if len(unorderable) == 1 else 'are'} not X.Y.Z, so the " + f"downgrade check is skipped for this plugin; the version must " + f"still differ from every published one") + record = {"name": name, "prefix": head_prefix, "paths": paths, + "base": highest, "head": head_version} + + if published: + if head_version in published: + record["kind"] = "unbumped" if head_version == highest else "reused" + failures.append(record) + if record["kind"] == "unbumped": + print(f" !! {name} — {files_phrase(paths)} changed, version still " + f"{head_version!r}") + else: + print(f" !! {name} — {files_phrase(paths)} changed and " + f"{head_version!r} is already published (base tip is at " + f"{highest!r})") + continue + + base_key, head_key = version_key(highest), version_key(head_version) + if base_key and head_key and head_key < base_key: + record["kind"] = "downgrade" + failures.append(record) + print(f" !! {name} — version went backwards, {highest} → {head_version}") + continue + + # The version is usable. The release it names still owes an entry, in + # this diff, where the reviewer looking at the content change can see it. + entry = changelog_mentions(args.head, head_prefix, head_version) + if entry is not True: + record["kind"] = "no_changelog" if entry is None else "no_entry" + failures.append(record) + print(f" !! {name} — {highest or 'new'} → {head_version}, but " + f"{head_prefix}/CHANGELOG.md " + f"{'does not exist' if entry is None else 'has no entry for it'}") + continue + + if highest is None: + print(f" ok {name} — new plugin at {head_version!r} " + f"({files_phrase(paths)}); no published version to bump from") + else: + print(f" ok {name} — {highest} → {head_version} ({files_phrase(paths)})") + + if not reported: + print(" ok no plugin directory touched — nothing to bump") + return EXIT_OK + + if failures: + plural = "" if len(failures) == 1 else "s" + sys.stdout.flush() + print(f"\nversion-bump guard: {len(failures)} plugin{plural} cannot ship as " + f"proposed.\n", file=sys.stderr) + for failure in failures: + for line in failure_report(failure): + print(line, file=sys.stderr) + print("", file=sys.stderr) + sys.stderr.flush() + return EXIT_UNBUMPED + + print("\nversion-bump guard: every touched plugin carries a new version.") + return EXIT_OK + + +if __name__ == "__main__": + sys.exit(main())