perf: make the deploy cost proportional to the change, not the whole site

.devcontainer/devcontainer.json

@@ -24,7 +24,8 @@
 		// Allow the container to access the host X11 display and EPICS CA
 		"--net=host",
 		// Make sure SELinux does not disable with access to host filesystems like tmp
-		"--security-opt=label=disable"
+		"--security-opt=label=disable",
+		"--device=/dev/net/tun"
 	],
 	"mounts": [
 		// Mount in the user terminal config folder so it can be edited

.github/workflows/_test.yml

@@ -8,6 +8,16 @@ jobs:
       - uses: actions/checkout@v7
       - name: Unit tests (widget logic + make-switcher parity)
         run: npm test
+
+      # publish.yml's gather steps are the most intricate code in the repo and used to
+      # be verifiable only by merging and watching a real deploy. The harness loads them
+      # out of the YAML and runs them against a mock gh, so they cannot drift.
+      - name: Install uv
+        uses: astral-sh/setup-uv@v8.2.0   # full version: setup-uv uses immutable releases
+        with:
+          enable-cache: false
+      - name: Workflow gather-step tests
+        run: npm run test:workflows
       # release.yml downloads every artifact in the run and attaches them, so name this
       # explicitly (not the default "artifact") and fail loudly if the plugin is missing
       # — it's a release asset.

.github/workflows/publish-dispatch.yml

@@ -36,6 +36,14 @@ jobs:
     uses: ./.github/workflows/publish.yml
     with:
       version-name: ${{ inputs.version-name }}          # "" on dispatch → pure durable gather
+      # Site-size policy, so it applies on EVERY path (call and dispatch alike) — hence a
+      # literal here rather than another input threaded down from ci.yml.
+      # This repo deliberately differs from the value the tutorial ships ("20"): our own
+      # site is ~223 MB packed across 24 releases, a fifth of the Pages cap, so capping
+      # would drop old versions for no gain. The capped path is covered by the unit tests
+      # and the workflow harness rather than by our own deploys. Set a real number here
+      # once a site nears the cap — see docs/how-to/keep-the-site-small.md.
+      max-releases: "0"
       pr: ${{ inputs.pr }}                               # set (dispatch only) → pin that fork head SHA
       dispatch-workflow: publish-dispatch.yml            # the file the tag re-dispatch re-fires
       retry-until: ${{ inputs.retry-until }}              # "" on workflow_call/no-retry dispatch → no auto-retry

.gitignore

@@ -1,3 +1,4 @@
 node_modules/
 docs/_build/
 *.log
+__pycache__/

CLAUDE.md

@@ -12,7 +12,7 @@ to `publish.yml`, not a separately consumed action.
 plugins/version-switcher.mjs                   # MyST directive + anywidget runtime (single file, no README — docs are in docs/)
 assemble/assemble.mjs                          # INTERNAL: dependency-free Node kernel for publish.yml's "Generate" step (the `generate` subcommand writes switcher.json + index.html)
 scripts/migrate.sh                             # operator gh-pages → durable-source migration (bash); two-phase: reversible cutover, then guarded --delete-gh-pages
-test/                                          # npm test suite (node, no framework)
+test/                                          # npm test suite (node, no framework) + test/workflow-harness/ (python: loads publish.yml's gather steps out of the YAML and runs them against a mock gh)
 docs/                                          # this repo's own docs (dogfoods the plugin)
 .github/workflows/docs.yml                     # PUBLIC reusable: build at the versioned BASE_URL → pack docs.zip → upload `docs` artifact (workflow_call; build-command input)
 .github/workflows/publish.yml                  # PUBLIC reusable ENGINE (PRIVILEGED): branches on the event into 3 jobs — deploy (gather+extract inline bash + assemble.mjs@job.workflow_sha + Pages), tag-re-dispatch (re-fire the shim), fork-warn. workflow_call ONLY; reached via the publish-dispatch.yml shim
@@ -39,23 +39,65 @@ public action.
 See [`docs/explanations/architecture.md`](docs/explanations/architecture.md) for the
 full rationale. In short:
 
-### The default branch is self-durable in the site (`_sources/`)
+### The default branch's durable copy lives in the Actions cache
 Releases are durable (Release `docs.zip` assets) and PRs are ephemeral by design, but
-the **default branch** had no permanent source — gathered from its latest CI artifact,
-which expires, after which it drops out and the guard hard-fails. So each deploy
-persists the default branch's `docs.zip` (the one it arrived as — current build,
-gathered run, or the fallback — copied verbatim) into the published site at
-`_sources/<branch>.zip` (excluded from version discovery), and a deploy whose fresh
-artifact is gone restores the branch from that durable in-site copy (fetched from
-`$PAGES_URL` — the live Pages URL resolved from the Pages API, so custom domains
-work, falling back to `https://<owner>.github.io/<repo>`; the same URL roots the
-`switcher.json` entries). Before the branch ever builds
-docs (mid gh-pages migration) a final rung reads a **published seed release**
-(`pages-default-seed`, created by `scripts/migrate.sh` from the old gh-pages `<default>/`
-tree) and persists it to `_sources` on that deploy — so a repo can cut over to the
-reusable workflow in one PR, before `docs→main`. Drafts can't be used: a `contents:read`
-deploy token can't read them (verified), so the seed is a published release on a
-sentinel tag, deleted at finalize once `_sources/<default>.zip` is live.
+the **default branch** has no permanent source — gathered from its latest CI artifact,
+which expires, after which it drops out and the guard hard-fails. So each
+default-branch deploy keeps a copy of its `docs.zip` in the Actions cache
+(content-addressed key `mvs-default-v1-<sha>`, so an unchanged branch adds no entry),
+and a deploy whose fresh artifact is gone restores from there.
+
+This used to be published *into the site* at `_sources/<branch>.zip`. That was
+permanent, but it shipped a multi-megabyte zip inside every Pages artifact forever on a
+site already pressing the 1 GB cap (see "The site has a hard size ceiling"). The cache
+is evicted after 7 days unread and by LRU past 10 GB, so a repo quiet for >1 week whose
+main artifact ALSO expired loses the rung and hard-fails loudly — accepted trade.
+`_sources` is no longer written; the READ survives as a deprecated one-shot rung so a
+site last deployed by an older `publish.yml` still has a source on its first upgraded
+deploy (fetched from `$PAGES_URL` — the live Pages URL from the Pages API, so custom
+domains work; the same URL roots the `switcher.json` entries).
+
+Before the branch ever builds docs (mid gh-pages migration) a final rung reads a
+**published seed release** (`pages-default-seed`, created by `scripts/migrate.sh` from
+the old gh-pages `<default>/` tree) — so a repo can cut over to the reusable workflow in
+one PR, before `docs→main`. Drafts can't be used: a `contents:read` deploy token can't
+read them (verified), so the seed is a published release on a sentinel tag, deleted at
+finalize. `migrate.sh --delete-gh-pages`'s guard now requires the default branch to be
+live in the new site AND to have a non-expired `docs` artifact of its own (it used to
+probe `_sources/<default>.zip`, which passed even while the content came from the seed
+— i.e. while gh-pages was still the only real copy).
+
+### The site has a hard size ceiling — cap the releases
+`upload-pages-artifact` tars the WHOLE site into one artifact and Pages rejects it over
+**1 GB**. blueapi at 131 released `docs.zip`s was at 452 MB, +~5 MB/release: a few years
+from deploys simply failing, silently, with no warning on the way. `publish.yml` takes
+**`max-releases`** (default `0` = unlimited, so upgrading never silently deletes
+versions); the tutorial's shim ships `20`. Set it as a LITERAL in the shim's `with:`,
+not threaded from `ci.yml` — the shim is the single caller on every path, so a literal
+there can't be bypassed by a manual dispatch. A deploy past 700 MB warns — measured on
+the PACKED artifact (the cap is on the gzipped tar; this repo's site is 734 MiB on disk
+and 223 MiB deployed, so `du` would cry wolf at a third of real usage). The tree is only
+packed to measure once it exceeds `SIZE_PROBE_BYTES`, since packing costs seconds; `tar
+-ch` matches upload-pages-artifact's `--dereference`, which inflates `stable/` into a
+full copy.
+
+Selection is `assemble.mjs`'s `selectReleases` (pure, unit-tested; it replaced an
+untested bash `case`), ranked by **`created_at`** — NOT `published_at`, which lies when
+an old release is re-published (blueapi's `1.3.2-a9`: created 2025-10, published
+2026-07, and under `published_at` it outranked the newer `1.11.3`). No version-number
+parsing: tags are too inconsistent across repos. The seed release is exempt from the cap.
+`getSortedTags()`/`orderVersions()` are untouched — that's switcher DISPLAY order, a
+separate concern.
+
+### The gather is cached, indexed and parallel
+The deploy's cost is O(whole site), not O(the change) — identical work whatever fired
+it. On blueapi that was 654 s against a 27–45 s docs build. Fixed by: an `actions/cache`
+entry of release zips keyed on the exact set of asset ids to publish (assets are
+immutable; files named by ASSET ID so a re-cut release can't serve stale bytes; pruned to
+the published set), the artifacts API paginated ONCE and indexed newest-per-head-SHA
+(it was re-paginated per open PR — 27× for blueapi, ~175 s), and parallel downloads
+(`xargs -P 8`). **Caches are saved only on the default branch**: a cache written on a PR
+ref is scoped to that PR and unreadable elsewhere.
 
 ### Reconstruct from durable sources, publish the whole tree
 Every deploy rebuilds the **complete** site from authoritative inputs — `main`'s
@@ -224,7 +266,8 @@ always, fork PRs only when the SHA carries that status.
 ## Developing
 
 ```bash
-npm test                    # run the test suite
+npm test                    # run the unit tests
+npm run test:workflows      # run publish.yml's gather steps against a mock gh (needs uv)
 npm run docs                # build docs (same command CI uses)
 npm run docs-dev            # live-preview docs with the plugin loaded from local plugins/
 ```

assemble/assemble.mjs

@@ -5,7 +5,13 @@
  * dir, then the extract step unzips them all into the site tree. This file is
  * the final step: given the populated site tree it orders the versions, writes
  * switcher.json + index.html, and prints the stable-alias source dir on stdout.
- * Exposed as one subcommand:
+ * Exposed as two subcommands, one at each end of the gather:
+ *
+ *   node assemble.mjs select-releases --default-branch <name> [--seed-tag <tag>]
+ *                                     [--max-releases <n>] < releases.json
+ *       → one TSV row per listed release deciding whether (and where) it lands in
+ *         the site. Runs BEFORE the gather, so publish.yml downloads only what it
+ *         will actually publish.
  *
  *   node assemble.mjs generate --site-dir <dir> --base-url <url> [--required <csv>]
  *       → write switcher.json + index.html into <dir>; print the stable-alias
@@ -17,17 +23,29 @@
  * `generate` file writes touch IO.
  */
 import { execFileSync } from "node:child_process";
-import { readdirSync, writeFileSync } from "node:fs";
+import { readdirSync, readFileSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 import { parseArgs } from "node:util";
 
 /** The `stable/` alias directory name (a fixed convention; see docs/ explanation). */
 export const STABLE_ALIAS = "stable";
 
+/** The release asset every gathered version arrives as (the docs.yml contract). */
+export const DOCS_ZIP = "docs.zip";
+
 /**
- * The in-site durable store directory (`_sources/<default>.zip`) persisted by
- * publish.yml's extract step each deploy. Published with the site but NOT a
- * version, so version discovery skips it.
+ * The sentinel tag carrying the pre-migration default-branch docs, published by
+ * scripts/migrate.sh so a repo can cut over before `docs→main` ever builds. Not a
+ * version: it stands in for the default branch, so it is exempt from `max-releases`.
+ */
+export const SEED_TAG = "pages-default-seed";
+
+/**
+ * The DEPRECATED in-site durable store (`_sources/<default>.zip`). publish.yml used
+ * to persist the default branch's docs.zip here every deploy; it now caches it
+ * instead, so newly assembled trees never contain this directory. The exclusion
+ * stays as insurance — it costs nothing, and a stray `_sources` dir must never be
+ * mistaken for a version.
  */
 export const SOURCES_DIR = "_sources";
 
@@ -91,6 +109,95 @@ export function orderVersions(builds, tags) {
 	return versions;
 }
 
+/**
+ * Newest first. Releases with no date sort last, and ties break on tag name so
+ * the selection is deterministic (two releases can share a timestamp).
+ */
+function byDateDesc(a, b) {
+	const ta = a.date ? Date.parse(a.date) : Number.NEGATIVE_INFINITY;
+	const tb = b.date ? Date.parse(b.date) : Number.NEGATIVE_INFINITY;
+	if (ta !== tb) return tb - ta;
+	return a.tag.localeCompare(b.tag);
+}
+
+/**
+ * Decide which releases contribute a version to the site, and why.
+ *
+ * Pure: takes the raw `GET /repos/{repo}/releases` array and returns one row per
+ * release in input order, so publish.yml can print a decision table that accounts
+ * for every release it listed. `dest` is the `site/<dir>` the release's docs.zip
+ * becomes, or null when skipped.
+ *
+ * Selection is by release METADATA, newest first — never by parsing the version
+ * number. Tags are wildly inconsistent across repos and a misparse silently drops
+ * a release; the API already knows the order.
+ *
+ * The key is `created_at` (the tagged commit's date), NOT `published_at`.
+ * `published_at` records when the release record was last published, so
+ * re-publishing an old release makes it look new: blueapi has `1.3.2-a9` created
+ * 2025-10-01 but published 2026-07-30, which under `published_at` outranks the
+ * genuinely newer `1.11.3`. `created_at` is immune to that.
+ *
+ * `maxReleases` caps how many releases the site publishes. It exists because the
+ * deploy uploads the WHOLE site as one artifact and GitHub Pages caps that at 1 GB:
+ * a repo with 131 released docs.zips was at 452 MB and climbing ~5 MB per release.
+ * 0 means unlimited. The seed release is never capped — it is not a version, it
+ * stands in for the default branch.
+ *
+ * @param {object[]} releases raw release objects
+ * @param {{defaultBranch: string, seedTag?: string, maxReleases?: number}} opts
+ * @returns {{tag: string, dest: string|null, date: string|null,
+ *            assetId: number|null, size: number, decision: string}[]}
+ */
+export function selectReleases(
+	releases = [],
+	{ defaultBranch, seedTag = SEED_TAG, maxReleases = 0 } = {},
+) {
+	const cap = Number(maxReleases) > 0 ? Number(maxReleases) : 0;
+
+	const rows = releases.map((release) => {
+		const tag = release?.tag_name ?? "";
+		const asset = (release?.assets ?? []).find((a) => a?.name === DOCS_ZIP);
+		const row = {
+			tag,
+			dest: null,
+			date: release?.created_at ?? release?.published_at ?? null,
+			assetId: asset?.id ?? null,
+			size: asset?.size ?? 0,
+			decision: "",
+		};
+		// Order matters: report the most fundamental reason a release is unusable.
+		// A `/`-tag is never built (the `tags: ['*']` trigger excludes it), so it has
+		// no BASE_URL-correct build and would nest a dir inside the site tree.
+		if (!tag) row.decision = "skip: release has no tag name";
+		else if (tag.includes("/")) row.decision = "skip: '/' in tag (never built)";
+		else if (tag === defaultBranch)
+			row.decision = "skip: same name as the default branch";
+		else if (!asset) row.decision = `skip: no ${DOCS_ZIP} asset`;
+		else if (tag === seedTag) {
+			row.dest = defaultBranch;
+			row.decision = `seed release → ${defaultBranch}`;
+		} else {
+			row.dest = tag;
+			row.decision = "gather";
+		}
+		return row;
+	});
+
+	if (cap > 0) {
+		// Rank only the rows that would otherwise be gathered, so a skipped release
+		// never consumes a slot. Everything past the cap is demoted in place.
+		const ranked = rows
+			.filter((row) => row.dest !== null && row.tag !== seedTag)
+			.sort(byDateDesc);
+		for (const row of ranked.slice(cap)) {
+			row.dest = null;
+			row.decision = `skip: beyond max-releases=${cap} (dated ${row.date ?? "unknown"})`;
+		}
+	}
+	return rows;
+}
+
 /**
  * Is `tag` a prerelease? Mirrors `release.yml`'s test (a PEP 440-style `a`,
  * `b`, or `rc` marker *following a digit*, e.g. `1.0a1`/`2.0rc1`) so "stable"
@@ -203,6 +310,81 @@ function csv(value) {
 		.filter(Boolean);
 }
 
+/**
+ * A TSV field that is never empty. `read` treats tab as IFS whitespace and
+ * COLLAPSES runs of it, so one empty field would shift every column after it in
+ * the consuming bash loop. "-" is the empty marker on the wire.
+ */
+function field(value) {
+	const s = value === null || value === undefined ? "" : String(value);
+	return s === "" ? "-" : s;
+}
+
+/**
+ * `select-releases --default-branch <name> [--seed-tag <tag>] [--max-releases <n>]`
+ * — read the releases JSON (as `gh api --paginate .../releases | jq -rs 'add'`
+ * gives it) on stdin, write one TSV row per release on stdout:
+ *
+ *     tag <TAB> dest <TAB> date <TAB> size <TAB> assetId <TAB> decision
+ *
+ * `dest` is "-" for a release that contributes nothing. Every listed release gets
+ * a row, so publish.yml's decision table accounts for all of them.
+ */
+function cmdSelectReleases(rest) {
+	const { values } = parseArgs({
+		args: rest,
+		options: {
+			"default-branch": { type: "string" },
+			"seed-tag": { type: "string" },
+			"max-releases": { type: "string" },
+		},
+	});
+	const defaultBranch = values["default-branch"];
+	if (!defaultBranch) {
+		throw new Error(
+			"usage: assemble.mjs select-releases --default-branch <name> [--seed-tag <tag>] [--max-releases <n>] < releases.json",
+		);
+	}
+	const raw = readFileSync(0, "utf8").trim();
+	const releases = raw ? JSON.parse(raw) : [];
+	const cap = Number(values["max-releases"] ?? 0) || 0;
+
+	const rows = selectReleases(releases, {
+		defaultBranch,
+		seedTag: values["seed-tag"] || SEED_TAG,
+		maxReleases: cap,
+	});
+	const gathered = rows.filter((row) => row.dest !== null);
+
+	console.error(
+		`${rows.length} release(s) listed, ${gathered.length} selected` +
+			(cap > 0 ? ` (max-releases=${cap})` : " (max-releases unlimited)"),
+	);
+	if (cap > 0 && rows.length - gathered.length > 0) {
+		const dropped = rows.filter((row) =>
+			row.decision.startsWith("skip: beyond"),
+		);
+		if (dropped.length > 0) {
+			console.error(
+				`${dropped.length} release(s) dropped by the cap — oldest kept: ` +
+					`${gathered.map((r) => r.tag).slice(-1)[0] ?? "(none)"}`,
+			);
+		}
+	}
+
+	for (const row of rows) {
+		const cells = [
+			field(row.tag),
+			field(row.dest),
+			field(row.date),
+			field(row.size),
+			field(row.assetId),
+			field(row.decision),
+		];
+		process.stdout.write(`${cells.join("\t")}\n`);
+	}
+}
+
 /**
  * `generate --site-dir --base-url [--required <csv>]` — write switcher.json +
  * index.html and emit the stable-alias source. Runs after all gathering, so it
@@ -273,8 +455,10 @@ export function main(argv = process.argv.slice(2)) {
 	switch (cmd) {
 		case "generate":
 			return cmdGenerate(rest);
+		case "select-releases":
+			return cmdSelectReleases(rest);
 		default:
-			throw new Error("usage: assemble.mjs generate ...");
+			throw new Error("usage: assemble.mjs generate|select-releases ...");
 	}
 }