diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 44ecbee..121af3a 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -24,7 +24,8 @@ // Allow the container to access the host X11 display and EPICS CA "--net=host", // Make sure SELinux does not disable with access to host filesystems like tmp - "--security-opt=label=disable" + "--security-opt=label=disable", + "--device=/dev/net/tun" ], "mounts": [ // Mount in the user terminal config folder so it can be edited diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml index 4bd6dfb..1503694 100644 --- a/.github/workflows/_test.yml +++ b/.github/workflows/_test.yml @@ -8,6 +8,16 @@ jobs: - uses: actions/checkout@v7 - name: Unit tests (widget logic + make-switcher parity) run: npm test + + # publish.yml's gather steps are the most intricate code in the repo and used to + # be verifiable only by merging and watching a real deploy. The harness loads them + # out of the YAML and runs them against a mock gh, so they cannot drift. + - name: Install uv + uses: astral-sh/setup-uv@v8.2.0 # full version: setup-uv uses immutable releases + with: + enable-cache: false + - name: Workflow gather-step tests + run: npm run test:workflows # release.yml downloads every artifact in the run and attaches them, so name this # explicitly (not the default "artifact") and fail loudly if the plugin is missing # — it's a release asset. diff --git a/.github/workflows/publish-dispatch.yml b/.github/workflows/publish-dispatch.yml index 8e138aa..a40c3e3 100644 --- a/.github/workflows/publish-dispatch.yml +++ b/.github/workflows/publish-dispatch.yml @@ -36,6 +36,14 @@ jobs: uses: ./.github/workflows/publish.yml with: version-name: ${{ inputs.version-name }} # "" on dispatch → pure durable gather + # Site-size policy, so it applies on EVERY path (call and dispatch alike) — hence a + # literal here rather than another input threaded down from ci.yml. + # This repo deliberately differs from the value the tutorial ships ("20"): our own + # site is ~223 MB packed across 24 releases, a fifth of the Pages cap, so capping + # would drop old versions for no gain. The capped path is covered by the unit tests + # and the workflow harness rather than by our own deploys. Set a real number here + # once a site nears the cap — see docs/how-to/keep-the-site-small.md. + max-releases: "0" pr: ${{ inputs.pr }} # set (dispatch only) → pin that fork head SHA dispatch-workflow: publish-dispatch.yml # the file the tag re-dispatch re-fires retry-until: ${{ inputs.retry-until }} # "" on workflow_call/no-retry dispatch → no auto-retry diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index e287fab..b395c8b 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -44,6 +44,16 @@ on: required: false default: "" type: string + max-releases: + description: >- + Publish only the N most recent releases (by the tagged commit's date), or 0 for + all of them. The site is deployed as ONE Pages artifact and Pages caps that at + 1 GB, so an old project with hundreds of released docs.zips will eventually + fail to deploy at all — a repo with 131 of them was already at 452 MB. Set this + in your publish-dispatch shim's `with:` block so it applies on every path. + required: false + default: "0" + type: string dispatch-workflow: description: >- Filename of the publish-dispatch shim in the consumer's repo, re-fired by @@ -101,6 +111,16 @@ jobs: REPO: ${{ github.repository }} DEFAULT: ${{ github.event.repository.default_branch }} SEED_TAG: pages-default-seed + # Site-size guard thresholds, both on the deployed artifact's scale. PROBE is the + # uncompressed size past which it is worth actually packing the tree to find out + # (the pack costs real seconds); WARN is the packed size that earns a warning, + # 700 MB against Pages' hard 1 GB. Overridable so the guard is testable and so a + # site with unusual compressibility can be tuned. + SIZE_PROBE_BYTES: "1610612736" + SIZE_WARN_BYTES: "734003200" + # DEPRECATED: the in-site durable store. Nothing writes it any more (the default + # branch's zip lives in the Actions cache); this is only still read as a one-shot + # fallback for a site last deployed by an older publish.yml. SOURCES_DIR: _sources steps: # The CONSUMER's repo, at the root cwd: assemble.mjs runs `git tag` here to @@ -155,25 +175,126 @@ jobs: echo "PAGES_URL=${url%/}" >> "$GITHUB_ENV" echo "Pages URL: ${url%/}" + # ── Which releases, and which of their zips do we already have? ───────── + # + # Selection is assemble.mjs's job (pure, unit-tested); this step only does IO. + # It runs BEFORE any download so the gather fetches only what will actually be + # published, and so the cache key describes exactly that set. + # + # Cached zips are named by ASSET ID, not by tag: an asset id identifies its bytes + # unambiguously, so a re-cut release can never be served from a stale cache entry. + - name: Select releases to publish + id: releases + env: + GH_TOKEN: ${{ github.token }} + MAX_RELEASES: ${{ inputs.max-releases }} + run: | + set -euo pipefail + gh api --paginate "repos/$REPO/releases" | jq -rs 'add // []' > "$RUNNER_TEMP/releases.json" + node "$GITHUB_WORKSPACE/.mvs/assemble/assemble.mjs" select-releases \ + --default-branch "$DEFAULT" --seed-tag "$SEED_TAG" \ + --max-releases "${MAX_RELEASES:-0}" \ + < "$RUNNER_TEMP/releases.json" > "$RUNNER_TEMP/releases.tsv" + # Key on the exact set of asset ids we intend to publish, so any change to + # that set misses and re-saves. restore-keys then falls back to the previous + # set, which shares nearly every zip — only the new release is downloaded. + digest=$(awk -F'\t' '$2 != "-" {print $5}' "$RUNNER_TEMP/releases.tsv" \ + | sort | sha256sum | cut -c1-32) + echo "cache-key=mvs-relzips-v1-$digest" >> "$GITHUB_OUTPUT" + + - name: Restore cached release zips + id: relcache + uses: actions/cache/restore@v6 + with: + path: ${{ runner.temp }}/relcache + key: ${{ steps.releases.outputs.cache-key }} + restore-keys: mvs-relzips-v1- + # Releases are permanent sources; the seed release (if present) stands in for the - # default branch until _sources/ is established — store it as $DEFAULT.zip so the - # priority-overwrite chain below can supersede it with a fresher source. + # default branch until the durable copy is established — store it as $DEFAULT.zip + # so the priority-overwrite chain below can supersede it with a fresher source. + # + # Prints a decision for every release the API listed, so a version missing from + # the site can be explained from this log alone. Downloads run in parallel: a repo + # with a hundred releases spent minutes here doing them one at a time. - name: Gather release artifacts env: GH_TOKEN: ${{ github.token }} run: | set -euo pipefail - tags=$(gh api --paginate "repos/$REPO/releases" \ - -q '.[] | select(any(.assets[]; .name=="docs.zip")) | .tag_name') - for tag in $tags; do - case "$tag" in */*) continue ;; esac - [ "$tag" = "$DEFAULT" ] && continue - [ "$tag" = "$SEED_TAG" ] && dest="$DEFAULT" || dest="$tag" - gh release download "$tag" --repo "$REPO" -p docs.zip \ - -O "$RUNNER_TEMP/gather/$dest.zip" \ - || echo "::warning::release $tag download failed — skipping" + relcache="$RUNNER_TEMP/relcache"; mkdir -p "$relcache" + todo="$RUNNER_TEMP/relcache-todo"; : > "$todo" + failed="$RUNNER_TEMP/relcache-failed"; mkdir -p "$failed" + + # Pass 1 — account for every release, and queue the cache misses. + printf '%-28s %-21s %-9s %s\n' TAG CREATED SIZE DECISION + while IFS=$'\t' read -r tag dest created size assetid decision; do + [ -n "$tag" ] || continue + if [ "$dest" != "-" ]; then + if [ -s "$relcache/$assetid.zip" ]; then + decision="$decision (cache hit)" + else + # Space-separated is safe: a git refname can never contain a space. + echo "$tag $assetid" >> "$todo" + decision="$decision (downloading)" + fi + fi + printf '%-28s %-21s %-9s %s\n' "$tag" "$created" "$size" "$decision" + # NB tab is IFS whitespace, so `read` COLLAPSES consecutive tabs — an empty + # field would shift every column after it. assemble.mjs emits "-", never "". + done < "$RUNNER_TEMP/releases.tsv" + + # Pass 2 — fetch the misses in parallel. Each worker is independent and + # records its own failure, so one bad asset cannot abort the rest. + n=$(grep -c . "$todo" || true) + echo "release zips: $(( $(awk -F'\t' '$2 != "-"' "$RUNNER_TEMP/releases.tsv" | wc -l) - n )) cached, $n to download" + if [ "$n" -gt 0 ]; then + export REPO relcache failed + # shellcheck disable=SC2016 + xargs -a "$todo" -P 8 -n 2 bash -c ' + set -uo pipefail + tag="$1"; assetid="$2" + if ! gh release download "$tag" --repo "$REPO" -p docs.zip \ + -O "$relcache/$assetid.zip" 2>"$failed/$assetid.err"; then + rm -f "$relcache/$assetid.zip" + touch "$failed/$assetid" + fi + ' _ + fi + + # Pass 3 — stage into the gather dir, in the TSV's order so the log is + # deterministic whatever order the downloads finished in. + while IFS=$'\t' read -r tag dest created size assetid decision; do + [ "$dest" = "-" ] && continue + if [ -f "$failed/$assetid" ]; then + echo "::warning::release $tag: docs.zip download failed — skipping. $(cat "$failed/$assetid.err" 2>/dev/null | head -1)" + elif [ -s "$relcache/$assetid.zip" ]; then + cp "$relcache/$assetid.zip" "$RUNNER_TEMP/gather/$dest.zip" + else + echo "::warning::release $tag: docs.zip missing after download — skipping" + fi + done < "$RUNNER_TEMP/releases.tsv" + + # Prune anything no longer selected, so a capped cache stays capped. + keep=$(awk -F'\t' '$2 != "-" {print $5 ".zip"}' "$RUNNER_TEMP/releases.tsv" | sort -u) + for f in "$relcache"/*.zip; do + [ -f "$f" ] || continue + grep -qxF "$(basename "$f")" <<<"$keep" || rm -f "$f" done + # Save only from the default branch. A cache written on a PR ref is scoped to that + # PR and never reused by anything else, so saving there would just churn the 10 GB + # repo quota. Skipped on an exact key hit — the entry is already there and cache + # entries are immutable. + - name: Save release zips to the cache + if: >- + github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && + steps.relcache.outputs.cache-hit != 'true' + uses: actions/cache/save@v6 + with: + path: ${{ runner.temp }}/relcache + key: ${{ steps.releases.outputs.cache-key }} + # Branch CI artifacts overwrite the seed (higher priority). Found via the # artifacts API by NAME (`docs` — the contract), never by workflow filename, # so the consumer's entry workflow can be called anything. An artifact's @@ -185,40 +306,95 @@ jobs: # only when the SHA carries a preview-approved status. No # $ARTIFACT_VERSION_NAME skip — the current build (staged next) overwrites # anything gathered here with the same name. + # + # The artifact list is paginated ONCE and indexed by head SHA. It used to be + # re-paginated per open PR, which on a busy repo (26 open PRs, thousands of + # artifacts) was ~175 s of identical API calls. - name: Gather branch CI artifacts env: GH_TOKEN: ${{ github.token }} run: | set -euo pipefail repo_id=$(gh api "repos/$REPO" -q .id) - gather_artifact() { - local filter="$1" dest="$2" label="$3" - local id; id=$(gh api --paginate \ - "repos/$REPO/actions/artifacts?name=docs&per_page=100" \ - | jq -rs "[.[].artifacts[] | select((.expired | not) and ($filter))] | sort_by(.created_at) | last | .id // empty") - [ -n "$id" ] || return 0 - # The API serves the artifact as a zip-of-the-artifact (docs.zip inside). - local tmp; tmp=$(mktemp -d) - { gh api "repos/$REPO/actions/artifacts/$id/zip" > "$tmp/artifact.zip" \ - && unzip -q "$tmp/artifact.zip" docs.zip -d "$tmp" \ - && mv "$tmp/docs.zip" "$RUNNER_TEMP/gather/$dest.zip"; } \ - || echo "::warning::$label artifact unavailable — skipping" - rm -rf "$tmp" + arts="$RUNNER_TEMP/artifacts.json" + gh api --paginate "repos/$REPO/actions/artifacts?name=docs&per_page=100" \ + | jq -rs '[.[].artifacts[] | select(.expired | not)]' > "$arts" + echo "$(jq length "$arts") non-expired 'docs' artifact(s) indexed" + + # Ascending by created_at so the last write per SHA is the newest artifact. + declare -A BY_SHA + while IFS=$'\t' read -r sha id created size run; do + [ -n "$sha" ] && BY_SHA["$sha"]="$id"$'\t'"$created"$'\t'"$size"$'\t'"$run" + done < <(jq -r 'sort_by(.created_at) | .[] + | [.workflow_run.head_sha, .id, .created_at, .size_in_bytes, .workflow_run.id] | @tsv' "$arts") + + # want.tsv: dest artifact-id label — resolved before any download, + # so the downloads can all run at once. + want="$RUNNER_TEMP/artifact-want.tsv"; : > "$want" + queue() { + local dest="$1" sha="$2" label="$3" meta="${BY_SHA[$2]:-}" + if [ -z "$meta" ]; then + echo " $label → no non-expired 'docs' artifact for ${sha:0:8} — skipping" + return 0 + fi + local id created size run + IFS=$'\t' read -r id created size run <<<"$meta" + echo " $label → artifact $id ($size B, built $created) from $GITHUB_SERVER_URL/$REPO/actions/runs/$run @ ${sha:0:8}" + printf '%s\t%s\t%s\n' "$dest" "$id" "$label" >> "$want" } - gather_artifact ".workflow_run.head_branch == \"$DEFAULT\" and .workflow_run.head_repository_id == $repo_id" \ - "$DEFAULT" "default-branch CI" + + echo "default branch:" + dsha=$(jq -r --arg b "$DEFAULT" --argjson r "$repo_id" ' + [.[] | select(.workflow_run.head_branch == $b and .workflow_run.head_repository_id == $r)] + | sort_by(.created_at) | last | .workflow_run.head_sha // empty' "$arts") + if [ -n "$dsha" ]; then + queue "$DEFAULT" "$dsha" "default-branch CI ($DEFAULT)" + else + echo " default-branch CI ($DEFAULT) → no non-expired 'docs' artifact — skipping" + fi + prs=$(gh pr list --repo "$REPO" --state open --limit 200 \ --json number,headRefOid,isCrossRepository -q '.[] | [.number, .headRefOid, .isCrossRepository] | @tsv') + echo "open PRs: $(grep -c . <<<"$prs" || true)" while IFS=$'\t' read -r num sha cross; do [ -z "$num" ] && continue if [ "$cross" = "true" ]; then approved=$(gh api "repos/$REPO/commits/$sha/statuses" \ -q 'any(.[]; .context=="preview-approved" and .state=="success")') - [ "$approved" = "true" ] || continue + if [ "$approved" != "true" ]; then + echo " PR #$num → fork PR, head ${sha:0:8} not preview-approved — skipping" + continue + fi fi - gather_artifact ".workflow_run.head_sha == \"$sha\"" "pr-$num" "PR #$num" + queue "pr-$num" "$sha" "PR #$num" done <<< "$prs" + # The API serves an artifact as a zip-of-the-artifact, with docs.zip inside. + # Three failure modes are reported distinctly because they need different + # fixes: download failure (transient), no docs.zip inside (an artifact built + # by a docs.yml older than the packing contract — re-run that build), and the + # not-found case already handled above. + if [ -s "$want" ]; then + export REPO + gather_dir="$RUNNER_TEMP/gather"; export gather_dir + # shellcheck disable=SC2016 + xargs -a "$want" -P 8 -d '\n' -I{} bash -c ' + set -uo pipefail + IFS=$'"'"'\t'"'"' read -r dest id label <<<"{}" + tmp=$(mktemp -d) + if ! gh api "repos/$REPO/actions/artifacts/$id/zip" > "$tmp/artifact.zip" 2>/dev/null; then + echo "::warning::$label: artifact $id download failed — skipping" + elif ! unzip -q "$tmp/artifact.zip" docs.zip -d "$tmp" >/dev/null 2>&1 || [ ! -f "$tmp/docs.zip" ]; then + # The FILE test, not unzip'"'"'s exit status: unzip exits 0 with only a + # "caution: filename not matched" note when the member is absent. + echo "::warning::$label: artifact $id contains no docs.zip — it predates the docs.zip packing contract. Re-run that build to refresh it. Skipping." + else + mv "$tmp/docs.zip" "$gather_dir/$dest.zip" + fi + rm -rf "$tmp" + ' + fi + # Highest priority: the current build overwrites anything gathered above. # Skipped on the dispatch paths (version-name is ""), which gather purely from durable sources. - name: Download this run's docs artifact @@ -237,24 +413,77 @@ jobs: [ -f "$src" ] || { echo "::error::in-run artifact not found at $src"; exit 1; } mv "$src" "$RUNNER_TEMP/gather/$ARTIFACT_VERSION_NAME.zip" - # Final fallback: if no prior step provided $DEFAULT.zip, try the durable in-site - # copy. Hard-fail if it isn't there — a deploy missing the default branch is a hole. + # The default branch's last known-good docs.zip. Releases are durable and PRs are + # ephemeral by design, but the DEFAULT BRANCH has no permanent source: it is + # gathered from its latest CI artifact, which expires, after which it would drop + # out and the guard below would hard-fail. + # + # This used to be solved by publishing the zip into the site itself + # (`_sources/.zip`) — permanent, but it shipped a multi-megabyte zip to + # every visitor's Pages artifact forever. The Actions cache holds it instead. The + # trade is real and deliberate: cache entries are evicted after 7 days without a + # read, and by LRU past 10 GB. A repo that goes quiet for over a week AND whose + # main CI artifact has expired loses this rung, where `_sources/` was forever. + # + # Key is a bare prefix so it never matches exactly; restore-keys then supplies the + # most recent entry. + - name: Restore the default branch's durable docs.zip + uses: actions/cache/restore@v6 + with: + path: ${{ runner.temp }}/defaultcache + key: mvs-default-v1- + restore-keys: mvs-default-v1- + + # Fallback chain for $DEFAULT.zip: current build → release/seed → branch CI (all + # above) → the cache → the deprecated in-site copy. Hard-fail if none of them + # produced it: a deploy missing the default branch publishes a hole. - name: Ensure default-branch zip + id: ensure-default env: GH_TOKEN: ${{ github.token }} run: | set -euo pipefail - [ -f "$RUNNER_TEMP/gather/$DEFAULT.zip" ] && exit 0 - if curl -fsSL "$PAGES_URL/$SOURCES_DIR/$DEFAULT.zip" \ - -o "$RUNNER_TEMP/gather/$DEFAULT.zip" 2>/dev/null; then - echo "::notice::$DEFAULT restored from durable in-site copy ($PAGES_URL/$SOURCES_DIR/$DEFAULT.zip)" + cache="$RUNNER_TEMP/defaultcache"; mkdir -p "$cache" + zip="$RUNNER_TEMP/gather/$DEFAULT.zip" + if [ -f "$zip" ]; then + echo "$DEFAULT supplied by an earlier rung ($(stat -c%s "$zip") B)" + elif [ -s "$cache/default.zip" ]; then + cp "$cache/default.zip" "$zip" + echo "::notice::$DEFAULT restored from the Actions cache ($(stat -c%s "$zip") B) \ + — its CI artifact has expired, so this deploy is serving the last cached build" + # DEPRECATED rung, kept so a site deployed by an older publish.yml still has a + # source on its first upgraded deploy. Nothing writes _sources/ any more, so it + # disappears from the site on that very deploy; this read is one-shot insurance. + elif curl -fsSL "$PAGES_URL/$SOURCES_DIR/$DEFAULT.zip" -o "$zip" 2>/dev/null; then + echo "::notice::$DEFAULT restored from the deprecated in-site copy \ + ($PAGES_URL/$SOURCES_DIR/$DEFAULT.zip) — it will be cached from here on" else - echo "::error::$DEFAULT: not provided by current build, releases, branch CI, or durable in-site copy — deploy aborted" + echo "::error::$DEFAULT: not provided by current build, releases, branch CI, \ + the Actions cache, or the deprecated in-site copy — deploy aborted" exit 1 fi + # Re-save only when the bytes actually changed: cache entries are immutable, so + # a content-addressed key means an unchanged default branch adds no entry. + new=$(sha256sum "$zip" | cut -c1-32) + old=$(cat "$cache/default.sha" 2>/dev/null || true) + if [ "$new" != "$old" ]; then + cp "$zip" "$cache/default.zip" + printf '%s' "$new" > "$cache/default.sha" + echo "cache-key=mvs-default-v1-$new" >> "$GITHUB_OUTPUT" + fi + + # Same rule as the release-zip cache: only the default branch writes, because a + # cache saved on a PR ref is scoped to that PR and no other run can read it. + - name: Save the default branch's durable docs.zip + if: >- + steps.ensure-default.outputs.cache-key != '' && + github.ref == format('refs/heads/{0}', github.event.repository.default_branch) + uses: actions/cache/save@v6 + with: + path: ${{ runner.temp }}/defaultcache + key: ${{ steps.ensure-default.outputs.cache-key }} # Unzip every gathered zip into SITE/. - # Persist the default-branch zip into SITE/_sources/ for future recovery. # # THE ZIP-SHAPE CONTRACT: a docs.zip must unzip to exactly ONE top-level # directory, whose CONTENTS become site/. The directory's NAME is @@ -288,10 +517,34 @@ jobs: fi rm -rf "$tmp" done - dzip="$RUNNER_TEMP/gather/$DEFAULT.zip" - [ -d "$RUNNER_TEMP/site/$DEFAULT" ] && [ -f "$dzip" ] \ - && { mkdir -p "$RUNNER_TEMP/site/$SOURCES_DIR"; cp "$dzip" "$RUNNER_TEMP/site/$SOURCES_DIR/$DEFAULT.zip"; } \ - || true + # upload-pages-artifact tars the whole site and GitHub Pages rejects the + # artifact over 1 GB. That ceiling arrives silently — deploys just start + # failing — so warn while there is still room to act. `max-releases` is the + # lever: it is unlimited by default, which is safe for a young project and a + # slow-motion footgun for an old one. + # + # The cap is on the COMPRESSED artifact, which is nothing like the size of the + # tree: this repo's own site is 734 MiB on disk but packs to 223 MiB, because + # HTML and JS compress ~3x. Comparing `du` against 1 GB would cry wolf at a + # third of the real usage. So measure the tree cheaply, and only pay for an + # actual pack when the tree is big enough for the answer to matter. + # + # -h matches upload-pages-artifact's --dereference: the `stable/` symlink is + # inflated to a full copy of a release in the real artifact, so a measurement + # that stored it as a symlink would under-read by a whole version. + bytes=$(du -sb "$RUNNER_TEMP/site" | cut -f1) + echo "assembled site: $(numfmt --to=iec --suffix=B "$bytes") uncompressed across \ + $(find "$RUNNER_TEMP/site" -mindepth 1 -maxdepth 1 -type d | wc -l) version dir(s)" + if [ "$bytes" -gt "$SIZE_PROBE_BYTES" ]; then + packed=$(tar -ch -C "$RUNNER_TEMP/site" . | gzip -1 -c | wc -c) + echo "packed artifact: ~$(numfmt --to=iec --suffix=B "$packed") (Pages rejects over 1 GB)" + if [ "$packed" -gt "$SIZE_WARN_BYTES" ]; then + echo "::warning title=Docs site approaching the Pages 1 GB limit::The \ + deployed artifact is ~$(numfmt --to=iec --suffix=B "$packed"), against a hard \ + 1 GB cap. Set publish.yml's \`max-releases\` input (in your publish-dispatch \ + shim) to publish only the N most recent releases." + fi + fi # `generate` orders versions, writes switcher.json + index.html, and prints the # stable-alias source dir (the newest release), or "" when no release is deployed yet. diff --git a/.gitignore b/.gitignore index 805bc35..d7f23b7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ node_modules/ docs/_build/ *.log +__pycache__/ diff --git a/CLAUDE.md b/CLAUDE.md index 4e7aea0..d65c0d5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -12,7 +12,7 @@ to `publish.yml`, not a separately consumed action. plugins/version-switcher.mjs # MyST directive + anywidget runtime (single file, no README — docs are in docs/) assemble/assemble.mjs # INTERNAL: dependency-free Node kernel for publish.yml's "Generate" step (the `generate` subcommand writes switcher.json + index.html) scripts/migrate.sh # operator gh-pages → durable-source migration (bash); two-phase: reversible cutover, then guarded --delete-gh-pages -test/ # npm test suite (node, no framework) +test/ # npm test suite (node, no framework) + test/workflow-harness/ (python: loads publish.yml's gather steps out of the YAML and runs them against a mock gh) docs/ # this repo's own docs (dogfoods the plugin) .github/workflows/docs.yml # PUBLIC reusable: build at the versioned BASE_URL → pack docs.zip → upload `docs` artifact (workflow_call; build-command input) .github/workflows/publish.yml # PUBLIC reusable ENGINE (PRIVILEGED): branches on the event into 3 jobs — deploy (gather+extract inline bash + assemble.mjs@job.workflow_sha + Pages), tag-re-dispatch (re-fire the shim), fork-warn. workflow_call ONLY; reached via the publish-dispatch.yml shim @@ -39,23 +39,65 @@ public action. See [`docs/explanations/architecture.md`](docs/explanations/architecture.md) for the full rationale. In short: -### The default branch is self-durable in the site (`_sources/`) +### The default branch's durable copy lives in the Actions cache Releases are durable (Release `docs.zip` assets) and PRs are ephemeral by design, but -the **default branch** had no permanent source — gathered from its latest CI artifact, -which expires, after which it drops out and the guard hard-fails. So each deploy -persists the default branch's `docs.zip` (the one it arrived as — current build, -gathered run, or the fallback — copied verbatim) into the published site at -`_sources/.zip` (excluded from version discovery), and a deploy whose fresh -artifact is gone restores the branch from that durable in-site copy (fetched from -`$PAGES_URL` — the live Pages URL resolved from the Pages API, so custom domains -work, falling back to `https://.github.io/`; the same URL roots the -`switcher.json` entries). Before the branch ever builds -docs (mid gh-pages migration) a final rung reads a **published seed release** -(`pages-default-seed`, created by `scripts/migrate.sh` from the old gh-pages `/` -tree) and persists it to `_sources` on that deploy — so a repo can cut over to the -reusable workflow in one PR, before `docs→main`. Drafts can't be used: a `contents:read` -deploy token can't read them (verified), so the seed is a published release on a -sentinel tag, deleted at finalize once `_sources/.zip` is live. +the **default branch** has no permanent source — gathered from its latest CI artifact, +which expires, after which it drops out and the guard hard-fails. So each +default-branch deploy keeps a copy of its `docs.zip` in the Actions cache +(content-addressed key `mvs-default-v1-`, so an unchanged branch adds no entry), +and a deploy whose fresh artifact is gone restores from there. + +This used to be published *into the site* at `_sources/.zip`. That was +permanent, but it shipped a multi-megabyte zip inside every Pages artifact forever on a +site already pressing the 1 GB cap (see "The site has a hard size ceiling"). The cache +is evicted after 7 days unread and by LRU past 10 GB, so a repo quiet for >1 week whose +main artifact ALSO expired loses the rung and hard-fails loudly — accepted trade. +`_sources` is no longer written; the READ survives as a deprecated one-shot rung so a +site last deployed by an older `publish.yml` still has a source on its first upgraded +deploy (fetched from `$PAGES_URL` — the live Pages URL from the Pages API, so custom +domains work; the same URL roots the `switcher.json` entries). + +Before the branch ever builds docs (mid gh-pages migration) a final rung reads a +**published seed release** (`pages-default-seed`, created by `scripts/migrate.sh` from +the old gh-pages `/` tree) — so a repo can cut over to the reusable workflow in +one PR, before `docs→main`. Drafts can't be used: a `contents:read` deploy token can't +read them (verified), so the seed is a published release on a sentinel tag, deleted at +finalize. `migrate.sh --delete-gh-pages`'s guard now requires the default branch to be +live in the new site AND to have a non-expired `docs` artifact of its own (it used to +probe `_sources/.zip`, which passed even while the content came from the seed +— i.e. while gh-pages was still the only real copy). + +### The site has a hard size ceiling — cap the releases +`upload-pages-artifact` tars the WHOLE site into one artifact and Pages rejects it over +**1 GB**. blueapi at 131 released `docs.zip`s was at 452 MB, +~5 MB/release: a few years +from deploys simply failing, silently, with no warning on the way. `publish.yml` takes +**`max-releases`** (default `0` = unlimited, so upgrading never silently deletes +versions); the tutorial's shim ships `20`. Set it as a LITERAL in the shim's `with:`, +not threaded from `ci.yml` — the shim is the single caller on every path, so a literal +there can't be bypassed by a manual dispatch. A deploy past 700 MB warns — measured on +the PACKED artifact (the cap is on the gzipped tar; this repo's site is 734 MiB on disk +and 223 MiB deployed, so `du` would cry wolf at a third of real usage). The tree is only +packed to measure once it exceeds `SIZE_PROBE_BYTES`, since packing costs seconds; `tar +-ch` matches upload-pages-artifact's `--dereference`, which inflates `stable/` into a +full copy. + +Selection is `assemble.mjs`'s `selectReleases` (pure, unit-tested; it replaced an +untested bash `case`), ranked by **`created_at`** — NOT `published_at`, which lies when +an old release is re-published (blueapi's `1.3.2-a9`: created 2025-10, published +2026-07, and under `published_at` it outranked the newer `1.11.3`). No version-number +parsing: tags are too inconsistent across repos. The seed release is exempt from the cap. +`getSortedTags()`/`orderVersions()` are untouched — that's switcher DISPLAY order, a +separate concern. + +### The gather is cached, indexed and parallel +The deploy's cost is O(whole site), not O(the change) — identical work whatever fired +it. On blueapi that was 654 s against a 27–45 s docs build. Fixed by: an `actions/cache` +entry of release zips keyed on the exact set of asset ids to publish (assets are +immutable; files named by ASSET ID so a re-cut release can't serve stale bytes; pruned to +the published set), the artifacts API paginated ONCE and indexed newest-per-head-SHA +(it was re-paginated per open PR — 27× for blueapi, ~175 s), and parallel downloads +(`xargs -P 8`). **Caches are saved only on the default branch**: a cache written on a PR +ref is scoped to that PR and unreadable elsewhere. ### Reconstruct from durable sources, publish the whole tree Every deploy rebuilds the **complete** site from authoritative inputs — `main`'s @@ -224,7 +266,8 @@ always, fork PRs only when the SHA carries that status. ## Developing ```bash -npm test # run the test suite +npm test # run the unit tests +npm run test:workflows # run publish.yml's gather steps against a mock gh (needs uv) npm run docs # build docs (same command CI uses) npm run docs-dev # live-preview docs with the plugin loaded from local plugins/ ``` diff --git a/assemble/assemble.mjs b/assemble/assemble.mjs index c204080..b5dc9f2 100644 --- a/assemble/assemble.mjs +++ b/assemble/assemble.mjs @@ -5,7 +5,13 @@ * dir, then the extract step unzips them all into the site tree. This file is * the final step: given the populated site tree it orders the versions, writes * switcher.json + index.html, and prints the stable-alias source dir on stdout. - * Exposed as one subcommand: + * Exposed as two subcommands, one at each end of the gather: + * + * node assemble.mjs select-releases --default-branch [--seed-tag ] + * [--max-releases ] < releases.json + * → one TSV row per listed release deciding whether (and where) it lands in + * the site. Runs BEFORE the gather, so publish.yml downloads only what it + * will actually publish. * * node assemble.mjs generate --site-dir --base-url [--required ] * → write switcher.json + index.html into ; print the stable-alias @@ -17,17 +23,29 @@ * `generate` file writes touch IO. */ import { execFileSync } from "node:child_process"; -import { readdirSync, writeFileSync } from "node:fs"; +import { readdirSync, readFileSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { parseArgs } from "node:util"; /** The `stable/` alias directory name (a fixed convention; see docs/ explanation). */ export const STABLE_ALIAS = "stable"; +/** The release asset every gathered version arrives as (the docs.yml contract). */ +export const DOCS_ZIP = "docs.zip"; + /** - * The in-site durable store directory (`_sources/.zip`) persisted by - * publish.yml's extract step each deploy. Published with the site but NOT a - * version, so version discovery skips it. + * The sentinel tag carrying the pre-migration default-branch docs, published by + * scripts/migrate.sh so a repo can cut over before `docs→main` ever builds. Not a + * version: it stands in for the default branch, so it is exempt from `max-releases`. + */ +export const SEED_TAG = "pages-default-seed"; + +/** + * The DEPRECATED in-site durable store (`_sources/.zip`). publish.yml used + * to persist the default branch's docs.zip here every deploy; it now caches it + * instead, so newly assembled trees never contain this directory. The exclusion + * stays as insurance — it costs nothing, and a stray `_sources` dir must never be + * mistaken for a version. */ export const SOURCES_DIR = "_sources"; @@ -91,6 +109,95 @@ export function orderVersions(builds, tags) { return versions; } +/** + * Newest first. Releases with no date sort last, and ties break on tag name so + * the selection is deterministic (two releases can share a timestamp). + */ +function byDateDesc(a, b) { + const ta = a.date ? Date.parse(a.date) : Number.NEGATIVE_INFINITY; + const tb = b.date ? Date.parse(b.date) : Number.NEGATIVE_INFINITY; + if (ta !== tb) return tb - ta; + return a.tag.localeCompare(b.tag); +} + +/** + * Decide which releases contribute a version to the site, and why. + * + * Pure: takes the raw `GET /repos/{repo}/releases` array and returns one row per + * release in input order, so publish.yml can print a decision table that accounts + * for every release it listed. `dest` is the `site/` the release's docs.zip + * becomes, or null when skipped. + * + * Selection is by release METADATA, newest first — never by parsing the version + * number. Tags are wildly inconsistent across repos and a misparse silently drops + * a release; the API already knows the order. + * + * The key is `created_at` (the tagged commit's date), NOT `published_at`. + * `published_at` records when the release record was last published, so + * re-publishing an old release makes it look new: blueapi has `1.3.2-a9` created + * 2025-10-01 but published 2026-07-30, which under `published_at` outranks the + * genuinely newer `1.11.3`. `created_at` is immune to that. + * + * `maxReleases` caps how many releases the site publishes. It exists because the + * deploy uploads the WHOLE site as one artifact and GitHub Pages caps that at 1 GB: + * a repo with 131 released docs.zips was at 452 MB and climbing ~5 MB per release. + * 0 means unlimited. The seed release is never capped — it is not a version, it + * stands in for the default branch. + * + * @param {object[]} releases raw release objects + * @param {{defaultBranch: string, seedTag?: string, maxReleases?: number}} opts + * @returns {{tag: string, dest: string|null, date: string|null, + * assetId: number|null, size: number, decision: string}[]} + */ +export function selectReleases( + releases = [], + { defaultBranch, seedTag = SEED_TAG, maxReleases = 0 } = {}, +) { + const cap = Number(maxReleases) > 0 ? Number(maxReleases) : 0; + + const rows = releases.map((release) => { + const tag = release?.tag_name ?? ""; + const asset = (release?.assets ?? []).find((a) => a?.name === DOCS_ZIP); + const row = { + tag, + dest: null, + date: release?.created_at ?? release?.published_at ?? null, + assetId: asset?.id ?? null, + size: asset?.size ?? 0, + decision: "", + }; + // Order matters: report the most fundamental reason a release is unusable. + // A `/`-tag is never built (the `tags: ['*']` trigger excludes it), so it has + // no BASE_URL-correct build and would nest a dir inside the site tree. + if (!tag) row.decision = "skip: release has no tag name"; + else if (tag.includes("/")) row.decision = "skip: '/' in tag (never built)"; + else if (tag === defaultBranch) + row.decision = "skip: same name as the default branch"; + else if (!asset) row.decision = `skip: no ${DOCS_ZIP} asset`; + else if (tag === seedTag) { + row.dest = defaultBranch; + row.decision = `seed release → ${defaultBranch}`; + } else { + row.dest = tag; + row.decision = "gather"; + } + return row; + }); + + if (cap > 0) { + // Rank only the rows that would otherwise be gathered, so a skipped release + // never consumes a slot. Everything past the cap is demoted in place. + const ranked = rows + .filter((row) => row.dest !== null && row.tag !== seedTag) + .sort(byDateDesc); + for (const row of ranked.slice(cap)) { + row.dest = null; + row.decision = `skip: beyond max-releases=${cap} (dated ${row.date ?? "unknown"})`; + } + } + return rows; +} + /** * Is `tag` a prerelease? Mirrors `release.yml`'s test (a PEP 440-style `a`, * `b`, or `rc` marker *following a digit*, e.g. `1.0a1`/`2.0rc1`) so "stable" @@ -203,6 +310,81 @@ function csv(value) { .filter(Boolean); } +/** + * A TSV field that is never empty. `read` treats tab as IFS whitespace and + * COLLAPSES runs of it, so one empty field would shift every column after it in + * the consuming bash loop. "-" is the empty marker on the wire. + */ +function field(value) { + const s = value === null || value === undefined ? "" : String(value); + return s === "" ? "-" : s; +} + +/** + * `select-releases --default-branch [--seed-tag ] [--max-releases ]` + * — read the releases JSON (as `gh api --paginate .../releases | jq -rs 'add'` + * gives it) on stdin, write one TSV row per release on stdout: + * + * tag dest date size assetId decision + * + * `dest` is "-" for a release that contributes nothing. Every listed release gets + * a row, so publish.yml's decision table accounts for all of them. + */ +function cmdSelectReleases(rest) { + const { values } = parseArgs({ + args: rest, + options: { + "default-branch": { type: "string" }, + "seed-tag": { type: "string" }, + "max-releases": { type: "string" }, + }, + }); + const defaultBranch = values["default-branch"]; + if (!defaultBranch) { + throw new Error( + "usage: assemble.mjs select-releases --default-branch [--seed-tag ] [--max-releases ] < releases.json", + ); + } + const raw = readFileSync(0, "utf8").trim(); + const releases = raw ? JSON.parse(raw) : []; + const cap = Number(values["max-releases"] ?? 0) || 0; + + const rows = selectReleases(releases, { + defaultBranch, + seedTag: values["seed-tag"] || SEED_TAG, + maxReleases: cap, + }); + const gathered = rows.filter((row) => row.dest !== null); + + console.error( + `${rows.length} release(s) listed, ${gathered.length} selected` + + (cap > 0 ? ` (max-releases=${cap})` : " (max-releases unlimited)"), + ); + if (cap > 0 && rows.length - gathered.length > 0) { + const dropped = rows.filter((row) => + row.decision.startsWith("skip: beyond"), + ); + if (dropped.length > 0) { + console.error( + `${dropped.length} release(s) dropped by the cap — oldest kept: ` + + `${gathered.map((r) => r.tag).slice(-1)[0] ?? "(none)"}`, + ); + } + } + + for (const row of rows) { + const cells = [ + field(row.tag), + field(row.dest), + field(row.date), + field(row.size), + field(row.assetId), + field(row.decision), + ]; + process.stdout.write(`${cells.join("\t")}\n`); + } +} + /** * `generate --site-dir --base-url [--required ]` — write switcher.json + * index.html and emit the stable-alias source. Runs after all gathering, so it @@ -273,8 +455,10 @@ export function main(argv = process.argv.slice(2)) { switch (cmd) { case "generate": return cmdGenerate(rest); + case "select-releases": + return cmdSelectReleases(rest); default: - throw new Error("usage: assemble.mjs generate ..."); + throw new Error("usage: assemble.mjs generate|select-releases ..."); } } diff --git a/blueapi-issue.md b/blueapi-issue.md new file mode 100644 index 0000000..c7ac8d2 --- /dev/null +++ b/blueapi-issue.md @@ -0,0 +1,147 @@ +# Docs site: slow CI, permanently one release behind, and heading for a hard wall + +Three separate problems in blueapi's Pages setup, with one shared root cause. Two are +fixed upstream in +[myst-version-switcher-plugin](https://github.com/DiamondLightSource/myst-version-switcher-plugin) +and need a pin bump plus one config line here; one is still open. + +Measured on blueapi's own runs, 2026-08-21. + +--- + +## 1. The publish job was slow, which is why it got switched off + +[PR #1614](https://github.com/DiamondLightSource/blueapi/pull/1614) narrowed `publish` +to tags only, one day after adoption, because it was adding many minutes to every PR. +That was the right call on the evidence. The evidence: + +| step, PR run [31081562099](https://github.com/DiamondLightSource/blueapi/actions/runs/31081562099) | time | +|---|---| +| `Gather release artifacts` — 188 releases listed, 131 `docs.zip` downloaded **one at a time** | 114 s | +| `Gather branch CI artifacts` — 26 open PRs, one **full artifacts-API pagination each** | 175 s | +| `Upload Pages artifact` — 452,549,958 bytes | 36 s | +| `Deploy to GitHub Pages` | 280 s | +| **deploy job total** | **654 s** | + +The docs *build* is 27–45 s. Everything else in CI finishes by ~200 s. Publish alone took +the run from ~200 s to ~750 s. + +The cause is that the deploy's cost is **O(whole site)**, not O(what changed): every +event re-downloaded ~453 MB of immutable release assets and re-uploaded a 452 MB +artifact, doing identical work regardless of which PR triggered it. + +**Fixed upstream.** Release assets are cached (`actions/cache`, keyed on the exact set of +asset ids, so a new release downloads exactly one zip); the artifacts API is paginated +once and indexed by head SHA instead of once per open PR; downloads run in parallel. The +deploy should land near ~100 s, which starts after `docs` (~50 s) and finishes inside +your 195 s `system-test` — off the critical path in wall-clock terms. + +## 2. The site is permanently one release behind — **still open** + +`https://diamondlightsource.github.io/blueapi/` is missing 1.18.1 right now: the switcher +lists 1.18.0 as `stable` and `/1.18.1/` 404s. This is not a one-off — **every** release +has been missing itself, appearing only when the *next* release is cut. + +### Root cause 2a — the tag re-dispatch races the `docs.zip` upload + +`publish.yml`'s `re-dispatch` job waits for the release **record**, but the deploy's +gather selects releases by their **`docs.zip` asset**. blueapi publishes releases from +the **GitHub UI**, so the record exists *before* the tag push even starts CI — the wait +is a no-op. Timeline for 1.18.1: + +| time (UTC) | event | +|---|---| +| 09:34:14 | release 1.18.1 published (UI) | +| 09:34:16 | tag CI run [32122292712](https://github.com/DiamondLightSource/blueapi/actions/runs/32122292712) starts | +| 09:34:50 | `re-dispatch` sees the release, dispatches immediately | +| 09:35:02 | dispatched deploy [32122349268](https://github.com/DiamondLightSource/blueapi/actions/runs/32122349268) lists releases — **1.18.1 has no assets yet** | +| 09:35:57 | `release / release` attaches `docs.zip` — 55 s too late | + +That run logged `Sorted versions: ["main","1.18.0",…]` and `Stable alias source: 1.18.0`. +1.18.0 missed itself the same way and only landed 11 days later, carried in by 1.18.1's +deploy. + +**Status: not yet fixed.** The obvious fix (wait for the asset, not the record) is +written and working, but it is on hold behind a larger question — whether publishing +should move off the inline CI path onto a `workflow_run` trigger, which would make this +race disappear entirely (`workflow_run` fires after the tag's CI run, `release.yml` +included, so the asset is already attached). That is being settled by a live experiment +upstream. Either way blueapi gets a fix; this note is so nobody re-debugs it meanwhile. + +### Root cause 2b — nothing ever self-heals + +```yaml + publish: + if: github.repository == 'DiamondLightSource/blueapi' && github.ref_type == 'tag' +``` + +Every PR and every push to `main` logs `publish → skipped`, so the *only* deploys are tag +re-dispatches — the exact path broken by 2a. A deploy that misses a source stays missed +until the next tag. + +**Suggested change**, keeping #1614's intent (no deploy on every PR push) while restoring +self-heal on every merge: + +```yaml + publish: + needs: [docs] + # don't publish a pages site in the fork's org + if: >- + github.repository == 'DiamondLightSource/blueapi' && + (github.ref_type == 'tag' || github.ref == 'refs/heads/main') +``` + +PR previews still work: **any** deploy gathers every open PR's build artifact, so PR docs +still appear at `/pr-/`, just refreshed on the next `main` deploy rather than on every +push. With problem 1 fixed you may not need to narrow it at all. + +## 3. The site is at 45% of a hard limit nobody warns you about + +`upload-pages-artifact` tars the **whole** site into one artifact, and GitHub Pages +rejects that artifact over **1 GB**. blueapi's last deploy uploaded **452,549,958 bytes** +across 139 versions, growing ~5 MB per release. That is roughly 110 more releases — +a couple of years — until deploys simply start failing, with no signal on the way. + +**Fixed upstream** by a new `max-releases` input. It defaults to `0` (unlimited) so +upgrading changes nothing on its own, and any deploy past 700 MB now warns. + +To opt in, set it in `.github/workflows/publish-dispatch.yml`, in the `with:` block — +**not** as an input threaded from `ci.yml`, so it applies on every path (inline publish, +tag re-dispatch, fork preview, manual re-deploy): + +```yaml +jobs: + publish: + uses: DiamondLightSource/myst-version-switcher-plugin/.github/workflows/publish.yml@ + with: + version-name: ${{ inputs.version-name }} + max-releases: "20" # ← add this + pr: ${{ inputs.pr }} + dispatch-workflow: publish-dispatch.yml + retry-until: ${{ inputs.retry-until }} +``` + +For blueapi that takes the released half from **453 MB to 73 MB**. Releases are ranked by +the tagged commit's date, and nothing is deleted — every release keeps its `docs.zip` +asset, so raising the cap brings the old versions straight back on the next deploy. + +Worth knowing when picking N: prereleases occupy slots. Your newest 20 releases include +`1.11.5-a1/a2/a3`, `1.11.1-a3` and `1.14.1-dev.1`, so `max-releases: 20` publishes 15 +real versions. + +--- + +## Actions for blueapi + +1. **Bump the pins** to the new tag once it is cut — currently `@v0.24` in three places: + `docs.yml` and `release.yml` in `ci.yml`, and `publish.yml` in `publish-dispatch.yml`. +2. **Restore self-heal**: the `ci.yml` `if:` change in 2b. +3. **Set `max-releases`** in `publish-dispatch.yml` (suggest `"20"`). +4. **Backfill 1.18.1 now** — its asset exists, so a manual re-deploy picks it up: + ```bash + gh workflow run publish-dispatch.yml --repo DiamondLightSource/blueapi + ``` +5. **Optional cleanup**: six PRs (#1246, #1504, #1512, #1552, #1553, #1561) have `docs` + artifacts predating the `docs.zip` packing contract, so they log + `artifact unavailable — skipping`. Harmless; re-running CI on those PRs clears it. The + upstream warning is now specific about this case rather than a catch-all. diff --git a/docs/explanations/architecture.md b/docs/explanations/architecture.md index 4a0e9f5..c29a25d 100644 --- a/docs/explanations/architecture.md +++ b/docs/explanations/architecture.md @@ -16,15 +16,49 @@ kinds and their durability are in the Releases are permanent, so old versions never vanish. PR previews come from CI artifacts and silently drop if the artifact expires and nothing rebuilds — fine for -*optional* preview docs. The **default branch** used to be the one fragile required -version (artifact-only, yet guarded — `assemble` hard-fails rather than publish a -site missing it). It is now self-durable: once it has built docs at least once, each -deploy persists its `docs.zip` into the published site at `_sources/.zip`, and -a deploy whose fresh artifact has expired restores the branch from that in-site copy — -removing the artifact-expiry hole. (A gh-pages migration still keeps its old `/main/` -content only until the default branch itself builds docs under the new pipeline; from -that point the in-site copy is its durable source — see -[migrate-from-gh-pages](../how-to/migrate-from-gh-pages.md).) +*optional* preview docs. The **default branch** is the one fragile *required* version +(artifact-only, yet guarded — `assemble` hard-fails rather than publish a site missing +it), so each deploy keeps a copy of its `docs.zip` in the **Actions cache**; a deploy +whose fresh artifact has expired restores the branch from there. + +That copy used to be published into the site itself, at `_sources/.zip`. That +was permanent, which the cache is not — but it also shipped a multi-megabyte zip inside +every Pages artifact, forever, on a site already pressing against a hard 1 GB ceiling +(see [below](#why-the-site-has-a-size-limit)). The cache is evicted after 7 days without +a read, and by LRU past 10 GB, so a repo that goes quiet for over a week *and* whose +default-branch CI artifact has also expired loses the rung and the deploy hard-fails — +loudly, and fixed by one push. That is the accepted trade. (A gh-pages migration keeps +its old `/main/` content only until the default branch builds docs under the new +pipeline — see [migrate-from-gh-pages](../how-to/migrate-from-gh-pages.md).) + +(why-the-site-has-a-size-limit)= +### Why the site has a size limit + +Reconstructing everything has one cost that grows without bound: `upload-pages-artifact` +tars the **whole** site into a single artifact, and GitHub Pages rejects that artifact +over **1 GB**. A long-lived project reaches it. blueapi, at 131 released `docs.zip`s, was +already at 452 MB and adding ~5 MB per release — a few years from deploys simply failing, +with nothing on the way to say so. + +Two things follow. + +`publish.yml` takes a **`max-releases`** input: publish only the N most recent releases, +ranked by the tagged commit's date. It defaults to `0` (unlimited), so upgrading never +silently deletes versions from an existing site — but the paved path in the +[tutorial](../tutorials/adding-to-a-fresh-repo.md) sets it, and a deploy whose *packed* +artifact passes 700 MB says so in a warning. Packed, not the tree on disk: HTML and JS +compress around 3×, so measuring the directory would cry wolf at a third of real usage. Older releases keep their `docs.zip` assets and +come back the moment you raise the cap. See +[keep-the-site-small](../how-to/keep-the-site-small.md). + +Ranking is on the release's `created_at`, never on parsing the version number: tags are +too inconsistent across repos for a parser to be safe, and `published_at` lies whenever an +old release is re-published (blueapi has a `1.3.2-a9` created in 2025 and published in +2026, which under `published_at` outranks the genuinely newer `1.11.3`). + +The same pressure is why the default branch's durable copy moved out of the site and into +the Actions cache, and why the gather caches release assets rather than re-downloading +~450 MB of immutable zips on every event. ### Why this replaced the `gh-pages` + `keep_files` model @@ -58,9 +92,8 @@ ordering: supersedes it ([community discussion #158055](https://github.com/orgs/community/discussions/158055)) — so the source can be flipped up front, with no downtime and no blank window. -- **A publish replaces the *whole* site, so the default branch must be durable before - any publish runs.** A publish that runs before `_sources/.zip` exists would - drop `//`. The migration therefore *seeds* the default branch (a published +- **A publish replaces the *whole* site, so the default branch must have a source before + any publish runs.** A publish with nothing to stage at `//` would drop it. The migration therefore *seeds* the default branch (a published `pages-default-seed` release captured from the old gh-pages tree) before the first publish — which is the pipeline PR's own CI. This makes that first publish safe even when the repo already serves Pages from Actions (where a publish deploys live @@ -241,9 +274,9 @@ The `stable` segment name is a fixed convention, hardcoded in the widget. asset packed by another pipeline (python-copier-template's `_release.yml` roots its zip at the tag name, not `html/`) still deploys. A zip with *no* single root directory is malformed → warning, skipped. -- **Default branch missing**: if `main` has no recent successful CI artifact, no durable - `_sources/main.zip` in the live site, and no migration seed release, the deploy - **hard-fails** rather than publish a site missing it. +- **Default branch missing**: if `main` has no recent successful CI artifact, no cached + copy, and no migration seed release, the deploy **hard-fails** rather than publish a + site missing it. - **PR build not yet green / SHA moved:** an open PR whose current head SHA has no successful CI run is skipped; its preview appears once the build passes. - **Merged/closed PR:** drops from the gather (open-PRs only) on the next deploy. diff --git a/docs/how-to/choose-publish-events.md b/docs/how-to/choose-publish-events.md new file mode 100644 index 0000000..780217d --- /dev/null +++ b/docs/how-to/choose-publish-events.md @@ -0,0 +1,72 @@ +# How-to: choose which events publish the site + +The [tutorial](../tutorials/adding-to-a-fresh-repo.md)'s `publish` job runs on **every** +event and lets `publish.yml` do the branching: + +```yaml + publish: + needs: [docs] + if: github.repository == 'ORG/REPO' + uses: ./.github/workflows/publish-dispatch.yml +``` + +Every deploy rebuilds the *whole* site from durable sources, so this is also what makes +the site **self-healing**: any deploy that came out wrong is corrected by the next one, +whatever triggered it. Narrowing the gate trades that away, so narrow it deliberately. + +## If per-PR deploys are too noisy + +Deploys serialise on the `pages` concurrency group, so a repo with a lot of PR churn may +not want one per push. + +Check the cost before narrowing, though. A deploy that reconstructs a large site used to +take many minutes, and most of that is gone: release assets are cached rather than +re-downloaded, the artifacts API is paginated once instead of once per open PR, and +downloads run in parallel. If it still takes minutes, the site itself is probably too +big — see [keep-the-site-small](keep-the-site-small.md) before trading away +self-healing. + +To narrow it anyway, publish on **tags and the default branch**: + +```yaml + publish: + needs: [docs] + if: >- + github.repository == 'ORG/REPO' && + (github.ref_type == 'tag' || github.ref == 'refs/heads/main') +``` + +You keep PR previews — **any** deploy gathers every open PR's build artifact, so +`/pr-/` still appears; it just refreshes on the next merge to the default branch +rather than on every push to the PR. And you keep self-healing, at merge cadence. + +:::{warning} +**Do not narrow it to tags alone.** With + +```yaml + if: github.repository == 'ORG/REPO' && github.ref_type == 'tag' # ← don't +``` + +the *only* deploys are release deploys, so nothing corrects a bad one until the next +release. A single missed source then persists for weeks: a real repo that did this +shipped a site that was permanently one release behind, because each release's deploy +was the only chance that release had to reach the site. +::: + +## What you cannot narrow away + +The `publish` job must still be reached on a **tag** push, even if you only want release +deploys. That is the event that triggers `publish.yml`'s `re-dispatch` job — the +`workflow_dispatch` bounce that is the only way a release's same-SHA deploy reaches the +Pages origin at all (see [the architecture explanation](../explanations/architecture.md)). + +Fork PRs are already handled without a gate: they reach the read-only `warn` job, never a +deploy. + +## Checking what your gate actually does + +On any run, `publish` showing as `skipped` in the job list means no deploy happened for +that event. When one does run, its `Gather release artifacts` step prints a decision for +every release the API listed — gathered, skipped and why — and the extract step reports +the assembled size and version count, so you can see whether a version made it in and +what kept it out. diff --git a/docs/how-to/keep-the-site-small.md b/docs/how-to/keep-the-site-small.md new file mode 100644 index 0000000..82dd847 --- /dev/null +++ b/docs/how-to/keep-the-site-small.md @@ -0,0 +1,93 @@ +# How-to: keep the published site under the Pages size limit + +Every deploy uploads the **whole** site as one artifact, and GitHub Pages rejects an +artifact over **1 GB**. Nothing warns you as you approach it; deploys simply start +failing once you arrive. + +The cap is on the **packed** artifact, not the directory tree — HTML and JS compress +around 3×, so a site that is 734 MiB on disk deploys as about 223 MiB. + +A docs build is a few megabytes, so this only bites long-lived projects — but it bites +them hard. [blueapi](https://github.com/DiamondLightSource/blueapi), with 131 released +`docs.zip`s, was at **452 MB** and adding ~5 MB per release: a few years of headroom, and +no signal on the way. + +## Check where you are + +Every deploy prints the size of the tree it assembled: + +``` +assembled site: 734MB uncompressed across 31 version dir(s) +``` + +Once that passes 1.5 GiB it also packs the tree to measure what will really be +uploaded, and warns past 700 MB of *packed* bytes: + +``` +packed artifact: ~712MB (Pages rejects over 1 GB) +``` + +Or measure the released half directly: + +```bash +gh api --paginate repos/ORG/REPO/releases \ + | jq -rs '[.[][].assets[] | select(.name=="docs.zip") | .size] | add / 1048576 | floor' +``` + +## Cap the number of published releases + +Set `max-releases` in your `publish-dispatch.yml` shim, in the `with:` block: + +```yaml +jobs: + publish: + uses: DiamondLightSource/myst-version-switcher-plugin/.github/workflows/publish.yml@ + with: + version-name: ${{ inputs.version-name }} + max-releases: "20" # ← publish only the 20 most recent releases + pr: ${{ inputs.pr }} + dispatch-workflow: publish-dispatch.yml + retry-until: ${{ inputs.retry-until }} +``` + +:::{important} +Set it in the **shim**, not in `ci.yml`. The shim is the single caller of the engine on +every path — inline publish, tag re-dispatch, fork preview and manual re-deploy alike — +so a literal there is the one place the policy cannot be bypassed. Threading it as an +input from `ci.yml` would leave manual dispatches publishing a different number of +versions from ordinary CI. +::: + +The default is `0`, meaning unlimited, so upgrading the pinned tag never removes versions +from a site that already has them. You opt in. + +### What the cap does and doesn't do + +Releases are ranked by the **tagged commit's date** (`created_at`), newest first, and the +top N are published. Nothing parses your version numbers — tags vary far too much between +projects for that to be safe, and `published_at` is unreliable because re-publishing an +old release stamps it as new. + +Capping only affects **what the site serves**. Every release keeps its `docs.zip` asset on +the GitHub Release, so raising the cap brings the old versions straight back on the next +deploy. Nothing is deleted. + +Dropped versions leave the switcher and their URLs 404. Pick N so that the versions people +actually link to stay in — and note that prereleases occupy slots too: blueapi's newest 20 +releases include 5 prereleases, so `max-releases: 20` publishes only 15 real ones. + +## What isn't capped + +- **The default branch** — always published; it is the one required version. +- **The migration seed release** (`pages-default-seed`) — it stands in for the default + branch rather than being a version of its own. +- **Open-PR previews** — naturally bounded by how many PRs you have open, and they leave + as PRs close. blueapi's 26 open PRs are about 91 MB. +- **Releases with no `docs.zip` asset** — never published in the first place, and they do + not consume a slot. + +## Related + +- [choose-publish-events](choose-publish-events.md) — how often deploys run +- [architecture](../explanations/architecture.md#why-the-site-has-a-size-limit) — why + reconstruct-everything has this cost at all diff --git a/docs/how-to/migrate-from-gh-pages.md b/docs/how-to/migrate-from-gh-pages.md index cfd3d76..274c621 100644 --- a/docs/how-to/migrate-from-gh-pages.md +++ b/docs/how-to/migrate-from-gh-pages.md @@ -10,7 +10,7 @@ between**: Pages source to Actions + open the environment policy. *Uploads and flips only — no deploy.* 2. **Open + merge your pipeline PR** — its CI runs the first publish, which reads the - seed and persists the default branch durably into the site. + seed to stage the default branch, and then builds the branch's own docs. 3. **`migrate.sh ORG/REPO --delete-gh-pages`** — verify the live site, then delete `gh-pages` + the seed release. @@ -23,7 +23,7 @@ durability** — and the safety of the migration hinges on that difference (see | version | source under the new model | durable? | |---|---|---| | released tags | a `docs.zip` asset on each **GitHub Release** | **yes, permanent** — but only once attached | -| default branch (`main`) | persisted each deploy into the site at `_sources/.zip` | **yes** — once one deploy has captured it (seeded in run 1) | +| default branch (`main`) | its own CI `docs` artifact, with a copy kept in the Actions cache | **yes** — once the branch builds its own docs (seeded in run 1 until then) | | open PRs (`pr-`) | each PR's build artifact | no — drops on merge/close | Two consequences drive the procedure: @@ -38,22 +38,23 @@ Two consequences drive the procedure: the extract takes the zip's single root directory whatever it is named, so python-copier-template's tag-name-rooted zips (`1.2.3/` rather than `html/`) need no re-cutting, which matters because an immutable release cannot be re-cut anyway. -2. **The default branch has no permanent source until a publish captures it.** Each - deploy persists the branch's `docs.zip` into the site (`_sources/.zip`) and - restores from there when the CI artifact expires — but before the *first* publish - there is nothing in the site. So run 1 **seeds** it: it captures the gh-pages +2. **The default branch has no source until it builds docs itself.** Each deploy + gathers it from its own CI artifact, keeping a copy in the Actions cache for when + that artifact expires — but before the branch has ever built under the new pipeline + there is nothing to gather. So run 1 **seeds** it: it captures the gh-pages `/` tree as a published seed release, which the first publish reads and - persists to `_sources/.zip`. After that the in-site copy carries `//` - — *before* the default branch ever builds docs itself. + stages as `//` — *before* the default branch ever builds docs itself. The + seed carries it until the branch's own build takes over. For *why* this ordering is safe — the non-destructive source flip and the seed-before-publish rule — see the [architecture explanation](../explanations/architecture.md#migrating-from-gh-pages). :::{important} The one rule that matters -*Keep `gh-pages` until `_sources/.zip` is live in the deployed site.* Run 1's -seed gets you there on the first publish; the `--delete-gh-pages` guard probes exactly -this. Deleting `gh-pages` is the **separate, gated final run** — never part of run 1 — -and it removes the seed release too. +*Keep `gh-pages` until the default branch builds its own docs under the new pipeline.* +Until then the seed — and `gh-pages` behind it — is the only copy there is. The +`--delete-gh-pages` guard checks exactly that, and will not delete before it holds. +Deleting `gh-pages` is the **separate, gated final run**, never part of run 1, and it +removes the seed release too. ::: ## Optional: rehearse on a fork first @@ -87,7 +88,7 @@ every version, and it deploys to *your* `github.io`, never upstream's. :::{note} What the fork trial does and doesn't cover It exercises the full prepare → publish path (backfill, seed, the first deploy that -persists `_sources/.zip`), and you can even rehearse the destructive finalize +stages `//` from the seed), and you can even rehearse the destructive finalize (`migrate.sh FORKORG/REPO --delete-gh-pages`) without risk — it only touches the fork. The `` pins still resolve to this project's upstream releases, so the reusable workflows behave identically. The only path it doesn't auto-cover is a release: push a @@ -132,7 +133,7 @@ It does the following, then **stops with `gh-pages` intact and still serving**: (flagged prerelease for `a`/`b`/`rc` tags). Tags containing `/` are skipped. 2. **Seed the default branch.** Capture the gh-pages `/` tree (or the `--seed-from ` directory, if the old site published it under another name) as - the published `pages-default-seed` release, so the default branch is durable before + the published `pages-default-seed` release, so the default branch has a source before any publish. 3. **Flip the Pages source → GitHub Actions** and **open the `github-pages` environment's `deployment_branch_policy`** to "no restriction" (so deploys from PR/tag refs — which @@ -149,31 +150,38 @@ Prepare the new pipeline + `myst.yml` changes from the after run 1 has seeded the default branch** (so its first publish is safe). Then open and merge it: its CI runs the first **publish** — with the source on Actions, the seed present, and the env policy open, it reconstructs the whole site -(default branch from the seed, the backfilled releases, any open PRs) and deploys it, -**persisting `_sources/.zip`** into the published site. Merging then has the -default branch build its own docs, so `_sources/.zip` refreshes with real -content and the seed becomes redundant. +(default branch from the seed, the backfilled releases, any open PRs) and deploys it. +Merging then has the default branch build its own docs, which supersede the seed and make +it redundant. Confirm the site is live on the new model (visit `https://ORG.github.io/REPO/` and the switcher) before finalizing. ## Step 4 — finalize (irreversible) -Once the publish has deployed and `_sources/.zip` is live: +Once the publish has deployed **and the default branch has built its own docs**: ```bash scripts/migrate.sh ORG/REPO --delete-gh-pages ``` -(Or run it right after merging with `--wait`: the guard then polls until the durable -copy goes live — up to 30 minutes — instead of failing immediately.) +(Or run it right after merging with `--wait`: the guard then polls until it passes — up +to 30 minutes — instead of failing immediately.) -This **guards** the deletion: it refuses unless -`https://ORG.github.io/REPO/_sources/.zip` returns `200` — i.e. the deployed -site holds a durable copy of the default branch, so the new model can reconstruct -`//` without `gh-pages`. It then verifies the live site, asks you to type the -repo name, deletes `gh-pages`, **and deletes the seed release** (the in-site `_sources` -copy supersedes it). After this, the rollback is gone. +This **guards** the deletion. It refuses unless both hold: + +1. `https://ORG.github.io/REPO//` returns `200` — the default branch is live in + the new Actions-deployed site; and +2. a non-expired `docs` artifact exists for the default branch — its CI really builds + `docs.zip` now. + +Condition 1 alone is not enough: it is satisfied while `//` is still coming from +the migration seed, which is precisely the state where `gh-pages` remains the only real +copy. Only once the branch builds its own docs is `gh-pages` genuinely redundant. + +The guard then verifies the live site, asks you to type the repo name, deletes +`gh-pages`, **and deletes the seed release** (the branch's own build supersedes it). +After this, the rollback is gone. :::{warning} Old pages that reference `gh-pages` at runtime Docs built under the old model sometimes embed a hardcoded version switcher that reads @@ -199,8 +207,8 @@ lost. That is exactly why the deletion is a separate, gated run. | flag | effect | |---|---| | `--dry-run` | Print the backfill + seed plan and the drop report + probe the current site only; upload nothing; skip the flip. | -| `--delete-gh-pages` | **The only mode that deletes.** Guard that `_sources/.zip` is live (`200`), verify the live site, then delete `gh-pages` **and the seed release**. | +| `--delete-gh-pages` | **The only mode that deletes.** Guard that the default branch is live *and* builds its own `docs` artifact, verify the live site, then delete `gh-pages` **and the seed release**. | | `--pages-ref ` | `gh-pages` ref to read (default `origin/gh-pages`). | | `--seed-from ` | `gh-pages` directory to seed the default branch from, when the old site published it under a different name (e.g. `latest/`). | | `--yes` | Skip the typed confirmation before deleting `gh-pages` (with `--delete-gh-pages`; use with care). | -| `--wait` | With `--delete-gh-pages`: poll (up to 30 min) until `_sources/.zip` goes live instead of failing the guard immediately — lets you finalize right after merging the pipeline PR. | +| `--wait` | With `--delete-gh-pages`: poll (up to 30 min) until the guard passes instead of failing immediately — lets you finalize right after merging the pipeline PR. | diff --git a/docs/myst.yml b/docs/myst.yml index c36febc..c453ccf 100644 --- a/docs/myst.yml +++ b/docs/myst.yml @@ -27,6 +27,8 @@ project: - file: how-to.md children: - file: how-to/migrate-from-gh-pages.md + - file: how-to/choose-publish-events.md + - file: how-to/keep-the-site-small.md - file: how-to/use-with-sphinx.md - file: how-to/contribute.md - file: explanations.md diff --git a/docs/reference/workflows.md b/docs/reference/workflows.md index 06ed672..b2becc1 100644 --- a/docs/reference/workflows.md +++ b/docs/reference/workflows.md @@ -89,6 +89,7 @@ re-dispatch, a fork-PR preview, a manual re-deploy). A reusable workflow can't b |---|---|---|---| | `version-name` | no | `""` | Version name of **this run's** `docs` artifact to stage directly, instead of gathering it from durable sources. Set by `ci.yml`'s inline publish (the run isn't a completed success yet, so the gather can't discover it — or would find a stale previous build). Empty → pure durable gather (the dispatch paths). | | `pr` | no | `""` | Fork PR number to approve (pins its head SHA as `preview-approved`) and preview. Set on the shim's `workflow_dispatch` path. | +| `max-releases` | no | `"0"` | Publish only the N most recent releases, ranked by the tagged commit's date (`created_at`); `0` = all of them. The site deploys as **one** Pages artifact against a hard **1 GB** cap, which a long-lived project will eventually hit. Set it as a **literal in the shim's `with:` block**, not threaded from `ci.yml`, so it applies on every path. See [keep-the-site-small](../how-to/keep-the-site-small.md). | | `dispatch-workflow` | no | `publish-dispatch.yml` | Filename of the shim in the consumer's repo, re-fired by the `re-dispatch` job. Override only if you renamed the shim. | | `retry-until` | no | `""` | Internal — epoch-seconds deadline for auto-retrying a wedged Pages origin. Set only by the `re-dispatch` job (from the release's publish time) when it re-fires the shim; carried unchanged through any retries. **The shim must declare + forward this input** (`workflow_dispatch` + the `with:` block below) even though consumers never set it themselves — `re-dispatch` always passes it, and a shim missing the input rejects the dispatch. Don't set manually. | @@ -99,15 +100,20 @@ Every deploy rebuilds the complete tree from authoritative inputs: | version kind | source | durability | |---|---|---| | current build | this run's `docs` artifact, staged via `version-name` (highest priority — overwrites any same-name gathered source) | n/a (just built) | -| default branch (e.g. `main`) | newest `docs` artifact built from that branch → durable `_sources/.zip` in the live site (hard-fail if neither exists) | durable — re-persisted into the site each deploy | -| released tags | the `docs.zip` asset attached to each **GitHub Release** (the migration seed release, if present, seeds the default-branch zip) | permanent | +| default branch (e.g. `main`) | newest `docs` artifact built from that branch → the **Actions cache** copy (hard-fail if neither exists) | cached — re-saved on each default-branch deploy; evicted after 7 days unread | +| released tags | the `docs.zip` asset attached to each **GitHub Release**, newest `max-releases` of them (the migration seed release, if present, seeds the default-branch zip and is never capped) | permanent | | open PRs (`pr-`) | each PR's `docs` artifact, keyed by current head SHA — internal always, fork PRs only when the SHA carries a `preview-approved` status | ephemeral — drops when the PR merges/closes | Branch and PR artifacts are found via the **artifacts API by name** (`docs` — the artifact name is the contract), not by workflow filename, so the consumer's entry -workflow can be called anything. The URLs baked into `switcher.json` (and the -`_sources` restore) use the site's live Pages URL from the **Pages API**, so a -custom domain (CNAME) works. +workflow can be called anything. The URLs baked into `switcher.json` use the site's +live Pages URL from the **Pages API**, so a custom domain (CNAME) works. + +Release assets are immutable, so the gather caches them (`actions/cache`, keyed on the +exact set of asset ids it intends to publish) and downloads only what it has not seen; +downloads run in parallel, and the artifacts API is paginated **once** and indexed by +head SHA rather than re-queried per PR. Caches are written only from the default branch, +because a cache saved on a PR ref is scoped to that PR and unreadable by any other run. A version no longer gathered (a merged/closed PR, a deleted release) is correctly dropped, because `deploy-pages` replaces the *entire* site. Sources are gathered in diff --git a/docs/tutorials/adding-to-a-fresh-repo.md b/docs/tutorials/adding-to-a-fresh-repo.md index 5e95ada..0fb6b72 100644 --- a/docs/tutorials/adding-to-a-fresh-repo.md +++ b/docs/tutorials/adding-to-a-fresh-repo.md @@ -142,6 +142,10 @@ jobs: with: # "" on dispatch → pure durable gather version-name: ${{ inputs.version-name }} + # Publish only the N most recent releases. A LITERAL, not an input threaded from + # ci.yml, so it applies on every path — the site deploys as one Pages artifact + # against a hard 1 GB cap. 0 = unlimited. See docs/how-to/keep-the-site-small.md. + max-releases: "20" # set (dispatch) → pin that fork head SHA pr: ${{ inputs.pr }} # the file the tag re-dispatch re-fires diff --git a/package-lock.json b/package-lock.json index e5be372..552bf3f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -81,9 +81,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT OR Apache-2.0", "optional": true, "os": [ @@ -101,9 +98,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT OR Apache-2.0", "optional": true, "os": [ @@ -121,9 +115,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT OR Apache-2.0", "optional": true, "os": [ @@ -141,9 +132,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT OR Apache-2.0", "optional": true, "os": [ @@ -278,9 +266,6 @@ "arm" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -298,9 +283,6 @@ "arm" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -318,9 +300,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -338,9 +317,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -358,9 +334,6 @@ "arm" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -378,9 +351,6 @@ "ia32" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -398,9 +368,6 @@ "ia32" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -418,9 +385,6 @@ "riscv64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -438,9 +402,6 @@ "s390x" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -458,9 +419,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -478,9 +436,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ diff --git a/package.json b/package.json index e255ec4..8eaeecf 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,8 @@ "scripts": { "test": "node test/test-url-logic.mjs && node test/test-assemble.mjs", "docs": "cd docs && myst build --html --strict", - "docs-dev": "cd docs && myst start" + "docs-dev": "cd docs && myst start", + "test:workflows": "uv run --quiet --with pyyaml python3 test/workflow-harness/test_gather.py" }, "files": [ "plugins/version-switcher.mjs", diff --git a/scripts/migrate.sh b/scripts/migrate.sh index 480e08a..f6193ca 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -14,9 +14,9 @@ # sources differ in durability. Releases become permanent once their docs.zip is # attached (backfill). The DEFAULT BRANCH has no permanent source until it builds # docs.zip itself — so we SEED it: capture the gh-pages / tree as a published -# seed release, which the first publish reads and persists to _sources/.zip in -# the site. After that the in-site copy is the durable source; gh-pages is no longer -# needed and deleting it is split off behind a guard: +# seed release, which the first publish reads and stages as //. gh-pages stops +# being needed once the branch builds its OWN docs.zip, and deleting it is split off +# behind a guard that waits for exactly that: # # 1. prepare (default) backfill releases → seed default branch → flip Pages # source to Actions + open the env policy, then STOP. @@ -24,9 +24,9 @@ # non-destructive: the last gh-pages deployment keeps # serving until the first Actions deploy supersedes it.) # 2. (you) open + merge your pipeline PR — its CI does the first publish, which reads -# the seed and persists _sources/.zip. -# 3. finalize (--delete-gh-pages) guard _sources/.zip is live → verify the -# live site → delete gh-pages + the seed release. +# the seed, and (once merged to the default branch) builds the branch's own docs. +# 3. finalize (--delete-gh-pages) guard the default branch builds + serves its own +# docs → verify the live site → delete gh-pages + the seed. # # Between runs 1 and 3, gh-pages still EXISTS (just unserved) and is the rollback: flip # the Pages source back to "Deploy from a branch" to restore serving instantly. @@ -38,16 +38,16 @@ # --dry-run print the backfill + seed plan, the versions the new model # will DROP, + probe the current site; upload nothing, skip # the flip. -# --delete-gh-pages finalize: guard _sources/.zip is live, verify the live -# site, then delete gh-pages + the seed release. The only mode -# that deletes. +# --delete-gh-pages finalize: guard the default branch builds and serves its own +# docs, verify the live site, then delete gh-pages + the seed +# release. The only mode that deletes. # --pages-ref gh-pages ref to read (default: origin/gh-pages) # --seed-from gh-pages dir to seed the default branch from, when it isn't # named after the branch (e.g. an old site publishing latest/). # --yes skip the typed confirmation before deleting gh-pages. -# --wait with --delete-gh-pages: poll until _sources/.zip goes -# live (up to 30 min) instead of failing the guard immediately — -# lets you run finalize right after merging the pipeline PR. +# --wait with --delete-gh-pages: poll until the guard passes (up to +# 30 min) instead of failing immediately — lets you run finalize +# right after merging the pipeline PR. set -euo pipefail REPO="" @@ -207,8 +207,8 @@ report_drops() { # The default branch has no durable source until it builds docs.zip under the new # pipeline, and a contents:read deploy can't read a draft — so capture the gh-pages # / tree as a docs.zip and PUBLISH it on a sentinel tag (not the branch name). -# assemble reads it once, stages //, and persists _sources/.zip; from -# then on the in-site copy carries it and this seed is dormant (deleted at finalize). +# assemble reads it and stages //; once the branch builds its own docs.zip +# that supersedes the seed, which then goes dormant (deleted at finalize). # This lets a repo cut over to the reusable workflow in ONE PR, before docs→default. seed_default_branch() { echo @@ -236,15 +236,15 @@ seed_default_branch() { else gh release create "$SEED_TAG" "$tmp/docs.zip" --repo "$REPO" --latest=false \ --title "Default-branch docs seed (migration)" \ - --notes "Temporary: seeds /$default/ until the default branch builds docs.zip under the new pipeline. Safe to delete once _sources/$default.zip is live in the site." + --notes "Temporary: seeds /$default/ until the default branch builds docs.zip under the new pipeline. Safe to delete once $default builds its own docs." fi rm -rf "$tmp" } -# Delete the seed release + its tag (the durable _sources copy supersedes it). +# Delete the seed release + its tag (the branch's own docs.zip supersedes it). delete_seed_release() { if gh release view "$SEED_TAG" --repo "$REPO" >/dev/null 2>&1; then - echo "-- Deleting the migration seed release '$SEED_TAG' (superseded by _sources) --" + echo "-- Deleting the migration seed release '$SEED_TAG' (superseded by the branch's own build) --" $DRY_RUN || gh release delete "$SEED_TAG" --repo "$REPO" --cleanup-tag --yes fi } @@ -273,36 +273,53 @@ verify() { } # --- guard for the irreversible delete -------------------------------------- -# gh-pages is the only recoverable copy of the default branch's docs until the SITE -# itself holds a durable copy at _sources/.zip — written each deploy once the -# branch has been gathered (from its own build, or from the migration seed). Probe the -# live site directly: if that copy is served, the default branch survives a gh-pages -# delete (and a later artifact expiry), so it is safe to remove. +# gh-pages is the only recoverable copy of the default branch's docs until the branch +# BUILDS ITS OWN under the new pipeline. Two conditions, both required: +# +# 1. // is served from the new Actions-deployed site, and +# 2. a non-expired `docs` artifact exists for the default branch, built from this +# repo — i.e. its CI really produces docs.zip now. +# +# (1) alone is not enough: it is still satisfied while // is coming from the +# migration seed, which is exactly the state where gh-pages is still the only real +# copy. This guard used to probe the in-site `_sources/.zip` instead; that +# file is no longer published (the default branch's durable copy lives in the Actions +# cache, which has no public URL to probe and is evictable), so the guard now waits for +# the stronger and more honest condition it always should have used. guard_default_durable() { - local default url code deadline + local default url code repo_id arts deadline default=$(gh repo view "$REPO" --json defaultBranchRef -q .defaultBranchRef.name) - url="$BASE/_sources/$default.zip" - echo "-- guard: is the default branch ($default) durable in the site (_sources)? --" + repo_id=$(gh api "repos/$REPO" -q .id) + url="$BASE/$default/index.html" + echo "-- guard: does '$default' build and serve its own docs yet? --" deadline=$(( $(date +%s) + 1800 )) while :; do code=$(curl -s -o /dev/null -w '%{http_code}' "$url?cb=$(date +%s%N)") - [ "$code" = "200" ] && break + arts=$(gh api --paginate "repos/$REPO/actions/artifacts?name=docs&per_page=100" 2>/dev/null \ + | jq -rs --arg b "$default" --argjson r "$repo_id" '[.[].artifacts[] + | select((.expired | not) + and .workflow_run.head_branch == $b + and .workflow_run.head_repository_id == $r)] | length' 2>/dev/null || echo 0) + if [ "$code" = "200" ] && [ "${arts:-0}" -gt 0 ]; then break; fi if ! $WAIT; then - echo "::error::REFUSING to delete gh-pages: $url is not live ($code)." >&2 - echo " The deployed site has no durable copy of '$default' yet, so gh-pages is still" >&2 - echo " the only recoverable copy. Open/merge the pipeline PR so its publish persists" >&2 - echo " _sources/$default.zip into the site, then retry (or rerun with --wait to poll)." >&2 + echo "::error::REFUSING to delete gh-pages — '$default' is not yet self-sufficient." >&2 + echo " /$default/ live in the new site : $code (want 200)" >&2 + echo " non-expired 'docs' artifacts : ${arts:-0} (want at least 1)" >&2 + echo " gh-pages is still the only recoverable copy. Merge your pipeline PR so the" >&2 + echo " default branch builds its own docs, let that publish deploy, then retry" >&2 + echo " (or rerun with --wait to poll)." >&2 exit 1 fi if [ "$(date +%s)" -ge "$deadline" ]; then - echo "::error::gave up after 30 min: $url still not live ($code). Has the pipeline" >&2 - echo " PR's publish deployed? Check the repo's Actions runs, then retry." >&2 + echo "::error::gave up after 30 min: /$default/ is $code and '$default' has ${arts:-0}" >&2 + echo " non-expired 'docs' artifacts. Has the pipeline PR merged and published?" >&2 + echo " Check the repo's Actions runs, then retry." >&2 exit 1 fi - echo " not live yet ($code) — waiting 15s (Ctrl-C to abort; gh-pages is untouched)" + echo " not ready yet (site $code, ${arts:-0} artifact(s)) — waiting 15s (Ctrl-C to abort; gh-pages is untouched)" sleep 15 done - echo " ok: $url is live — '$default' is durable independently of gh-pages" + echo " ok: /$default/ is live and '$default' builds its own docs — gh-pages is redundant" } echo "== migrate: $REPO (pages-ref=$PAGES_REF, dry-run=$DRY_RUN, delete=$DELETE_GH_PAGES) ==" @@ -335,7 +352,7 @@ if $DELETE_GH_PAGES; then # git-over-https push from the temp clone would need a separately configured # credential helper (`gh auth setup-git`). gh api --method DELETE "repos/$REPO/git/refs/heads/${PAGES_REF#origin/}" - delete_seed_release # the in-site _sources copy now carries the default branch + delete_seed_release # the branch's own docs.zip now carries the default branch echo "Done. Site is served from GitHub Actions; gh-pages removed." exit 0 fi @@ -382,8 +399,8 @@ echo "flip the source back to 'Deploy from a branch' to restore it instantly)." echo echo "Next:" echo " 1. Open + merge your pipeline PR. Its CI runs the first publish, which reads" -echo " the seed and persists _sources/.zip into the deployed site." -echo " 2. Once that publish has deployed, finalize (verify + delete gh-pages and the" -echo " seed). The guard checks _sources/.zip is live first:" +echo " the seed to stage // in the deployed site." +echo " 2. Once the default branch has built its own docs, finalize (verify + delete" +echo " gh-pages and the seed). The guard waits for that build first:" echo " scripts/migrate.sh $REPO --delete-gh-pages" -echo " (add --wait to run it right after merging — it polls until the copy is live)" +echo " (add --wait to run it right after merging — it polls until the guard passes)" diff --git a/test/test-assemble.mjs b/test/test-assemble.mjs index c9acbc4..b904591 100644 --- a/test/test-assemble.mjs +++ b/test/test-assemble.mjs @@ -18,6 +18,7 @@ import { preferredVersion, renderRedirect, renderSwitcher, + selectReleases, stablePlan, switcherStruct, } from "../assemble/assemble.mjs"; @@ -215,4 +216,120 @@ const toMain = renderRedirect("main"); assert.match(toMain, /url=\.\/main\/index\.html/); ok("renderRedirect targets the fallback dir before the first release"); +// --- selectReleases: which releases become site dirs, and why ----------------- +// Shaped like the raw `GET /repos/{repo}/releases` payload publish.yml pipes in. +const rel = (tag, created, { docsZip = true, size = 100 } = {}) => ({ + tag_name: tag, + created_at: created, + // Deliberately NOT the ranking key: re-publishing an old release stamps a fresh + // published_at, which is why selectReleases ranks on created_at instead. + published_at: "2026-07-30T16:07:14Z", + assets: docsZip ? [{ name: "docs.zip", id: `id-${tag}`, size }] : [], +}); + +// Deliberately NOT in date order, to prove the cap ranks by date rather than by +// the order the API happened to return. Every one shares a published_at, so any +// test that passes here is ranking on created_at. +const releases = [ + rel("1.0", "2026-01-01T00:00:00Z"), + rel("3.0", "2026-03-01T00:00:00Z"), + rel("2.0", "2026-02-01T00:00:00Z"), +]; +const gatheredOf = (rows) => + rows.filter((r) => r.dest !== null).map((r) => r.dest); + +assert.deepEqual( + gatheredOf(selectReleases(releases, { defaultBranch: "main" })), + ["1.0", "3.0", "2.0"], +); +ok("selectReleases gathers every release when max-releases is unlimited"); + +// A row per listed release either way — the decision table must account for all. +assert.equal(selectReleases(releases, { defaultBranch: "main" }).length, 3); +ok("selectReleases returns one row per listed release"); + +const capped = selectReleases(releases, { + defaultBranch: "main", + maxReleases: 2, +}); +assert.deepEqual(gatheredOf(capped).sort(), ["2.0", "3.0"]); +assert.match( + capped.find((r) => r.tag === "1.0").decision, + /beyond max-releases=2 \(dated 2026-01-01/, +); +ok( + "selectReleases keeps the newest max-releases by created_at, not published_at", +); + +assert.deepEqual( + gatheredOf( + selectReleases(releases, { defaultBranch: "main", maxReleases: 0 }), + ), + ["1.0", "3.0", "2.0"], +); +ok("selectReleases treats max-releases 0 as unlimited"); + +assert.deepEqual( + gatheredOf( + selectReleases(releases, { defaultBranch: "main", maxReleases: 9 }), + ), + ["1.0", "3.0", "2.0"], +); +ok("selectReleases is a no-op when there are fewer releases than the cap"); + +// The seed release stands in for the default branch, so it is not a version and +// must never lose its slot to the cap — even when it is the oldest release there is. +const withSeed = selectReleases( + [rel("pages-default-seed", "2020-01-01T00:00:00Z"), ...releases], + { defaultBranch: "main", maxReleases: 1 }, +); +assert.deepEqual(gatheredOf(withSeed).sort(), ["3.0", "main"]); +ok("selectReleases exempts the seed release from the cap"); + +// Skipped releases must not consume a cap slot either. +const withJunk = selectReleases( + [ + rel("release/1.0", "2026-04-01T00:00:00Z"), + rel("4.0", "2026-04-02T00:00:00Z"), + ...releases, + ], + { defaultBranch: "main", maxReleases: 2 }, +); +assert.deepEqual(gatheredOf(withJunk).sort(), ["3.0", "4.0"]); +ok("selectReleases does not spend a cap slot on a skipped release"); + +const skips = selectReleases( + [ + rel("release/1.0", "2026-01-01T00:00:00Z"), + rel("main", "2026-01-02T00:00:00Z"), + rel("5.0", "2026-01-03T00:00:00Z", { docsZip: false }), + ], + { defaultBranch: "main" }, +); +assert.deepEqual(gatheredOf(skips), []); +assert.match(skips[0].decision, /'\/' in tag/); +assert.match(skips[1].decision, /same name as the default branch/); +assert.match(skips[2].decision, /no docs.zip asset/); +ok( + "selectReleases skips '/'-tags, default-branch-named tags and asset-less releases", +); + +// The asset id rides along so publish.yml can key its download cache on it. +assert.equal( + selectReleases(releases, { defaultBranch: "main" })[0].assetId, + "id-1.0", +); +ok("selectReleases carries the docs.zip asset id through for cache keying"); + +// A release with no created_at at all must not win the cap, and must not throw. +const undated = selectReleases( + [ + { tag_name: "9.9", assets: [{ name: "docs.zip", id: "id-9.9", size: 1 }] }, + rel("3.0", "2026-03-01T00:00:00Z"), + ], + { defaultBranch: "main", maxReleases: 1 }, +); +assert.deepEqual(gatheredOf(undated), ["3.0"]); +ok("selectReleases sorts undated releases last rather than failing"); + console.log(`\nAll ${passed} checks passed.`); diff --git a/test/workflow-harness/bin/gh b/test/workflow-harness/bin/gh new file mode 100755 index 0000000..0bdbbd7 --- /dev/null +++ b/test/workflow-harness/bin/gh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Mock gh for the publish.yml gather harness. Records every call. +set -uo pipefail +echo "gh $*" >> "$MOCK_CALLS" +case "$*" in + "api --paginate repos/$REPO/releases") cat "$MOCK_DATA/releases.json" ;; + "api repos/$REPO -q .id") echo 12345 ;; + "api --paginate repos/$REPO/actions/artifacts?name=docs&per_page=100") + cat "$MOCK_DATA/artifacts.json" ;; + "pr list --repo $REPO --state open --limit 200 --json number,headRefOid,isCrossRepository -q .[] | [.number, .headRefOid, .isCrossRepository] | @tsv") + cat "$MOCK_DATA/prs.tsv" ;; +esac +case "$1$2" in + releasedownload) + tag="$3"; out="" + while [ $# -gt 0 ]; do [ "$1" = "-O" ] && out="$2"; shift; done + if grep -qxF "$tag" "$MOCK_DATA/download-fails" 2>/dev/null; then + echo "HTTP 404: not found" >&2; exit 1 + fi + "$MOCK_DATA/mkzip.sh" "$out" "release-$tag"; exit 0 ;; +esac +if [[ "$*" == api*actions/artifacts/*/zip ]]; then + id=$(sed 's#.*artifacts/\([0-9]*\)/zip.*#\1#' <<<"$*") + if grep -qxF "$id" "$MOCK_DATA/artifact-fails" 2>/dev/null; then exit 1; fi + if grep -qxF "$id" "$MOCK_DATA/artifact-nodocs" 2>/dev/null; then + "$MOCK_DATA/mkartifact.sh" /dev/stdout "$id" nodocs; exit 0 + fi + "$MOCK_DATA/mkartifact.sh" /dev/stdout "$id" ok; exit 0 +fi +if [[ "$*" == api*commits/*/statuses* ]]; then + sha=$(sed 's#.*commits/\([^/]*\)/statuses.*#\1#' <<<"$*") + grep -qxF "$sha" "$MOCK_DATA/approved" 2>/dev/null && echo true || echo false + exit 0 +fi +exit 0 diff --git a/test/workflow-harness/fixtures.py b/test/workflow-harness/fixtures.py new file mode 100644 index 0000000..d74493f --- /dev/null +++ b/test/workflow-harness/fixtures.py @@ -0,0 +1,37 @@ +"""Synthetic release/artifact/PR payloads shared by the harness tests.""" + +def rel(tag, created, asset_id=None, size=100): + a = [] if asset_id is None else [{"name": "docs.zip", "id": asset_id, "size": size}] + return {"tag_name": tag, "created_at": created, "published_at": created, "assets": a} + +def art(aid, sha, created, branch=None, repo_id=12345, run_id=999): + return {"id": aid, "expired": False, "created_at": created, "size_in_bytes": 4096, + "workflow_run": {"id": run_id, "head_sha": sha, "head_branch": branch, + "head_repository_id": repo_id}} + +RELEASES = [ + rel("3.0", "2026-03-01T00:00:00Z", 301), + rel("2.0", "2026-02-01T00:00:00Z", 201), + rel("1.0", "2026-01-01T00:00:00Z", 101), + rel("release/x", "2026-04-01T00:00:00Z", 401), # '/' tag -> skip + rel("0.9", "2025-12-01T00:00:00Z", None), # no docs.zip -> skip + rel("pages-default-seed", "2020-01-01T00:00:00Z", 1), +] +ARTIFACTS = [ + art(9001, "mainsha", "2026-05-01T00:00:00Z", branch="main"), + art(9000, "mainsha", "2026-04-01T00:00:00Z", branch="main"), # older, must lose + art(9002, "forkbranch", "2026-05-01T00:00:00Z", branch="main", repo_id=777), # fork, must lose + art(9100, "pr7sha", "2026-05-02T00:00:00Z"), + art(9200, "pr8sha", "2026-05-02T00:00:00Z"), +] +PRS = [(7, "pr7sha", False), (8, "pr8sha", True), (9, "pr9sha", False)] + +def do(tmp, maxrel="0", **kw): + env, rt, data = setup(tmp, RELEASES, ARTIFACTS, PRS, **kw) + exprs = {"github.token": "t", "inputs.max-releases": maxrel, "runner.temp": rt} + r1 = run("Select releases to publish", env, exprs) + assert r1.returncode == 0, r1.stderr + r2 = run("Gather release artifacts", env, exprs) + r3 = run("Gather branch CI artifacts", env, exprs) + return env, rt, data, r1, r2, r3 + diff --git a/test/workflow-harness/mkartifact.sh b/test/workflow-harness/mkartifact.sh new file mode 100755 index 0000000..e843b46 --- /dev/null +++ b/test/workflow-harness/mkartifact.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# $1 = out (may be /dev/stdout), $2 = marker, $3 = ok|nodocs +set -euo pipefail +d=$(mktemp -d) +if [ "$3" = ok ]; then + "$(dirname "$0")/mkzip.sh" "$d/docs.zip" "artifact-$2" + ( cd "$d" && zip -rq artifact.zip docs.zip ) +else + echo hi > "$d/other.txt"; ( cd "$d" && zip -rq artifact.zip other.txt ) +fi +cat "$d/artifact.zip" > "$1"; rm -rf "$d" diff --git a/test/workflow-harness/mkzip.sh b/test/workflow-harness/mkzip.sh new file mode 100755 index 0000000..eba6b10 --- /dev/null +++ b/test/workflow-harness/mkzip.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +# $1 = output path, $2 = marker. Produces a valid single-root docs.zip. +set -euo pipefail +d=$(mktemp -d); mkdir -p "$d/html"; echo "$2" > "$d/html/index.html" +( cd "$d" && zip -rq out.zip html ) +mkdir -p "$(dirname "$1")"; cp "$d/out.zip" "$1"; rm -rf "$d" diff --git a/test/workflow-harness/run.py b/test/workflow-harness/run.py new file mode 100644 index 0000000..bf34dce --- /dev/null +++ b/test/workflow-harness/run.py @@ -0,0 +1,66 @@ +""" +Harness for the bash inside publish.yml's `deploy` job. + +The gather steps are the workflow's most intricate logic — release selection, +an asset cache, a SHA-indexed artifact lookup, parallel downloads, and four +distinct failure modes — and they used to be testable only by merging and +watching a real deploy. This loads the steps straight out of the YAML (so the +tests cannot drift from the workflow), substitutes the ${{ }} expressions, and +runs them against a mock `gh` and synthetic release/artifact payloads. + +Run with `npm run test:workflows`. +""" +import yaml, re, subprocess, os, sys, json, shutil, pathlib +H = os.path.dirname(os.path.abspath(__file__)) +REPO_DIR = os.path.dirname(os.path.dirname(H)) +d = yaml.safe_load(open(f"{REPO_DIR}/.github/workflows/publish.yml")) +steps = {s.get("name"): s for s in d["jobs"]["deploy"]["steps"] if s.get("run")} + +WANT = ["Select releases to publish", "Gather release artifacts", "Gather branch CI artifacts"] + +def run(step_name, env, exprs): + script = steps[step_name]["run"] + script = re.sub(r"\$\{\{\s*([^}]*?)\s*\}\}", lambda m: exprs.get(m.group(1), ""), script) + # step-level env: from the step's own `env:` block, after expression substitution + senv = dict(env) + for k, v in (steps[step_name].get("env") or {}).items(): + senv[k] = re.sub(r"\$\{\{\s*([^}]*?)\s*\}\}", lambda m: exprs.get(m.group(1), ""), str(v)) + p = subprocess.run(["bash", "-c", script], env=senv, capture_output=True, text=True) + return p + +def setup(tmp, releases, artifacts, prs, *, approved=(), dlfail=(), artfail=(), artnodocs=()): + data = os.path.join(tmp, "data") + os.makedirs(data, exist_ok=True) + json.dump(releases, open(f"{data}/releases.json", "w")) + # the real API wraps each page: {"total_count": N, "artifacts": [...]} + json.dump({"total_count": len(artifacts), "artifacts": artifacts}, open(f"{data}/artifacts.json", "w")) + open(f"{data}/prs.tsv", "w").write("".join(f"{n}\t{s}\t{str(c).lower()}\n" for n, s, c in prs)) + for name, vals in [("approved", approved), ("download-fails", dlfail), + ("artifact-fails", artfail), ("artifact-nodocs", artnodocs)]: + open(f"{data}/{name}", "w").write("".join(f"{v}\n" for v in vals)) + for f in ("mkzip.sh", "mkartifact.sh"): + shutil.copy(f"{H}/{f}", f"{data}/{f}"); os.chmod(f"{data}/{f}", 0o755) + ws = os.path.join(tmp, "ws"); os.makedirs(ws, exist_ok=True) + # COPY, don't symlink: assemble.mjs's `import.meta.url === file://argv[1]` main-guard + # compares a resolved path against an unresolved one, so a symlinked checkout would + # silently never run main(). CI does a real sparse-checkout, so this mirrors it. + mvs = os.path.join(ws, ".mvs", "assemble") + if os.path.exists(mvs): shutil.rmtree(mvs) + shutil.copytree(f"{REPO_DIR}/assemble", mvs) + rt = os.path.join(tmp, "runner"); os.makedirs(f"{rt}/gather", exist_ok=True); os.makedirs(f"{rt}/site", exist_ok=True) + env = dict(os.environ) + env.update({ + "PATH": f"{H}/bin:" + os.environ["PATH"], + "MOCK_DATA": data, "MOCK_CALLS": os.path.join(tmp, "calls.log"), + "RUNNER_TEMP": rt, "GITHUB_WORKSPACE": ws, + "GITHUB_OUTPUT": os.path.join(tmp, "out.txt"), + "GITHUB_SERVER_URL": "https://github.com", + "REPO": "acme/widget", "DEFAULT": "main", + "SEED_TAG": "pages-default-seed", "SOURCES_DIR": "_sources", + # Real defaults are 1.5 GiB / 700 MB; a test fixture that size is impractical, + # so the guard reads them from env and the size tests dial them down. + "SIZE_PROBE_BYTES": "1610612736", "SIZE_WARN_BYTES": "734003200", + }) + open(env["GITHUB_OUTPUT"], "w").close() + open(env["MOCK_CALLS"], "w").close() + return env, rt, data diff --git a/test/workflow-harness/test_gather.py b/test/workflow-harness/test_gather.py new file mode 100644 index 0000000..3ed2e34 --- /dev/null +++ b/test/workflow-harness/test_gather.py @@ -0,0 +1,199 @@ +"""Behaviour tests for publish.yml's gather steps. See run.py for how they load.""" + +import os, sys, tempfile + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from fixtures import ARTIFACTS, PRS, RELEASES +from run import run, setup + +fails = [] + + +def check(name, cond, detail=""): + print((" ok - " if cond else " FAIL - ") + name + (f"\n {detail}" if not cond and detail else "")) + if not cond: + fails.append(name) + + +def do(tmp, maxrel="0", **kw): + """Run the three gather steps in order against one synthetic repo.""" + env, rt, data = setup(tmp, RELEASES, ARTIFACTS, PRS, **kw) + exprs = {"github.token": "t", "inputs.max-releases": maxrel, "runner.temp": rt} + r1 = run("Select releases to publish", env, exprs) + assert r1.returncode == 0, r1.stderr + r2 = run("Gather release artifacts", env, exprs) + r3 = run("Gather branch CI artifacts", env, exprs) + return env, rt, data, r1, r2, r3 + +print("\n-- unlimited: all valid releases gathered, junk skipped --") +with tempfile.TemporaryDirectory() as tmp: + env, rt, data, r1, r2, r3 = do(tmp) + check("gather steps exit 0", r2.returncode == 0 and r3.returncode == 0, r2.stderr + r3.stderr) + got = sorted(os.listdir(f"{rt}/gather")) + check("releases + seed(main) + PR previews staged", + got == ["1.0.zip", "2.0.zip", "3.0.zip", "main.zip", "pr-7.zip"], got) + check("'/'-tag and asset-less release skipped", "release/x.zip" not in got and "0.9.zip" not in got) + check("unapproved fork PR #8 skipped", "pr-8.zip" not in got) + check("PR #9 with no artifact skipped", "pr-9.zip" not in got) + check("main.zip came from branch CI, not the seed release", + open(f"{rt}/gather/main.zip","rb").read() != open(f"{rt}/relcache/1.zip","rb").read()) + check("all release zips cached by asset id", + sorted(os.listdir(f"{rt}/relcache")) == ["1.zip", "101.zip", "201.zip", "301.zip"], + os.listdir(f"{rt}/relcache")) + calls = open(env["MOCK_CALLS"]).read() + check("artifacts API paginated exactly ONCE (was once per PR)", + calls.count("actions/artifacts?name=docs") == 1, + f"count={calls.count('actions/artifacts?name=docs')}") + check("newest artifact per SHA wins", "artifacts/9001/zip" in calls and "artifacts/9000/zip" not in calls) + check("fork-owned same-named branch excluded", "artifacts/9002/zip" not in calls) + +print("\n-- cache hit: second run downloads nothing --") +with tempfile.TemporaryDirectory() as tmp: + env, rt, data, *_ = do(tmp) + # rerun the release gather in the SAME runner temp: relcache is already populated + exprs = {"github.token": "t", "inputs.max-releases": "0", "runner.temp": rt} + open(env["MOCK_CALLS"], "w").close() + r = run("Gather release artifacts", env, exprs) + check("second run exits 0", r.returncode == 0, r.stderr) + check("no release downloaded on a full cache hit", + "release download" not in open(env["MOCK_CALLS"]).read()) + check("log reports all cached", "4 cached, 0 to download" in r.stdout, r.stdout[:400]) + +print("\n-- cap: only the newest N, seed exempt, cache pruned --") +with tempfile.TemporaryDirectory() as tmp: + env, rt, data, r1, r2, r3 = do(tmp, maxrel="2") + got = sorted(os.listdir(f"{rt}/gather")) + check("cap keeps the 2 newest releases", "3.0.zip" in got and "2.0.zip" in got and "1.0.zip" not in got, got) + check("seed still supplies main", "main.zip" in got) + check("cache holds only what is published", + sorted(os.listdir(f"{rt}/relcache")) == ["1.zip", "201.zip", "301.zip"], os.listdir(f"{rt}/relcache")) + check("cap decision is explained in the log", "beyond max-releases=2" in r2.stdout, r2.stdout[:300]) + +print("\n-- failures are isolated, not fatal --") +with tempfile.TemporaryDirectory() as tmp: + env, rt, data, r1, r2, r3 = do(tmp, dlfail=["2.0"], artfail=[9100], artnodocs=[9001]) + check("release gather still exits 0", r2.returncode == 0, r2.stderr) + check("artifact gather still exits 0", r3.returncode == 0, r3.stderr) + got = sorted(os.listdir(f"{rt}/gather")) + check("a failed release download does not block the others", "3.0.zip" in got and "1.0.zip" in got) + check("the failed release is absent", "2.0.zip" not in got, got) + check("failed download is warned", "::warning::release 2.0" in r2.stdout, r2.stdout[-400:]) + check("failed release is NOT left in the cache", "201.zip" not in os.listdir(f"{rt}/relcache")) + check("artifact download failure warned", "artifact 9100 download failed" in r3.stdout, r3.stdout[-500:]) + check("artifact without docs.zip warned distinctly", "contains no docs.zip" in r3.stdout, r3.stdout[-500:]) + check("main falls back to the seed release when its artifact is bad", "main.zip" in got, got) + +print("\n-- empty repo: no releases, no artifacts, no PRs --") +with tempfile.TemporaryDirectory() as tmp: + env, rt, data = setup(tmp, [], [], []) + exprs = {"github.token": "t", "inputs.max-releases": "0", "runner.temp": rt} + a = run("Select releases to publish", env, exprs) + b = run("Gather release artifacts", env, exprs) + c = run("Gather branch CI artifacts", env, exprs) + check("all three steps survive an empty repo", + a.returncode == 0 and b.returncode == 0 and c.returncode == 0, + a.stderr + b.stderr + c.stderr) + check("nothing staged", os.listdir(f"{rt}/gather") == []) + +print("\n-- default-branch fallback chain --") +import subprocess + +def ensure(tmp, *, gathered=None, cached=None, in_site=None): + """Run Ensure-default-branch-zip with a chosen rung populated.""" + env, rt, data = setup(tmp, [], [], []) + ex = {"github.token": "t", "runner.temp": rt} + env["PAGES_URL"] = in_site or "http://127.0.0.1:1" # unroutable => the curl rung fails + for path, marker in ((f"{rt}/gather/main.zip", gathered), + (f"{rt}/defaultcache/default.zip", cached)): + if marker: + os.makedirs(os.path.dirname(path), exist_ok=True) + subprocess.run([f"{data}/mkzip.sh", path, marker], check=True) + return run("Ensure default-branch zip", env, ex), rt, env + +with tempfile.TemporaryDirectory() as tmp: + r, rt, env = ensure(tmp, gathered="fresh") + check("a gathered main.zip wins", r.returncode == 0 and "supplied by an earlier rung" in r.stdout, r.stdout + r.stderr) + check("a fresh main.zip is written to the cache dir", os.path.exists(f"{rt}/defaultcache/default.zip")) + out = open(env["GITHUB_OUTPUT"]).read() + check("a fresh main.zip requests a cache save", "cache-key=mvs-default-v1-" in out, out) + +with tempfile.TemporaryDirectory() as tmp: + r, rt, _ = ensure(tmp, cached="cached") + check("falls back to the Actions cache when CI artifact is gone", + r.returncode == 0 and "restored from the Actions cache" in r.stdout, r.stdout + r.stderr) + check("the cached zip is staged", os.path.exists(f"{rt}/gather/main.zip")) + +with tempfile.TemporaryDirectory() as tmp: + r, rt, _ = ensure(tmp) + check("hard-fails when no rung supplies the default branch", r.returncode != 0) + check("the failure names every rung tried", "the Actions cache" in r.stdout and "deploy aborted" in r.stdout, r.stdout) + +with tempfile.TemporaryDirectory() as tmp: + # Same bytes already cached => no new immutable cache entry requested. + env, rt, data = setup(tmp, [], [], []) + ex = {"github.token": "t", "runner.temp": rt} + env["PAGES_URL"] = "http://127.0.0.1:1" + os.makedirs(f"{rt}/defaultcache", exist_ok=True) + subprocess.run([f"{data}/mkzip.sh", f"{rt}/gather/main.zip", "same"], check=True) + subprocess.run([f"{data}/mkzip.sh", f"{rt}/defaultcache/default.zip", "same"], check=True) + import hashlib + h = hashlib.sha256(open(f"{rt}/gather/main.zip","rb").read()).hexdigest()[:32] + open(f"{rt}/defaultcache/default.sha","w").write(h) + r = run("Ensure default-branch zip", env, ex) + check("an unchanged default branch requests no cache save", + r.returncode == 0 and "cache-key=" not in open(env["GITHUB_OUTPUT"]).read(), + open(env["GITHUB_OUTPUT"]).read()) + +print("\n-- extract: zip shape contract and the size guard --") +with tempfile.TemporaryDirectory() as tmp: + env, rt, data = setup(tmp, [], [], []) + ex = {"github.token": "t", "runner.temp": rt} + subprocess.run([f"{data}/mkzip.sh", f"{rt}/gather/main.zip", "main"], check=True) + subprocess.run([f"{data}/mkzip.sh", f"{rt}/gather/1.0.zip", "onepointoh"], check=True) + open(f"{rt}/gather/broken.zip", "w").write("not a zip") + # Two roots => malformed, must warn and skip rather than guess. + d2 = os.path.join(tmp, "two"); os.makedirs(f"{d2}/a"); os.makedirs(f"{d2}/b") + subprocess.run(["zip", "-rq", f"{rt}/gather/tworoots.zip", "a", "b"], cwd=d2, check=True) + r = run("Extract artifacts into site", env, ex) + got = sorted(os.listdir(f"{rt}/site")) + check("extract exits 0 despite malformed zips", r.returncode == 0, r.stderr) + check("only well-formed zips become version dirs", got == ["1.0", "main"], got) + check("_sources is no longer published into the site", "_sources" not in got, got) + check("an unreadable zip warns", "could not unzip" in r.stdout, r.stdout) + check("a two-root zip warns", "exactly one top-level directory" in r.stdout, r.stdout) + check("the site size is reported", "assembled site:" in r.stdout, r.stdout) + check("a small site does not trip the 1 GB warning", + "approaching the Pages 1 GB limit" not in r.stdout) + check("a small site is not packed just to measure it", + "packed artifact:" not in r.stdout, r.stdout) + +# The guard's whole point is the warning, and it compares the PACKED size — measuring +# the tree with du would cry wolf, since HTML/JS packs ~3x (this repo's own site is +# 734 MiB on disk and 223 MiB deployed). Dial the thresholds down rather than build a +# multi-gigabyte fixture. +with tempfile.TemporaryDirectory() as tmp: + env, rt, data = setup(tmp, [], [], []) + ex = {"github.token": "t", "runner.temp": rt} + subprocess.run([f"{data}/mkzip.sh", f"{rt}/gather/main.zip", "main"], check=True) + env["SIZE_PROBE_BYTES"] = "1" # always pack + env["SIZE_WARN_BYTES"] = "1" # always warn + r = run("Extract artifacts into site", env, ex) + check("past the probe threshold the tree is actually packed", + "packed artifact:" in r.stdout, r.stdout) + check("past the warn threshold a warning is emitted", + "::warning title=Docs site approaching the Pages 1 GB limit" in r.stdout, r.stdout) + check("the warning names max-releases as the lever", "max-releases" in r.stdout, r.stdout) + +with tempfile.TemporaryDirectory() as tmp: + env, rt, data = setup(tmp, [], [], []) + ex = {"github.token": "t", "runner.temp": rt} + subprocess.run([f"{data}/mkzip.sh", f"{rt}/gather/main.zip", "main"], check=True) + env["SIZE_PROBE_BYTES"] = "1" # pack it... + env["SIZE_WARN_BYTES"] = str(10**12) # ...but it is nowhere near the cap + r = run("Extract artifacts into site", env, ex) + check("a packed site under the cap reports but does not warn", + "packed artifact:" in r.stdout and "approaching the Pages 1 GB limit" not in r.stdout, + r.stdout) + +print(f"\n{'FAILURES: ' + ', '.join(fails) if fails else 'all harness checks passed'}") +sys.exit(1 if fails else 0)