diff --git a/.github/workflows/_lint.yml b/.github/workflows/_lint.yml index 2abfb14..c101dbd 100644 --- a/.github/workflows/_lint.yml +++ b/.github/workflows/_lint.yml @@ -1,9 +1,16 @@ on: workflow_call: +# Declared here, not inherited: a reusable workflow's token is whatever the caller +# grants, and lint needs nothing but the checkout (plus the releases API, which is +# public read). +permissions: + contents: read + jobs: lint: runs-on: ubuntu-latest + timeout-minutes: 10 steps: - uses: actions/checkout@v7 @@ -20,14 +27,24 @@ jobs: # Two errors are suppressed: actionlint's `job` context schema is stale and does not # know `job.workflow_repository` / `job.workflow_sha`, both of which are real and # documented, and which publish-gh-pages.yml depends on to version-match assemble.mjs. + # + # PINNED, and checked. This is the only binary the repo executes that it did not + # build, and resolving `releases/latest` at run time meant every lint run took + # whatever bytes that tag pointed at — while `astral-sh/setup-uv` two files over + # is pinned to a full version with a comment explaining why. Renovate bumps + # ACTIONLINT_VERSION (see renovate.json); it cannot compute the new digest, so + # ACTIONLINT_SHA256 must be updated in the SAME PR. Until it is, the checksum + # fails and the job goes red — which is the point: it fails closed, not open. - name: Lint workflow files env: GH_TOKEN: ${{ github.token }} + ACTIONLINT_VERSION: 1.7.12 + ACTIONLINT_SHA256: 8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8 run: | set -euo pipefail - ver=$(gh api repos/rhysd/actionlint/releases/latest --jq .tag_name | tr -d v) - gh release download "v$ver" --repo rhysd/actionlint \ - -p "actionlint_${ver}_linux_amd64.tar.gz" -O "$RUNNER_TEMP/al.tgz" + gh release download "v$ACTIONLINT_VERSION" --repo rhysd/actionlint \ + -p "actionlint_${ACTIONLINT_VERSION}_linux_amd64.tar.gz" -O "$RUNNER_TEMP/al.tgz" + echo "$ACTIONLINT_SHA256 $RUNNER_TEMP/al.tgz" | sha256sum -c - tar xzf "$RUNNER_TEMP/al.tgz" --no-same-owner -C "$RUNNER_TEMP" actionlint "$RUNNER_TEMP/actionlint" -color \ -ignore 'property "workflow_(repository|sha)" is not defined' diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml index 6672e08..e6723d0 100644 --- a/.github/workflows/_test.yml +++ b/.github/workflows/_test.yml @@ -1,9 +1,14 @@ on: workflow_call: +# Declared here, not inherited — see _lint.yml. The tests are pure Node + Python. +permissions: + contents: read + jobs: test: runs-on: ubuntu-latest + timeout-minutes: 15 steps: - uses: actions/checkout@v7 - name: Unit tests (widget logic + make-switcher parity) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c0b13e0..7f2acd0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,6 +9,13 @@ name: CI # PR's critical path and a Pages failure never shows as a red check against an author who # cannot act on it. This workflow therefore needs no elevated permissions, and its `name:` # below is load-bearing — publish.yml matches on it. +# The token this workflow's own jobs get, and the ceiling for the reusable workflows it +# calls: nothing here builds or tests with anything but read access. Declared at the top +# rather than left to the repo/org default, which may be write-all. The `release` job +# overrides it upward for the one job that needs to create a Release. +permissions: + contents: read + on: pull_request: push: diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 50e1000..e22cb16 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -12,6 +12,16 @@ on: required: false default: docs/_build/html type: string + base-path: + description: >- + URL path the SITE is served at, without the version segment. Empty (the + default) means /, which is where a project Pages site lives + (https://ORG.github.io/REPO/). Set it to "/" for a custom domain (CNAME) or an + ORG.github.io repo — both serve from the root, where a / prefix would + 404 every root-absolute asset. + required: false + default: "" + type: string # REUSABLE docs build — the UNPRIVILEGED half, shared across MyST projects. Builds # the docs at the versioned BASE_URL and uploads this build's `docs` artifact @@ -37,6 +47,10 @@ permissions: jobs: build: runs-on: ubuntu-latest + # Generous: build-command is the consumer's, and a docs build with a Python env to + # solve can legitimately take a while. It only has to beat the 6-hour default, so a + # hung build fails within the hour instead of holding a runner all day. + timeout-minutes: 30 steps: # fetch-depth: 0 brings the full history *and tags* — a build-command may # derive a version from `git describe` (this repo's docs do, to pin the @@ -62,26 +76,64 @@ jobs: # Version name = the site sub-dir this build is served at, and the BASE_URL it # must be built with: pr- for PRs, otherwise the ref name (the default branch, - # or a tag without `/`). No sanitisation — every version name is URL/path-safe. + # or a tag without `/`). # # This is the OTHER half of the contract, and it is not passed anywhere: the # gather independently re-derives the same name for each source it stages (a # release's tag, a PR's number, the repo's default_branch) and unpacks the zip at # site/. Both sides compute the same string from the same facts, so # this rule and assemble.mjs's dest rule must change together. - - name: Compute version name + # + # The name is VALIDATED, not assumed clean. In THIS repo it is clean by + # construction (`tags: ['*']` never builds a `/`-tag), but this is a reusable + # workflow: a consumer triggering it on `push: branches: ['**']` would let anyone + # who can push a branch choose the string — and git accepts `$( )`, backticks and + # quotes in a ref name. Hence both halves here: the ref comes through `env:` so + # the shell never parses it as script text, and the character set is then checked + # so it stays a safe URL path segment and site dir. + - name: Compute version name and BASE_URL id: ver + env: + EVENT: ${{ github.event_name }} + PR_NUMBER: ${{ github.event.pull_request.number }} + REF_NAME: ${{ github.ref_name }} + BASE_PATH: ${{ inputs.base-path }} + REPO_NAME: ${{ github.event.repository.name }} run: | set -euo pipefail - if [ "${{ github.event_name }}" = pull_request ]; then - name="pr-${{ github.event.pull_request.number }}" + if [ "$EVENT" = pull_request ]; then + name="pr-$PR_NUMBER" else - name="${{ github.ref_name }}" + name="$REF_NAME" + fi + case "$name" in + "" | "." | "..") ok=false ;; + *[!A-Za-z0-9._-]*) ok=false ;; + *) ok=true ;; + esac + if [ "$ok" != true ]; then + echo "::error::version name '$name' is not usable as a URL path segment \ + — it must be non-empty and contain only A-Z a-z 0-9 . _ - (a '/' in a branch \ + or tag name is the usual cause). Build this ref under a name that is, or \ + exclude it from the workflow's triggers." + exit 1 fi echo "version-name=$name" >> "$GITHUB_OUTPUT" - # BASE_URL must match the versioned sub-path the build is served at, or its - # root-absolute assets 404 — see the version-name contract above. + # BASE_URL must match the versioned sub-path the site is served at, or the + # build's root-absolute assets 404. The site half of it is base-path, which + # defaults to / — right for a project Pages site, wrong for a custom + # domain or an ORG.github.io repo, which serve at the root. Normalise to + # exactly one leading slash and no trailing one, so "/", "/repo" and "repo/" + # all compose the same way. + base="${BASE_PATH:-/$REPO_NAME}" + base="/${base#/}" + base="${base%/}" + echo "BASE_URL=$base/$name" >> "$GITHUB_ENV" + echo "version name: $name BASE_URL: $base/$name" + + # BASE_URL arrives via GITHUB_ENV from the step above (it needs base-path's + # normalisation, which an `env:` expression cannot do). # VERSION_NAME is the bare version token, exported for builds # that need it directly — e.g. a Sphinx conf.py setting pydata-sphinx-theme's # switcher `version_match` — without parsing it back out of BASE_URL. @@ -89,7 +141,6 @@ jobs: # and is run with `eval` so compound commands work. - name: Build docs env: - BASE_URL: /${{ github.event.repository.name }}/${{ steps.ver.outputs.version-name }} VERSION_NAME: ${{ steps.ver.outputs.version-name }} BUILD_COMMAND: ${{ inputs.build-command }} run: | diff --git a/.github/workflows/publish-gh-pages.yml b/.github/workflows/publish-gh-pages.yml index de84248..3bdbf59 100644 --- a/.github/workflows/publish-gh-pages.yml +++ b/.github/workflows/publish-gh-pages.yml @@ -57,6 +57,10 @@ jobs: # not being from a fork, so there is nothing left to branch on here. deploy: runs-on: ubuntu-latest + # The whole deploy: a cold release-zip cache, the assemble, the upload, and up to + # ~6 min of origin verification at the end. 45 leaves room for a slow Pages backend + # while still failing the same day. + timeout-minutes: 45 permissions: contents: read # checkout + read release assets actions: read # gather cross-run docs artifacts @@ -66,12 +70,22 @@ jobs: environment: name: github-pages url: ${{ steps.deployment.outputs.page_url }} - # Cancelling a superseded deploy is safe: every deploy reconstructs the whole site + # QUEUE superseded deploys; do not cancel them. The old `cancel-in-progress: true` + # was argued from artifact correctness — every deploy reconstructs the whole site # from durable sources, so the next one gathers everything the cancelled one would - # have. + # have — and that argument still holds. It is just not the question. Cancelling + # kills the job mid-`deploy-pages`, in the window between the artifact being + # uploaded and the Pages backend finishing with it, which is a backend state we do + # not control and cannot inspect; GitHub's own Pages starter workflow uses `false` + # for exactly this reason, and this workflow already carries a step for detecting a + # wedged origin, so the failure it guards against is not hypothetical here. + # + # The cost is bounded: a concurrency group holds at most ONE pending run, and a + # newer arrival cancels the older PENDING one (before it starts, harmlessly). So a + # burst of pushes costs one extra full-site deploy, not a pile-up. concurrency: group: pages - cancel-in-progress: true + cancel-in-progress: false env: REPO: ${{ github.repository }} # The branch that was actually BUILT — see the header: github.ref is the default @@ -81,16 +95,24 @@ jobs: DEFAULT: ${{ github.event.repository.default_branch }} SEED_TAG: pages-default-seed steps: - # The CONSUMER's repo, at the root cwd: assemble.mjs runs `git tag` here to - # order that repo's deployed versions, so it needs their tags (fetch-depth: 0). + # The CONSUMER's repo, at the root cwd: assemble.mjs runs `git tag` here to order + # that repo's deployed versions. It needs the TAGS, not the history — and this + # runs on every deploy of every consumer, some with years of commits, so ask for + # exactly that rather than `fetch-depth: 0`. (docs.yml deliberately still clones + # in full: a build-command may run `git describe`.) - uses: actions/checkout@v7 with: - fetch-depth: 0 + fetch-depth: 1 + fetch-tags: true # THIS reusable workflow's repo + commit (the `job` context resolves to the # reusable file, not the caller), so assemble.mjs matches the ref the consumer # pinned publish-gh-pages.yml@ to — automatically, no version bump. # Sparse: only assemble/ is needed. + # + # `.mvs` is a RESERVED path in the consumer's workspace (documented as such in + # docs/reference/workflows.md): the checkout lands inside their tree, so a repo + # that has its own `.mvs/` would collide. - name: Fetch assemble.mjs uses: actions/checkout@v7 with: @@ -222,15 +244,25 @@ jobs: if [ -s "$relcache/$assetid.zip" ]; then hits=$((hits + 1)) decision="$decision (cache hit)" - elif gh release download "$tag" --repo "$REPO" -p docs.zip \ - -O "$relcache/$assetid.zip" 2>"$RUNNER_TEMP/relcache-download.err"; then + # Fetch by ASSET ID, not `gh release download -p docs.zip`, so the + # bytes match the id they are cached under. The cache is id-addressed + # precisely because an id names immutable bytes — but a re-cut release + # between the listing above and this download would have handed the NEW + # docs.zip to the OLD id, cached for every later deploy to trust. + elif gh api "repos/$REPO/releases/assets/$assetid" \ + -H 'Accept: application/octet-stream' \ + > "$relcache/$assetid.zip" 2>"$RUNNER_TEMP/relcache-download.err" \ + && [ -s "$relcache/$assetid.zip" ] \ + && [ "$(head -c 2 "$relcache/$assetid.zip")" = PK ]; then downloads=$((downloads + 1)) decision="$decision (downloaded)" else - # Leave no truncated zip behind for the next run's cache to trust. + # Leave nothing behind for the next run's cache to trust: a redirect to + # a file means a FAILED request still wrote an error body, so the exit + # status alone is not enough — hence the zip magic check above. rm -f "$relcache/$assetid.zip" decision="$decision (DOWNLOAD FAILED — skipped)" - echo "::warning::release $tag: docs.zip download failed — skipping. $(head -1 "$RUNNER_TEMP/relcache-download.err" 2>/dev/null)" >> "$warnings" + echo "::warning::release $tag: docs.zip (asset $assetid) download failed — skipping. $(head -1 "$RUNNER_TEMP/relcache-download.err" 2>/dev/null)" >> "$warnings" fi if [ -s "$relcache/$assetid.zip" ]; then cp "$relcache/$assetid.zip" "$RUNNER_TEMP/gather/$dest.zip" @@ -288,6 +320,7 @@ jobs: # Fork approval is an API call per cross-repo PR, so settle it here and pass # the answer in — assemble.mjs stays free of network IO. prs="$RUNNER_TEMP/prs.tsv"; : > "$prs" + pr_limit=200 while IFS=$'\t' read -r num sha cross; do [ -n "$num" ] || continue approved=true @@ -296,9 +329,17 @@ jobs: -q 'any(.[]; .context=="preview-approved" and .state=="success")') fi printf '%s\t%s\t%s\n' "$num" "$sha" "$approved" >> "$prs" - done < <(gh pr list --repo "$REPO" --state open --limit 200 \ + done < <(gh pr list --repo "$REPO" --state open --limit "$pr_limit" \ --json number,headRefOid,isCrossRepository \ -q '.[] | [.number, .headRefOid, .isCrossRepository] | @tsv') + # A truncated listing is otherwise indistinguishable from a repo that happens + # to have exactly $pr_limit open PRs: the missing ones simply get no preview, + # with nothing in the log to explain it. + if [ "$(wc -l < "$prs")" -ge "$pr_limit" ]; then + echo "::warning::more than $pr_limit open PRs — only the first $pr_limit were \ + considered for previews, so some open PRs will have none. max-prs caps how many \ + are published; this caps how many are even looked at." + fi # The API serves an artifact as a zip-of-the-artifact, with docs.zip inside. @@ -430,29 +471,50 @@ jobs: fi rm -rf "$tmp" done - # Report the assembled size. Pages rejects an artifact over 1 GB and - # deploy-pages warns when the upload exceeds it; `max-releases` / `max-prs` are - # the levers (see docs/how-to/keep-the-site-small.md). This is the tree on - # disk, so it reads high — the cap applies to the compressed upload, and - # HTML/JS packs around 3x. - echo "assembled site: $(du -sh "$RUNNER_TEMP/site" | cut -f1) on disk across \ - $(find "$RUNNER_TEMP/site" -mindepth 1 -maxdepth 1 -type d | wc -l) version dir(s)" # `generate` orders versions, writes switcher.json + index.html, and prints the # stable-alias source dir (the newest release), or "" when no release is deployed yet. # Hard-fails if the default branch is absent (Ensure step guarantees the zip, but - # the extract could still fail if the zip was malformed). + # the extract could still fail if the zip was malformed). The last thing to touch + # the tree, so it also stamps the deploy and reports the final size. - name: Generate switcher.json and stable alias run: | set -euo pipefail stable_src=$(node "$GITHUB_WORKSPACE/.mvs/assemble/assemble.mjs" generate \ - --site-dir "$RUNNER_TEMP/site" --base-url "$PAGES_URL" --required "$DEFAULT") + --site-dir "$RUNNER_TEMP/site" --base-url "$PAGES_URL" \ + --default-branch "$DEFAULT" --required "$DEFAULT") # An `&& ln … || true` one-liner here would also swallow a FAILED symlink, not # just the empty-stable_src case — and a missing stable/ alias is silent damage. if [ -n "$stable_src" ]; then ln -s "$stable_src" "$RUNNER_TEMP/site/stable" fi + # The deploy stamp the verify step below polls for. It has to change on EVERY + # deploy, which switcher.json does not: that file lists the version set, so an + # ordinary docs edit on the default branch reproduces it byte-for-byte and a + # stale origin would compare equal. run + attempt + the clock are unique per + # deploy by construction. GITHUB_RUN_* are default runner env vars, so this + # needs no expression. + # + # Safe at the site root: assemble.mjs's discoverVersions returns DIRECTORIES + # only, so a file here is never mistaken for a version. + printf 'run=%s\nattempt=%s\nutc=%s\n' \ + "$GITHUB_RUN_ID" "$GITHUB_RUN_ATTEMPT" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + > "$RUNNER_TEMP/site/deploy-id.txt" + + # The size that actually gets uploaded: -L follows stable/, which + # upload-pages-artifact materialises as a full COPY of the newest release + # (it tars with --dereference). Reported after the alias exists, because a + # number that omits the largest single version is worse than no number on a + # metric whose whole job is to warn before the 1 GB cap. The cap applies to + # the compressed upload and HTML/JS packs around 3x, so this reads high. + echo "assembled site: $(du -shL "$RUNNER_TEMP/site" | cut -f1) on disk across \ + $(find "$RUNNER_TEMP/site" -mindepth 1 -maxdepth 1 -type d | wc -l) version dir(s)" + if [ -n "$stable_src" ]; then + echo " ...of which stable/ is a second copy of $stable_src \ + ($(du -shL "$RUNNER_TEMP/site/stable" | cut -f1)) — see docs/how-to/keep-the-site-small.md" + fi + - name: Upload Pages artifact uses: actions/upload-pages-artifact@v5 with: @@ -469,11 +531,19 @@ jobs: # deploy-pages reports success once the deployment RECORD flips active, even if the # Pages backend fails to update the SERVED origin — a wedged site can serve a stale # artifact indefinitely while every deploy reports OK. This turns that into a red - # check instead of silently serving old docs: confirm the live switcher.json - # byte-matches the one just assembled, polling up to ~6 min because a re-serve can - # take minutes to reach the origin. The loop exits the instant it matches, so a - # fast deploy isn't slowed. On failure, dump the origin headers (a frozen - # etag/last-modified is the wedge signature). + # check instead of silently serving old docs, polling up to ~6 min because a + # re-serve can take minutes to reach the origin. The loop exits the instant both + # files match, so a fast deploy isn't slowed. On failure, dump the origin headers + # (a frozen etag/last-modified is the wedge signature). + # + # TWO files, and deploy-id.txt is the one that does the work. switcher.json only + # changes when the VERSION SET changes, so on the commonest deploy of all — a docs + # edit on the default branch — it is byte-identical to what is already served and + # compares equal on attempt 1 whether or not the origin moved. That is precisely + # the case where a wedge goes unnoticed. The stamp is unique per deploy, so it can + # only match if these bytes are the ones being served. switcher.json stays as the + # secondary check: it is the file the widget actually reads, so a mismatch there + # means a partially-served deploy even when the stamp is current. - name: Verify the deployed origin matches what we assembled id: verify env: @@ -483,27 +553,38 @@ jobs: set -euo pipefail [ -n "$PAGE_URL" ] || { echo "::error::deploy-pages did not report a page_url"; exit 1; } local_json="$SITE_DIR/switcher.json" - [ -f "$local_json" ] || { echo "::error::no switcher.json at $local_json"; exit 1; } + local_id="$SITE_DIR/deploy-id.txt" + for f in "$local_json" "$local_id"; do + [ -f "$f" ] || { echo "::error::nothing to verify against: no $f"; exit 1; } + done # Command substitution strips trailing newlines from both sides, so the - # comparison is robust to a curl/file trailing-newline difference. - want=$(cat "$local_json") - url="${PAGE_URL%/}/switcher.json" + # comparisons are robust to a curl/file trailing-newline difference. + want_json=$(cat "$local_json") + want_id=$(cat "$local_id") + base="${PAGE_URL%/}" timeout_s=360 deadline=$(( $(date +%s) + timeout_s )) attempt=0 while :; do attempt=$((attempt + 1)) - got=$(curl -fsS "$url?cb=$(date +%s%N)" || true) # cache-bust → hit the origin - if [ -n "$got" ] && [ "$got" = "$want" ]; then - echo "origin matches the assembled switcher.json (attempt $attempt)" + # cache-bust → hit the origin, not a CDN copy + cb=$(date +%s%N) + got_id=$(curl -fsS "$base/deploy-id.txt?cb=$cb" || true) + got_json=$(curl -fsS "$base/switcher.json?cb=$cb" || true) + if [ "$got_id" = "$want_id" ] && [ "$got_json" = "$want_json" ]; then + echo "origin is serving this deploy (attempt $attempt)" exit 0 fi if [ "$(date +%s)" -ge "$deadline" ]; then - echo "::error::live switcher.json still does not match the assembled one after ${timeout_s}s — Pages origin likely stale/wedged" + stale="deploy-id.txt" + [ "$got_id" = "$want_id" ] && stale="switcher.json" + echo "::error::live $stale still does not match the assembled one after ${timeout_s}s — Pages origin likely stale/wedged" echo "--- origin headers (frozen etag/last-modified ⇒ wedged) ---" - curl -sI "$url?cb=$(date +%s%N)" | grep -iE '^(HTTP/|etag|last-modified|age)' | tr -d '\r' - echo "--- assembled ($local_json) ---"; printf '%s\n' "$want" - echo "--- live ($url) ---"; printf '%s\n' "$got" + curl -sI "$base/deploy-id.txt?cb=$(date +%s%N)" | grep -iE '^(HTTP/|etag|last-modified|age)' | tr -d '\r' + echo "--- assembled deploy-id ---"; printf '%s\n' "$want_id" + echo "--- live deploy-id ---"; printf '%s\n' "$got_id" + echo "--- assembled switcher.json ---"; printf '%s\n' "$want_json" + echo "--- live switcher.json ---"; printf '%s\n' "$got_json" exit 1 fi sleep 5 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 1812f53..ead0747 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -4,7 +4,15 @@ name: Publish # nothing but call the engine (publish-gh-pages.yml) when CI finishes. This repo's copy is # IDENTICAL to a consumer's but for the `uses:` path — theirs pins # DiamondLightSource/myst-version-switcher-plugin/.github/workflows/publish-gh-pages.yml@, -# this one uses the local file so the working tree is what gets tested. +# this one uses the local file, so what deploys is main's tree rather than a pinned +# release tag. +# +# Note what that does NOT mean. Under `workflow_run` GitHub always runs the DEFAULT +# BRANCH's copy of this file, and `uses: ./…` resolves against that same commit — so a +# PR that changes publish-gh-pages.yml or assemble.mjs is never exercised by a real +# deploy before it merges. Engine changes are covered by test/workflow-harness/, which +# loads the steps out of the YAML and runs them; there is no way to make workflow_run +# run a PR's version of a workflow. # # Publishing lives here rather than as a job in ci.yml because it is O(the whole site) and # independent of the change, so nesting it put a large constant on every PR's critical diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 260cef3..43133d7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -22,9 +22,16 @@ name: Release on: workflow_call: +# The one workflow here that needs a write token. Declared so the job cannot silently +# hold more than it uses; the CALLER must grant `contents: write` as well, since a +# reusable workflow can only ever narrow what it is given. +permissions: + contents: write + jobs: release: runs-on: ubuntu-latest + timeout-minutes: 15 steps: # Every artifact produced earlier in the run, merged into dist/ (docs.zip, and # for this repo version-switcher.mjs). @@ -41,11 +48,23 @@ jobs: TAG: ${{ github.ref_name }} run: | set -euo pipefail - # Flag PEP 440-style prerelease tags — a/b/rc following a digit, e.g. - # 1.0a1 / 2.0rc1 (mirrors assemble.mjs's isPrerelease). Anchoring on the - # digit keeps tags that merely contain those letters out. + # Flag a prerelease tag — a marker following a digit (with an optional + # separator), so 1.0a1 / 2.0rc1 / 1.1.0-beta.1 / 2.0.0-rc.1 are prereleases + # while release-1.0 and 1.0-candidate are not. + # + # `markers` is assemble.mjs's MARKERS list VERBATIM, and the two are held + # together by a drift test (test/workflow-harness/test_release_prerelease.py) + # that runs a fixture list through both implementations — a comment alone had + # already let them diverge (this test was case-sensitive; the regex is not). + # There is no shared implementation to call: this workflow runs in the + # consumer's repo and deliberately checks nothing out. + markers='alpha|beta|preview|pre|rc|dev|a|b|c' prerelease="" - case "$TAG" in *[0-9]a*|*[0-9]b*|*[0-9]rc*) prerelease="--prerelease" ;; esac + # grep -E has no lookahead, so the trailing [^a-z] is spelled as a consumed + # character (or end-of-string); -i covers both cases, which `case` globs did not. + if printf '%s' "$TAG" | grep -Eqi "[0-9][._-]?($markers)([^a-z]|\$)"; then + prerelease="--prerelease" + fi if gh release view "$TAG" --repo "$REPO" >/dev/null 2>&1; then # The Release already exists (e.g. published from the UI on a mutable repo). diff --git a/CLAUDE.md b/CLAUDE.md index 3d94d1a..26f6947 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -81,8 +81,9 @@ set). There is NO size guard in the workflow. `actions/deploy-pages` already compares the uploaded artifact against 1 GB and warns (`ONE_GIGABYTE` in `src/internal/deployment.js`, reading the artifact's stored/compressed size), so packing the tree to re-measure it was -duplicated work with a worse number. The engine just reports `du -sh` of the assembled -tree; the recovery — lower the caps, re-run, nothing is deleted — lives in +duplicated work with a worse number. The engine just reports `du -shL` of the assembled +tree — with -L, and AFTER the `stable/` symlink exists, because upload-pages-artifact +dereferences it and the newest release is therefore uploaded twice; the recovery — lower the caps, re-run, nothing is deleted — lives in docs/how-to/keep-the-site-small.md and in the tutorial's copy-paste `publish.yml`. Selection is `assemble.mjs`'s `selectReleases` (pure, unit-tested), ranked by @@ -181,6 +182,12 @@ the repo's `default_branch`). Nothing is threaded between them — `docs.yml` ha `version-name` output any more (it existed only for the deleted injection step). The two rules must change together. +The name is now VALIDATED (`[A-Za-z0-9._-]`, non-empty) rather than assumed clean: it is +clean by construction HERE, but docs.yml is reusable and a consumer can trigger it on any +ref, and git allows `$( )` and backticks in a ref name. Related invariant, enforced by +`test_shape.py` across every workflow: **no `${{ }}` inside any `run:` body** — context +values come through `env:` only. + ### The PR gets a link, never a red check A successful deploy posts a `docs-preview` commit status on `workflow_run.head_sha` (or, for a dispatched fork preview, the SHA the Approve step @@ -206,6 +213,11 @@ they live in `if:` expressions the gather harness can't reach. fork's PR triggered it (pwn-request). Fork builds reach the site only via a maintainer dispatching `publish.yml` with `pr`, which pins approval to that head SHA. +### PR previews are IN the public switcher — deliberately +`switcherStruct` maps every discovered dir, `pr-` included, so open PR numbers show in +the dropdown of the live site. That is intended (a reviewer can jump to a preview from any +page), not an oversight — don't "fix" it without asking. + ### Self-referencing assemble.mjs (no separate action) `publish.yml` sparse-checks-out **this** repo's `assemble/` at `job.workflow_repository` + `job.workflow_sha` — the `job` context resolves to the file that defines the running @@ -231,14 +243,24 @@ Without this, assets and links break under the versioned GitHub Pages sub-path. version name is computed in `docs.yml` and re-derived identically by the gather as the `site/` dir — see "The version name is computed twice, never passed". +The `/` half is `docs.yml`'s **`base-path`** input (default: `/`). A custom +domain or an `ORG.github.io` repo serves at the ROOT, so those pass `base-path: "/"` — +otherwise switcher.json (which takes its URLs from the Pages API, and so follows the +CNAME automatically) would be right while every page it links to 404s its assets. + ### `assemble` degrades gracefully on first deploy With no releases and no other branches, `assemble` produces a single-entry `switcher.json` for the current build and an `index.html` redirecting to it, rather than failing. The "preferred" version (the redirect target, flagged `preferred: true` in switcher.json, rendered with a ★) is the newest deployed -non-prerelease tag, falling back to `main`/`master`. Prerelease detection mirrors -`release.yml` (an `a`/`b`/`rc` marker following a digit, PEP 440 style — so -`1.0a1`/`2.0rc1` are prereleases but `release-1.0` is not). +non-prerelease tag, falling back to the default branch (threaded in as +`generate --default-branch`, so it need not be `main`/`master`). Prerelease detection +mirrors `release.yml` — one marker list (`MARKERS`) covering PEP 440 and hyphenated +semver, following a digit with an optional separator, so `1.0a1`/`2.0rc1`/`1.1.0-beta.1` +are prereleases while `release-1.0` and `1.0-candidate` are not. The two implementations +are held together by `test/workflow-harness/test_release_prerelease.py`, not by comment. +Tag ORDER is `compareTags`, not `git tag --sort=-v:refname`, which ranks a hyphenated +prerelease above its own release. ### `stable/` alias When a non-prerelease release is deployed, the site serves a `stable/` symlink @@ -268,8 +290,11 @@ Two top-level workflows: `ci.yml` builds, `publish.yml` listens for it finishing ONLY. **One** job, `deploy`: sparse-checks out `assemble.mjs` at `job.workflow_sha` so it matches the pinned ref (see "Self-referencing assemble.mjs"), runs inline gather + extract, then `assemble.mjs generate`, `upload-pages-artifact`, - `deploy-pages` and the origin verify — carrying the `github-pages` environment, perms, - and `concurrency: {group: pages, cancel-in-progress: true}`. No event branching: the + `deploy-pages` and the origin verify (which polls `site/deploy-id.txt`, a per-deploy + stamp — `switcher.json` alone can't detect a wedge, since it is byte-identical whenever + the version set is unchanged) — carrying the `github-pages` environment, perms, + and `concurrency: {group: pages, cancel-in-progress: false}` — a superseded deploy is + QUEUED, never killed mid-`deploy-pages`. No event branching: the caller already established success + non-fork. Inputs are `pr`, `max-releases` and `max-prs`. @@ -277,8 +302,8 @@ Sub-workflows of `ci.yml`: - `_lint.yml` — biome - `_test.yml` — `npm test` - `docs.yml` — **reusable build, parameterised for cross-repo reuse.** Compute the - version name (`pr-` / default-branch / tag) → run `build-command` (required - input) with `BASE_URL` + `VERSION_NAME` set (the latter for builds that need the + version name (`pr-` / default-branch / tag) and `BASE_URL` (from `base-path`) → + run `build-command` (required input) with `BASE_URL` + `VERSION_NAME` set (the latter for builds that need the bare token, e.g. a Sphinx conf.py setting pydata's switcher `version_match`) → pack `docs.zip` (single root dir `html/`, staged so any `html-dir` works) → upload the `docs` artifact. No deploy; `contents: read` only. Installs uv unconditionally and relies on diff --git a/assemble/assemble.mjs b/assemble/assemble.mjs index fe0052c..90d0a1e 100644 --- a/assemble/assemble.mjs +++ b/assemble/assemble.mjs @@ -21,7 +21,8 @@ * (the default branch's, plus each eligible open PR's), ready for * `xargs -0 -n 3`; a decision for every candidate goes to stderr. * - * node assemble.mjs generate --site-dir --base-url [--required ] + * node assemble.mjs generate --site-dir --base-url + * [--default-branch ] [--required ] * → write switcher.json + index.html into ; print the stable-alias * source dir (the newest deployed release) on stdout, or nothing. Also * exit-1s if a --required branch is absent from the site. @@ -80,6 +81,26 @@ export function cacheKey(rows = []) { return `${RELZIPS_CACHE_PREFIX}${digest}`; } +/** + * A site-size cap (`max-releases` / `max-prs`) as a non-negative integer; 0 means + * unlimited. Throws on anything else. + * + * This used to be `Number(x) || 0`, which points the wrong way: `Number("3O")` is NaN + * and `|| 0` turns NaN into 0 — so a typo in a consumer's publish.yml silently + * DISABLED the cap that exists to stop a 1 GB deploy failure. A cap you cannot read is + * not a cap of 0. + */ +export function parseCap(value, name = "cap") { + if (value === undefined || value === null || value === "") return 0; + const text = String(value).trim(); + if (!/^\d+$/.test(text)) { + throw new Error( + `${name} must be a non-negative integer (0 = unlimited), got ${JSON.stringify(value)}`, + ); + } + return Number(text); +} + /** Run a git command and return its non-empty stdout lines. */ function gitLines(args) { const out = execFileSync("git", args, { encoding: "utf8" }); @@ -100,27 +121,36 @@ export function discoverVersions(siteDir) { } /** - * Tags newest-first (semver-aware), matching `git tag -l --sort=-v:refname`. + * Tags newest-first, ordered by `compareTags` (NOT `git tag --sort=-v:refname`). * Tags containing `/` are dropped: the build trigger (`tags: ['*']`) never builds * them, so they have no matching `BASE_URL` build and would only create nested * site dirs. Every other tag is used verbatim as its `site/` dir name. + * + * git's version sort is not usable here: without a `versionsort.suffix` entry per + * marker it ranks `1.1.0-beta.1` ABOVE `1.1.0`, and that list would have to be kept + * in step with `isPrerelease` by hand. Sorting here keeps "what order" and "what + * counts as a prerelease" in one module, sharing one marker list. */ export function getSortedTags() { - return gitLines(["tag", "-l", "--sort=-v:refname"]).filter( - (tag) => !tag.includes("/"), - ); + return orderTags(gitLines(["tag", "-l"]).filter((tag) => !tag.includes("/"))); } /** - * Order the gathered versions: `master`, `main`, then tags newest-first, then any - * leftover directories (e.g. PR previews) alphabetically. `tags` must already be - * newest-first; `builds` are the directory names under `site/`. + * Order the gathered versions: the default branch, then `master`/`main` if they are + * present and distinct from it (a repo mid-rename may carry both), then tags + * newest-first, then any leftover directories (e.g. PR previews) alphabetically. + * `tags` must already be newest-first; `builds` are the directory names under `site/`. + * + * `defaultBranch` is optional so callers that have no opinion keep the historical + * `master`/`main` behaviour: a consumer whose default branch is `develop` would + * otherwise land it in the leftover bucket, sorted among the `pr-` previews. */ -export function orderVersions(builds, tags) { +export function orderVersions(builds, tags, defaultBranch) { const remaining = new Set(builds); const versions = []; - for (const version of ["master", "main", ...tags]) { + for (const version of [defaultBranch, "master", "main", ...tags]) { + if (!version) continue; if (remaining.has(version)) { versions.push(version); remaining.delete(version); @@ -139,8 +169,10 @@ export function orderVersions(builds, tags) { /** * Comparator: newest first, undated last. Timestamp ties break on `keyOf` ascending * — arbitrary, but a total order, which is what matters: an unstable ranking would - * change the published set (and so the cache key) on identical inputs. Shared by the - * release cap and the PR cap so the two cannot drift apart. + * change the published set (and so the cache key) on identical inputs. The ONLY + * "which of these is newest?" rule in this file: the release cap, the PR cap, the + * default-branch artifact and the per-SHA index all reach it through `byDateDesc` or + * `newest`, so none of them can break a tie differently from the others. */ function byDateDesc(keyOf) { return (a, b) => { @@ -181,7 +213,7 @@ export function selectReleases( releases = [], { defaultBranch, seedTag = SEED_TAG, maxReleases = 0 } = {}, ) { - const cap = Number(maxReleases) > 0 ? Number(maxReleases) : 0; + const cap = parseCap(maxReleases, "max-releases"); const rows = releases.map((release) => { const tag = release?.tag_name ?? ""; @@ -260,7 +292,7 @@ export function selectArtifacts( artifacts = [], { defaultBranch, repoId, prs = [], maxPrs = 0 } = {}, ) { - const cap = Number(maxPrs) > 0 ? Number(maxPrs) : 0; + const cap = parseCap(maxPrs, "max-prs"); const live = artifacts.filter((a) => a && !a.expired); const meta = (artifact) => ({ @@ -278,19 +310,20 @@ export function selectArtifacts( .map((a) => ({ date: a?.created_at ?? null, key: String(a?.id), a })) .sort(byDateDesc((x) => x.key))[0].a; - // Newest artifact per head SHA, for the PR lookups. - const bySha = new Map(); + // Newest artifact per head SHA, for the PR lookups — through `newest`, so BOTH + // lookups break a timestamp tie the same way. This loop used to keep whichever + // artifact the API happened to return last, i.e. page order, which is not a rule + // at all: two artifacts created in the same second (a matrix, a re-run) could + // swap the preview between deploys. + const groups = new Map(); for (const artifact of live) { const sha = artifact?.workflow_run?.head_sha; if (!sha) continue; - const best = bySha.get(sha); - if ( - !best || - Date.parse(artifact.created_at) >= Date.parse(best.created_at) - ) { - bySha.set(sha, artifact); - } + const list = groups.get(sha); + if (list) list.push(artifact); + else groups.set(sha, [artifact]); } + const bySha = new Map([...groups].map(([sha, list]) => [sha, newest(list)])); const rows = []; @@ -346,26 +379,96 @@ export function selectArtifacts( } /** - * Is `tag` a prerelease? Mirrors `release.yml`'s test (a PEP 440-style `a`, - * `b`, or `rc` marker *following a digit*, e.g. `1.0a1`/`2.0rc1`) so "stable" - * means the same thing repo-wide. Anchoring on the digit keeps tags that merely - * contain those letters (`release-1.0`, `beta-program`) out of the prerelease set. + * The prerelease markers, longest-first so `preview` is not shadowed by `pre`. + * ONE list, used three ways: to test a tag (`isPrerelease`), to rank a prerelease + * below the release it qualifies (`compareTags`), and — as the identical literal — + * by release.yml's `--prerelease` test, which `test_release_prerelease.py` holds to + * this file. Covers PEP 440 (`1.0a1`, `2.0rc1`, `1.0.dev0`) and the hyphenated + * semver spellings (`1.1.0-beta.1`, `2.0.0-rc.1`). + */ +const MARKERS = "alpha|beta|preview|pre|rc|dev|a|b|c"; + +/** + * A marker anywhere in the tag, immediately after a digit (with an optional `.`, + * `-` or `_` between). The digit anchor keeps tags that merely contain those + * letters (`release-1.0`, `beta-program`) out of the prerelease set; the trailing + * lookahead keeps a longer word (`1.0-candidate`, `1.0-canary`) out of it too. + */ +const PRERELEASE_RE = new RegExp(`\\d[._-]?(${MARKERS})(?![a-z])`, "i"); + +/** The same marker, but anchored at the START of a version's trailing segment. */ +const PRERELEASE_SUFFIX_RE = new RegExp(`^[._-]?(${MARKERS})(?![a-z])`, "i"); + +/** + * Is `tag` a prerelease? Mirrors `release.yml`'s `--prerelease` test so "stable" + * means the same thing repo-wide — the two are held together by a drift test + * (`test/workflow-harness/test_release_prerelease.py`) rather than by comment alone. */ export function isPrerelease(tag) { - return /\d(a|b|rc)/i.test(tag); + return PRERELEASE_RE.test(tag); +} + +/** A tag split into runs of digits and non-digits, for segment-wise comparison. */ +function tagTokens(tag) { + // A leading `v` is a spelling of the same version (`v1.2.3` ≡ `1.2.3`), so drop + // it — but only before a digit, or `version-2` would become `ersion-2`. + return ( + String(tag) + .replace(/^v(?=\d)/i, "") + .match(/\d+|\D+/g) ?? [] + ); +} + +/** + * Compare two tags, NEWEST FIRST (so it sorts descending), with one invariant that + * `git tag --sort=-v:refname` does not give us: **every prerelease of X sorts after + * X**. Digit runs compare numerically (so `1.10` > `1.9`), other runs as strings. + * + * When one tag is the other plus a trailing segment, that segment decides: a + * prerelease marker (`1.1.0` vs `1.1.0-beta.1`) demotes the longer tag, anything + * else (`1.1` vs `1.1.1`) promotes it. Ties fall back to the raw string, so the + * order is total — an unstable ranking would change the published set on identical + * inputs. + */ +export function compareTags(a, b) { + const ta = tagTokens(a); + const tb = tagTokens(b); + for (let i = 0; i < Math.min(ta.length, tb.length); i += 1) { + if (ta[i] === tb[i]) continue; + const na = /^\d+$/.test(ta[i]); + const nb = /^\d+$/.test(tb[i]); + if (na && nb) return Number(tb[i]) - Number(ta[i]); + return ta[i] < tb[i] ? 1 : -1; + } + if (ta.length !== tb.length) { + const longer = ta.length > tb.length ? ta : tb; + const rest = longer.slice(Math.min(ta.length, tb.length)).join(""); + const demoted = PRERELEASE_SUFFIX_RE.test(rest); + // `sign` is +1 when `a` is the longer tag: demoted ⇒ a sorts later. + const sign = ta.length > tb.length ? 1 : -1; + return demoted ? sign : -sign; + } + return String(a).localeCompare(String(b)); +} + +/** Tags newest-first by `compareTags`. Pure; `getSortedTags` is the IO wrapper. */ +export function orderTags(tags = []) { + return [...tags].sort(compareTags); } /** * The preferred (stable) version: the newest non-prerelease tag that is actually - * deployed, else `main`, else `master`, else the first version. `tags` must be - * newest-first; `versions` is the deployed set (output of `orderVersions`). + * deployed, else the default branch, else `main`, else `master`, else the first + * version. `tags` must be newest-first; `versions` is the deployed set (output of + * `orderVersions`). `defaultBranch` is optional — see `orderVersions`. */ -export function preferredVersion(versions, tags) { +export function preferredVersion(versions, tags, defaultBranch) { for (const tag of tags) { if (!isPrerelease(tag) && versions.includes(tag)) return tag; } - if (versions.includes("main")) return "main"; - if (versions.includes("master")) return "master"; + for (const branch of [defaultBranch, "main", "master"]) { + if (branch && versions.includes(branch)) return branch; + } return versions[0] ?? null; } @@ -374,7 +477,7 @@ export function preferredVersion(versions, tags) { * * When `preferred` is a genuine deployed non-prerelease *tag*, the site publishes * a `stable/` alias pointing at it and the root redirects to the constant - * `stable/` URL. Before the first release (preferred is `main`/`master`/a + * `stable/` URL. Before the first release (preferred is the default branch or a * leftover, or a prerelease-only fallback) there is no `stable/` and the root * redirects straight to that fallback version. * @@ -382,8 +485,8 @@ export function preferredVersion(versions, tags) { * `stableSrc` is the version dir to alias as `stable/` (or null); `redirectTarget` * is the dir the root `index.html` should redirect to. */ -export function stablePlan(versions, tags) { - const preferred = preferredVersion(versions, tags); +export function stablePlan(versions, tags, defaultBranch) { + const preferred = preferredVersion(versions, tags, defaultBranch); if (preferred && tags.includes(preferred) && !isPrerelease(preferred)) { return { preferred, stableSrc: preferred, redirectTarget: STABLE_ALIAS }; } @@ -428,16 +531,34 @@ export function renderSwitcher(baseUrl, versions, preferred) { return JSON.stringify(switcherStruct(baseUrl, versions, preferred), null, 2); } -/** Root redirect to `target` (relative, so it is host- and repo-agnostic). */ +/** Escape a value for an HTML attribute or text node. */ +function escapeHtml(value) { + return String(value) + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """) + .replace(/'/g, "'"); +} + +/** + * Root redirect to `target` (relative, so it is host- and repo-agnostic). + * + * `target` is a version name, which docs.yml validates down to `[A-Za-z0-9._-]` — so + * the escaping is belt-and-braces. It stays because this renders HTML from a value + * that ultimately derives from a ref name, and the two rules live in different files: + * loosening the workflow's character set should not be able to inject markup here. + */ export function renderRedirect(target) { + const safe = escapeHtml(target); return ` - Redirecting to ${target} + Redirecting to ${safe} - - + + @@ -493,7 +614,7 @@ function cmdSelectReleases(rest) { } const raw = readFileSync(0, "utf8").trim(); const releases = raw ? JSON.parse(raw) : []; - const cap = Number(values["max-releases"] ?? 0) || 0; + const cap = parseCap(values["max-releases"], "--max-releases"); const rows = selectReleases(releases, { defaultBranch, @@ -575,7 +696,7 @@ function cmdSelectArtifacts(rest) { return { num, sha: sha ?? "", approved: approved === "true" }; }); - const cap = Number(values["max-prs"] ?? 0) || 0; + const cap = parseCap(values["max-prs"], "--max-prs"); const rows = selectArtifacts(artifacts, { defaultBranch, repoId, @@ -610,7 +731,7 @@ function cmdSelectArtifacts(rest) { } /** - * `generate --site-dir --base-url [--required ]` — write switcher.json + + * `generate --site-dir --base-url [--default-branch ] [--required ]` — write switcher.json + * index.html and emit the stable-alias source. Runs after all gathering, so it * also hard-fails (exit 1) if a `--required` branch is absent from the site. */ @@ -620,6 +741,7 @@ function cmdGenerate(rest) { options: { "site-dir": { type: "string" }, "base-url": { type: "string" }, + "default-branch": { type: "string" }, required: { type: "string" }, }, }); @@ -627,7 +749,7 @@ function cmdGenerate(rest) { const baseUrl = values["base-url"]; if (!siteDir || !baseUrl) { throw new Error( - "usage: assemble.mjs generate --site-dir --base-url [--required ]", + "usage: assemble.mjs generate --site-dir --base-url [--default-branch ] [--required ]", ); } @@ -646,10 +768,17 @@ function cmdGenerate(rest) { // Tags are used verbatim as site dirs (getSortedTags drops `/`-tags), so // ordering + preferred + stable compare directly against the discovered dirs. + // The default branch is not assumed to be main/master: it heads the switcher and + // is the preferred version until a release is deployed. + const defaultBranch = values["default-branch"]; const tags = getSortedTags(); - const versions = orderVersions(builds, tags); - const preferred = preferredVersion(versions, tags); - const { stableSrc, redirectTarget } = stablePlan(versions, tags); + const versions = orderVersions(builds, tags, defaultBranch); + const preferred = preferredVersion(versions, tags, defaultBranch); + const { stableSrc, redirectTarget } = stablePlan( + versions, + tags, + defaultBranch, + ); // Diagnostics go to stderr so stdout carries only the stable-alias source. console.error(`Sorted versions: ${JSON.stringify(versions)}`); diff --git a/docs/explanations/architecture.md b/docs/explanations/architecture.md index 16f8bd0..f06ea04 100644 --- a/docs/explanations/architecture.md +++ b/docs/explanations/architecture.md @@ -299,13 +299,20 @@ The `stable` segment name is a fixed convention, hardcoded in the widget. - **PR build not yet green / SHA moved:** an open PR whose current head SHA has no successful CI run is skipped; its preview appears once the build passes. - **Merged/closed PR:** drops from the gather (open-PRs only) on the next deploy. -- **Prereleases:** excluded from `preferred`/redirect (an `a`/`b`/`rc` marker - following a digit, PEP 440 style — parity with the release workflow; a tag that - merely contains those letters, like `release-1.0`, is not a prerelease), but still - listed in the switcher if gathered. -- **Concurrency:** `concurrency: { group: pages, cancel-in-progress: true }`. Cancelling - a superseded deploy is safe *because* every deploy reconstructs the whole site — the - one that replaces it gathers everything the cancelled one would have. +- **Prereleases:** excluded from `preferred`/redirect (a marker following a digit, with + an optional separator — `1.0a1`, `2.0rc1`, `1.1.0-beta.1`, `2.0.0-rc.1` — parity with + the release workflow, held by a drift test; a tag that merely contains those letters, + like `release-1.0` or `1.0-candidate`, is not a prerelease), but still listed in the + switcher if gathered. They also sort *below* the release they qualify, which is why + the tag order is computed in `assemble.mjs` rather than by `git tag --sort=-v:refname`. +- **Concurrency:** `concurrency: { group: pages, cancel-in-progress: false }`. Deploys + serialise, and a superseded one is *queued*, not killed. Cancelling would be safe for + our artifact — every deploy reconstructs the whole site, so the next gathers whatever + a cancelled one would have — but the kill lands inside `deploy-pages`, between the + upload and the Pages backend finishing with it, and that state is not ours to reason + about. GitHub's Pages starter workflow uses `false` for the same reason. The cost is + bounded: a group holds one pending run, and a newer arrival replaces the pending one + before it starts, so a burst costs one extra deploy rather than a queue of them. ## The release-layer cache diff --git a/docs/how-to/choose-publish-events.md b/docs/how-to/choose-publish-events.md index 21bffcf..c8d16c5 100644 --- a/docs/how-to/choose-publish-events.md +++ b/docs/how-to/choose-publish-events.md @@ -18,8 +18,10 @@ whatever triggered it. Narrowing the trigger trades that away, so narrow it deli Publishing is no longer on any PR's critical path — it runs after CI, on its own run — so "it slows my PRs down" is not a reason to narrow it any more. Deploys do still serialise -on the `pages` concurrency group, but each one supersedes the last (`cancel-in-progress: -true`), which is safe precisely because every deploy reconstructs everything. +on the `pages` concurrency group, and a burst of pushes costs at most one extra deploy: +a group holds one pending run, so a third arrival replaces the queued second rather than +adding to it. A deploy already in flight is never cancelled — see +[architecture](../explanations/architecture.md). If a deploy takes minutes, the site itself is probably too big rather than the trigger too broad — see [keep-the-site-small](keep-the-site-small.md). diff --git a/docs/how-to/keep-the-site-small.md b/docs/how-to/keep-the-site-small.md index 94f3649..3d892d9 100644 --- a/docs/how-to/keep-the-site-small.md +++ b/docs/how-to/keep-the-site-small.md @@ -28,8 +28,15 @@ actually be uploaded): ``` assembled site: 734M on disk across 31 version dir(s) + ...of which stable/ is a second copy of 4.2.1 (12M) ``` +The second line is not a rounding detail: `stable/` is a symlink in the assembled tree, +but `actions/upload-pages-artifact` tars with `--dereference`, so the newest release is +uploaded **twice**. Budget for it — the largest release always counts double — and note +that the deploy that first publishes a release is the one that grows the site by 2× that +release, not 1×. + Or measure the released half directly: ```bash @@ -91,6 +98,8 @@ releases. `max-releases: 30` keeps it near 100 MB. - **The default branch** — always published; it is the one required version. - **The migration seed release** (`pages-default-seed`) — it stands in for the default branch rather than being a version of its own. +- **The `stable/` alias** — a full copy of the newest non-prerelease release, as above. + Lowering `max-releases` never removes it; only having no release at all does. ## Related diff --git a/docs/how-to/use-a-custom-domain.md b/docs/how-to/use-a-custom-domain.md new file mode 100644 index 0000000..063c2d9 --- /dev/null +++ b/docs/how-to/use-a-custom-domain.md @@ -0,0 +1,56 @@ +# How-to: serve the site from a custom domain + +A GitHub Pages **project site** is served at `https://ORG.github.io/REPO/`, so every +version lives under `/REPO/`. Two setups break that assumption — both serve from the +**root** of their host: + +- a **custom domain** (a `CNAME` on the repo's Pages settings), e.g. + `https://docs.example.org/` +- an **`ORG.github.io` / `USER.github.io` repo**, whose Pages site *is* the host root + +In either case the built HTML must be rooted at `/`, not at `/REPO/`, or every +root-absolute asset and link in it 404s. + +## Set `base-path` + +One input on `docs.yml`, in your `ci.yml`: + +```yaml +jobs: + docs: + uses: DiamondLightSource/myst-version-switcher-plugin/.github/workflows/docs.yml@ + with: + build-command: make docs + base-path: "/" # ← the site is served at the root, not at /REPO/ +``` + +The build then runs with `BASE_URL=/` — `/main`, `/pr-12`, `/2.1` — which +is where those pages actually live on a root-served site. + +`base-path` is the path to the *site*, without the version segment. It defaults to +`/`. Leading and trailing slashes are normalised, so `/`, `/docs` and `docs/` all +work. + +## Nothing else changes + +`publish.yml` needs no configuration for this. It reads the site's live URL from the +**Pages API**, which already reports the custom domain, and bakes that into +`switcher.json`'s entries — so the switcher's links follow the domain automatically. + +That split is worth knowing when debugging: a site whose **switcher links are right but +whose pages are broken** (missing CSS, 404s on every internal link) is a `base-path` that +was never set. The Pages API told the publish half the truth; nobody told the build half. + +## Checklist + +1. Add the domain in **Settings → Pages → Custom domain** (or use an `ORG.github.io` + repo) and let the DNS check pass. +2. Set `base-path: "/"` on the `docs.yml` job. +3. Re-run CI on the default branch, then let `publish.yml` deploy. +4. Open the site and check that a version's CSS loads and that the switcher's entries + point at the custom domain. + +## Related + +- [workflows reference](../reference/workflows.md) — the full `docs.yml` input table +- [use-with-sphinx](use-with-sphinx.md) — the same `BASE_URL` contract from a Sphinx build diff --git a/docs/how-to/use-with-sphinx.md b/docs/how-to/use-with-sphinx.md index 1e29c70..a6b9034 100644 --- a/docs/how-to/use-with-sphinx.md +++ b/docs/how-to/use-with-sphinx.md @@ -17,7 +17,8 @@ by following this page by hand. ## 1. Configure the theme's switcher in `conf.py` `docs.yml` exports two env vars to your build: `BASE_URL` (the sub-path this build is -served at, `/REPO/`) and `VERSION_NAME` (the bare version token — +served at — `/REPO/`, or `/` if you +[set one](use-a-custom-domain.md)) and `VERSION_NAME` (the bare version token — `pr-`, the default branch, or the tag). Wire them into the theme: ```python diff --git a/docs/reference/workflows.md b/docs/reference/workflows.md index 07cc29f..9bda6ab 100644 --- a/docs/reference/workflows.md +++ b/docs/reference/workflows.md @@ -52,6 +52,7 @@ preinstalled Node, so `build-command` can be `make` / `npx` / `tox` / `npm` driv |---|---|---|---| | `build-command` | **yes** | — | Command that builds the HTML site. Run with `BASE_URL` and `VERSION_NAME` set. Fold any project setup (`npm ci`, `cp CONFIG`, apt deps) into it. | | `html-dir` | no | `docs/_build/html` | Directory the build writes the site to. Its contents are staged into `docs.zip`'s `html/` root, so any name works. | +| `base-path` | no | `""` (→ `/`) | URL path the site is served at, without the version segment. Set to `/` for a custom domain or an `ORG.github.io` repo — see [custom domains](../how-to/use-a-custom-domain.md). | ## `publish-gh-pages.yml` — assemble + deploy (privileged) @@ -68,7 +69,29 @@ preinstalled Node, so `build-command` can be `make` / `npx` / `tox` / `npm` driv Both caps exist because the site deploys as **one** Pages artifact against a hard **1 GB** limit, which a long-lived project will eventually hit. Set them as **literals** in `publish.yml`'s `with:` block so they apply on every path. See -[keep-the-site-small](../how-to/keep-the-site-small.md). +[keep-the-site-small](../how-to/keep-the-site-small.md). A cap that is not a +non-negative integer fails the deploy rather than being read as `0`/unlimited. + +:::{note} `.mvs/` is reserved +The job checks your repo out at the workspace root and this project's `assemble/` into +`.mvs/` beside it, so a repo of your own with a `.mvs/` directory would collide. Nothing +else in the workspace is touched, and the checkout is discarded with the runner. +::: + +### What approving a fork preview grants + +The `pr` input publishes fork-authored HTML and JavaScript to your Pages site, and a +project Pages site shares an **origin** with every other Pages site in the org: +`https://ORG.github.io`. Same origin means the same `localStorage`, the same +service-worker scope and any cookies scoped to that host — so an approved preview is not +sandboxed from the rest of the organisation's documentation. + +The mechanism is deliberately tight: the approval is a commit status **pinned to the head +SHA**, so a later push drops the preview until a maintainer re-approves, which closes the +approve-then-push hole. What it cannot do is judge the content for you. Treat approving as +a **code review of the built output** — look at what the PR adds to the docs, not only at +whether CI is green. If you only need to check that the docs build, CI already told you +that without publishing anything. ## What it reports back @@ -97,7 +120,10 @@ Every deploy rebuilds the complete tree from authoritative inputs: Branch and PR artifacts are found via the **artifacts API by name** (`docs` — the artifact name is the contract), not by workflow filename, so the consumer's entry workflow can be called anything. The URLs baked into `switcher.json` use the site's -live Pages URL from the **Pages API**, so a custom domain (CNAME) works. +live Pages URL from the **Pages API**, so a custom domain (CNAME) is reflected there +without configuration — but the *pages* those URLs point at are built at whatever +`docs.yml`'s `base-path` says, so a custom domain needs that input set too. See +[use-a-custom-domain](../how-to/use-a-custom-domain.md). Release assets are immutable, so the gather caches them (`actions/cache`, keyed on the exact set of asset ids it intends to publish) and downloads only what it has not seen — @@ -112,8 +138,13 @@ saves are gated on the default branch having been the thing that was built. A version no longer gathered (a merged/closed PR, a deleted release) is correctly dropped, because `deploy-pages` replaces the *entire* site. Sources are gathered in -priority order (releases lowest, then branch CI, then the current build highest), so a -fresher source always wins when names collide. +priority order, so a fresher source always wins when names collide. There are two rungs, +not three: a release's `docs.zip` (or the migration seed) first, then the branch/PR CI +artifact over the top. The build that triggered the deploy is not a rung of its own — +under `workflow_run` the engine runs *after* that build completed, so its `docs` +artifact is simply the newest one the artifacts API lists. The default branch has a +third fallback below both: the Actions cache copy, used only when its CI artifact has +expired. ## The `docs.zip` / version-name contracts @@ -131,11 +162,14 @@ without passing anything between them: is skipped with a warning rather than guessed at. - **The version name is the site sub-dir *and* the `BASE_URL`.** It is `pr-` for PRs, else the ref name (the default branch, or a tag without `/`). `docs.yml` sets - `BASE_URL=/REPO/`; nothing passes that name onward, because the gather - re-derives the identical string from the same facts (a release's tag, a PR's number, - the repo's `default_branch`) and unpacks the zip at `site/`. Same name - on both sides, so assets never 404. There is **no sanitisation**: version names are - clean by construction (the `tags: ['*']` trigger never builds `/`-tags). + `BASE_URL=/`; nothing passes that name onward, because the + gather re-derives the identical string from the same facts (a release's tag, a PR's + number, the repo's `default_branch`) and unpacks the zip at `site/`. + Same name on both sides, so assets never 404. The name is **validated**, not assumed + clean: it must be non-empty and match `[A-Za-z0-9._-]`, or the build fails with the + reason. In this repo it is clean by construction (the `tags: ['*']` trigger never + builds `/`-tags), but `docs.yml` is reusable and a consumer can trigger it on any ref + they like — git allows `$( )`, backticks and quotes in a ref name. ## Internals: `assemble.mjs` @@ -148,10 +182,22 @@ workflow does the IO: - **`select-artifacts`** — which CI artifact becomes the default branch, and which open PRs get a preview, with fork approval resolved by the caller beforehand. - **`generate`** — version ordering, prerelease detection, `switcher.json` and the root - redirect, the `stable/` alias source, and the required-branch guard. - -Both `select-*` commands rank with the same comparator, so the release cap and the PR cap -cannot drift apart on how they break a timestamp tie. The `gh`/`unzip`/`mv` IO + redirect, the `stable/` alias source, and the required-branch guard. (The deploy stamp + the verify step polls, `deploy-id.txt`, is written by the workflow alongside it — it is + runner state, not a decision.) + +Every version the site serves appears in `switcher.json`, **PR previews included** — a +`pr-` entry in the dropdown is deliberate, so a reviewer can jump to a preview from +any page. They are ordered after the default branch and the tags, and disappear when the +PR closes. + +Tags are ordered by `compareTags`, not by `git tag --sort=-v:refname`: git's version sort +ranks `1.1.0-beta.1` *above* `1.1.0`, and the prerelease rule and the ordering rule are +better off sharing one marker list than being kept in step by hand. + +All the "which of these is newest?" questions — the release cap, the PR cap, the +default-branch artifact, the newest artifact per head SHA — go through one comparator, so +they cannot break a timestamp tie differently from each other. The `gh`/`unzip`/`mv` IO plumbing lives as inline bash steps in `publish-gh-pages.yml`'s `deploy` job — named and separated so step timing and failures are visible in the GH Actions UI, and covered by a harness that loads those steps out of the YAML and runs them against a mock `gh`. diff --git a/docs/tutorials/adding-to-a-fresh-repo.md b/docs/tutorials/adding-to-a-fresh-repo.md index 6b60046..023ddfd 100644 --- a/docs/tutorials/adding-to-a-fresh-repo.md +++ b/docs/tutorials/adding-to-a-fresh-repo.md @@ -153,7 +153,9 @@ jobs: That is the whole publish setup: no job in `ci.yml`, no shim, nothing to thread through. A maintainer publishes a fork preview by running this workflow from the Actions tab with -the `pr` number. +the `pr` number — which serves that fork's HTML and JS from your Pages origin, shared +with the rest of the org's sites, so read [what you are +approving](../reference/workflows.md#what-approving-a-fork-preview-grants) before you do. ## 4. Push your branch and open a PR diff --git a/package.json b/package.json index dc5f771..69999bc 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,7 @@ "test": "node test/test-url-logic.mjs && node test/test-assemble.mjs", "docs": "cd docs && myst build --html --strict", "docs-dev": "cd docs && myst start", - "test:workflows": "uv run --quiet --with pyyaml python3 test/workflow-harness/test_gather.py && uv run --quiet --with pyyaml python3 test/workflow-harness/test_shape.py" + "test:workflows": "uv run --quiet --with pyyaml python3 test/workflow-harness/test_gather.py && uv run --quiet --with pyyaml python3 test/workflow-harness/test_shape.py && uv run --quiet --with pyyaml python3 test/workflow-harness/test_release_prerelease.py" }, "files": [ "plugins/version-switcher.mjs", diff --git a/renovate.json b/renovate.json index 9bfb02d..af191f1 100644 --- a/renovate.json +++ b/renovate.json @@ -23,5 +23,20 @@ "minor" ] } + ], + "customManagers": [ + { + "customType": "regex", + "description": "Bump the actionlint pinned in _lint.yml. Renovate cannot compute the tarball's sha256, so ACTIONLINT_SHA256 has to be updated by hand in the same PR \u2014 until it is, the lint job fails the checksum and goes red.", + "managerFilePatterns": [ + "/^\\.github/workflows/_lint\\.yml$/" + ], + "matchStrings": [ + "ACTIONLINT_VERSION: (?\\S+)" + ], + "depNameTemplate": "rhysd/actionlint", + "datasourceTemplate": "github-releases", + "extractVersionTemplate": "^v(?.+)$" + } ] } diff --git a/test/test-assemble.mjs b/test/test-assemble.mjs index 36231aa..3b2cff6 100644 --- a/test/test-assemble.mjs +++ b/test/test-assemble.mjs @@ -15,7 +15,9 @@ import { discoverVersions, isPrerelease, missingRequired, + orderTags, orderVersions, + parseCap, preferredVersion, RELZIPS_CACHE_PREFIX, renderRedirect, @@ -74,6 +76,22 @@ assert.deepEqual(orderVersions(["pr-10", "pr-2", "main"], []), [ ]); ok("orderVersions sorts pr- previews numerically, not lexically"); +// a `develop` default branch heads the list instead of landing in the leftover +// bucket among the pr- previews. +assert.deepEqual(orderVersions(["pr-3", "2.1", "develop"], tags, "develop"), [ + "develop", + "2.1", + "pr-3", +]); +ok("orderVersions puts a named default branch first"); + +// main/master still come after it when a repo mid-rename carries both. +assert.deepEqual(orderVersions(["main", "develop"], [], "develop"), [ + "develop", + "main", +]); +ok("orderVersions keeps main after the named default branch"); + // --- discoverVersions: directory names under the site root --- const site = mkdtempSync(join(tmpdir(), "assemble-site-")); for (const d of ["main", "2.1", "2.0"]) mkdirSync(join(site, d)); @@ -100,6 +118,64 @@ assert.equal(isPrerelease("beta-program"), false); assert.equal(isPrerelease("stable-2.0"), false); ok("isPrerelease ignores a/b/rc letters not following a digit"); +// the hyphenated/dotted semver spellings, and the long marker words. These used to +// read as STABLE, which made `stable/` alias a beta. +assert.ok(isPrerelease("1.1.0-beta.1")); +assert.ok(isPrerelease("1.0.0-rc1")); +assert.ok(isPrerelease("v1.2.3-alpha")); +assert.ok(isPrerelease("2.0.0-pre.1")); +assert.ok(isPrerelease("1.0.dev0")); +assert.ok(isPrerelease("1.3.2-a9")); +ok("isPrerelease flags hyphenated semver + long-form markers"); + +// a longer word that merely STARTS with a marker is not one (the trailing lookahead). +assert.equal(isPrerelease("1.0-candidate"), false); +assert.equal(isPrerelease("2.0-canary"), false); +assert.equal(isPrerelease("1.0-preflight"), false); +ok("isPrerelease ignores longer words starting with a marker"); + +// --- compareTags/orderTags: newest first, prereleases below their own release --- +// THE invariant: for any tag X, every prerelease of X sorts after X. git's +// -v:refname does the opposite for the hyphenated forms. +assert.deepEqual( + orderTags([ + "1.0.0", + "1.1.0-beta.1", + "1.1.0", + "1.0.0-rc1", + "2.0.0a1", + "2.0.0", + ]), + ["2.0.0", "2.0.0a1", "1.1.0", "1.1.0-beta.1", "1.0.0", "1.0.0-rc1"], +); +ok("orderTags ranks each prerelease below the release it qualifies"); + +// digit runs compare numerically, so 1.10 is newer than 1.9 (not lexically before). +assert.deepEqual(orderTags(["1.9.0", "1.10.0", "1.9.1"]), [ + "1.10.0", + "1.9.1", + "1.9.0", +]); +ok("orderTags compares numeric segments numerically"); + +// a trailing segment that is NOT a prerelease marker means a newer version. +assert.deepEqual(orderTags(["1.1", "1.1.1"]), ["1.1.1", "1.1"]); +ok("orderTags treats a non-marker trailing segment as newer"); + +// a `v` prefix is a spelling of the same version, so it does not reorder anything. +assert.deepEqual(orderTags(["v1.0.0", "v1.2.0", "v1.1.0"]), [ + "v1.2.0", + "v1.1.0", + "v1.0.0", +]); +ok("orderTags ignores a leading v when comparing"); + +// total order: reordering the input cannot change the result (an unstable ranking +// would change the published set, and so the cache key, on identical inputs). +const shuffled = ["1.1.0-beta.1", "2.0.0", "1.0.0", "2.0.0a1", "1.1.0"]; +assert.deepEqual(orderTags(shuffled), orderTags([...shuffled].reverse())); +ok("orderTags is a total order (input order cannot change it)"); + // --- preferredVersion: newest deployed stable tag, else main --- assert.equal(preferredVersion(["main", "2.1", "2.0"], tags), "2.1"); ok("preferredVersion picks the newest deployed stable tag"); @@ -116,6 +192,19 @@ ok("preferredVersion falls back to main when no stable tag is deployed"); assert.equal(preferredVersion(["main", "2.0"], ["2.1", "2.0"]), "2.0"); ok("preferredVersion ignores tags with no deployed build"); +// the bug this pair guards: with the beta ranked above 1.1.0 by git's sort AND read +// as stable, preferredVersion returned the beta. +const semverTags = orderTags(["1.1.0", "1.1.0-beta.1"]); +assert.equal( + preferredVersion(["main", "1.1.0", "1.1.0-beta.1"], semverTags), + "1.1.0", +); +ok("preferredVersion prefers a release over its own hyphenated prerelease"); + +// a consumer whose default branch is neither main nor master. +assert.equal(preferredVersion(["develop", "pr-3"], [], "develop"), "develop"); +ok("preferredVersion falls back to the named default branch"); + // --- stablePlan: redirect target + stable-alias source --- // a deployed non-prerelease release → stable/ alias + root → stable/. assert.deepEqual(stablePlan(["main", "2.1", "2.0"], tags), { @@ -141,6 +230,33 @@ assert.deepEqual(stablePlan(["3.0rc1"], ["3.0rc1"]), { }); ok("stablePlan never aliases a prerelease as stable"); +// ...including the hyphenated spelling, which used to slip through as a real release. +assert.deepEqual(stablePlan(["main", "1.1.0-beta.1"], ["1.1.0-beta.1"]), { + preferred: "main", + stableSrc: null, + redirectTarget: "main", +}); +ok("stablePlan never aliases a hyphenated prerelease as stable"); + +// --- parseCap: a cap that cannot be read is not a cap of 0 --- +assert.equal(parseCap(undefined), 0); +assert.equal(parseCap(""), 0); +assert.equal(parseCap("0"), 0); +assert.equal(parseCap("30"), 30); +assert.equal(parseCap(30), 30); +ok("parseCap accepts non-negative integers, absent means unlimited"); + +// `Number(x) || 0` turned every one of these into 0 — i.e. UNLIMITED — so a typo in a +// consumer's publish.yml silently disabled the 1 GB guard. +for (const bad of ["abc", "-1", "3O", "1.5", "1e3", " ", "30 releases"]) { + assert.throws( + () => parseCap(bad, "max-releases"), + /max-releases must be/, + `parseCap(${JSON.stringify(bad)})`, + ); +} +ok("parseCap throws on anything that is not a non-negative integer"); + // --- missingRequired: required branches absent from the discovered site dirs --- // versions are the discovered site dirs; the default branch and every gathered // PR/preview are among them by generate time. Names compare verbatim (no rule). @@ -216,6 +332,13 @@ const toMain = renderRedirect("main"); assert.match(toMain, /url=\.\/main\/index\.html/); ok("renderRedirect targets the fallback dir before the first release"); +// defence in depth: docs.yml validates the version name, but the two rules live in +// different files and this one renders HTML. +const nasty = renderRedirect('x">