From 318063d47b49202b81a92e81bfe1cc76b3501be3 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Fri, 3 Jul 2026 13:29:42 +0000 Subject: [PATCH 1/2] fix: auto-retry via fresh dispatch when the Pages origin wedges MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit External evidence (actions/deploy-pages#383) and today's incident (origin frozen on the pre-tag switcher.json for 83+ minutes despite deploy-pages reporting success) both show that waiting longer does not recover a wedged Pages origin — only a genuinely new deploy attempt does. Widening the verify timeout further isn't a realistic fix, so on verify failure the deploy job now re-dispatches itself fresh (capped at 2 retries) instead of just red-checking. Must be a fresh workflow_dispatch, not a job rerun — rerunning stacks a second github-pages artifact under the same run and breaks deploy-pages' artifact lookup, which we also hit today. --- .github/workflows/publish-dispatch.yml | 12 ++++++--- .github/workflows/publish.yml | 35 +++++++++++++++++++++++++- 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/.github/workflows/publish-dispatch.yml b/.github/workflows/publish-dispatch.yml index 9c9cd65..c328aa8 100644 --- a/.github/workflows/publish-dispatch.yml +++ b/.github/workflows/publish-dispatch.yml @@ -24,14 +24,20 @@ on: description: External fork PR number to approve (pins its head SHA) and preview. Leave empty to just re-deploy. required: false default: "" + verify-retry: + description: >- + Internal — auto-retry counter for a wedged Pages origin. Don't set manually. + required: false + default: "0" jobs: publish: uses: ./.github/workflows/publish.yml with: - version-name: ${{ inputs.version-name }} # "" on dispatch → pure durable gather - pr: ${{ inputs.pr }} # set (dispatch only) → pin that fork head SHA - dispatch-workflow: publish-dispatch.yml # the file the tag re-dispatch re-fires + version-name: ${{ inputs.version-name }} # "" on dispatch → pure durable gather + pr: ${{ inputs.pr }} # set (dispatch only) → pin that fork head SHA + dispatch-workflow: publish-dispatch.yml # the file the tag re-dispatch re-fires + verify-retry: ${{ inputs.verify-retry || '0' }} # "" on workflow_call → publish.yml's own default permissions: contents: read actions: write # publish.yml's re-dispatch job re-fires this shim via gh diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 18b9bc6..40b12de 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -51,6 +51,13 @@ on: required: false default: publish-dispatch.yml type: string + verify-retry: + description: >- + Internal — auto-retry counter for a wedged Pages origin, incremented by the + deploy job itself when it re-dispatches. Don't set manually. + required: false + default: "0" + type: string jobs: # Fork PR: read-only token, never deploys — surface the manual-opt-in hint on the PR. @@ -79,7 +86,7 @@ jobs: runs-on: ubuntu-latest permissions: contents: read # checkout + read release assets - actions: read # gh run download (this run's + cross-run docs artifacts) + actions: write # gather cross-run docs artifacts (read) + retry-dispatch on a wedged origin (write) pages: write # deploy to Pages id-token: write # deploy-pages OIDC statuses: write # set the preview-approved status on a fork PR head SHA @@ -310,6 +317,7 @@ jobs: # so a fast/first-of-SHA deploy isn't slowed. On failure, dump the origin headers # (a frozen etag/last-modified is the wedge signature). - name: Verify the deployed origin matches what we assembled + id: verify env: PAGE_URL: ${{ steps.deployment.outputs.page_url }} SITE_DIR: ${{ runner.temp }}/site @@ -343,6 +351,31 @@ jobs: sleep 5 done + # A verify failure means the origin is wedged (see comment above) — and per + # observed external evidence (actions/deploy-pages#383: waiting longer does NOT + # recover a wedged origin, only a genuinely NEW deploy attempt does), the fix is a + # fresh workflow_dispatch, not a longer timeout. This must be a fresh dispatch and + # NOT a job rerun — rerunning would stack a second `github-pages` artifact under + # this same run and break deploy-pages' artifact lookup ("Multiple artifacts named + # github-pages"). Capped at 2 retries so a genuine outage still ends up red instead + # of looping forever. + - name: Retry via a fresh dispatch if the origin is wedged + if: failure() && steps.verify.outcome == 'failure' + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + WF: ${{ inputs.dispatch-workflow }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + RETRY: ${{ inputs.verify-retry }} + run: | + set -euo pipefail + if [ "$RETRY" -ge 2 ]; then + echo "::error::origin still wedged after $RETRY retries — giving up, manual re-dispatch needed" + exit 0 + fi + gh workflow run "$WF" --repo "$REPO" --ref "$DEFAULT_BRANCH" -f verify-retry="$((RETRY + 1))" + echo "::notice::origin wedged — dispatched a fresh retry ($((RETRY + 1))/2)" + # A tag can't deploy inline (same SHA as the default-branch push → Pages drops it # unless the event is workflow_dispatch). Wait for the release, then re-dispatch the # shim so the deploy runs on a workflow_dispatch event and re-serves. From 2e243eb453f20cccd385bbb0f6f2a7b0002cb566 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Fri, 3 Jul 2026 13:52:13 +0000 Subject: [PATCH 2/2] fix: thread a retry-until deadline instead of a retry counter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the attempt-count cap with a wall-clock deadline anchored on the release's publish time (gh release view --json publishedAt), not the underlying commit's timestamp. This repo's own release flow allows tagging a commit long after it was merged, so a commit-age check would wrongly refuse to retry in exactly the same-SHA dedup scenario (deploy-pages#383) this exists to handle. The deadline is computed once by the re-dispatch job and carried unchanged through any retries, and is naturally scoped to tag/release deploys only (retry-until is empty for push/PR/manual dispatches, which can't hit this wedge class — they're always first-of-SHA). --- .github/workflows/publish-dispatch.yml | 9 +++--- .github/workflows/publish.yml | 40 ++++++++++++++++---------- 2 files changed, 30 insertions(+), 19 deletions(-) diff --git a/.github/workflows/publish-dispatch.yml b/.github/workflows/publish-dispatch.yml index c328aa8..8e138aa 100644 --- a/.github/workflows/publish-dispatch.yml +++ b/.github/workflows/publish-dispatch.yml @@ -24,11 +24,12 @@ on: description: External fork PR number to approve (pins its head SHA) and preview. Leave empty to just re-deploy. required: false default: "" - verify-retry: + retry-until: description: >- - Internal — auto-retry counter for a wedged Pages origin. Don't set manually. + Internal — epoch-seconds deadline for auto-retrying a wedged Pages origin. + Don't set manually. required: false - default: "0" + default: "" jobs: publish: @@ -37,7 +38,7 @@ jobs: version-name: ${{ inputs.version-name }} # "" on dispatch → pure durable gather pr: ${{ inputs.pr }} # set (dispatch only) → pin that fork head SHA dispatch-workflow: publish-dispatch.yml # the file the tag re-dispatch re-fires - verify-retry: ${{ inputs.verify-retry || '0' }} # "" on workflow_call → publish.yml's own default + retry-until: ${{ inputs.retry-until }} # "" on workflow_call/no-retry dispatch → no auto-retry permissions: contents: read actions: write # publish.yml's re-dispatch job re-fires this shim via gh diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 40b12de..904bdc6 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -51,12 +51,13 @@ on: required: false default: publish-dispatch.yml type: string - verify-retry: + retry-until: description: >- - Internal — auto-retry counter for a wedged Pages origin, incremented by the - deploy job itself when it re-dispatches. Don't set manually. + Internal — epoch-seconds deadline for auto-retrying a wedged Pages origin. + Empty means no retry budget (set only by the tag re-dispatch job, from the + release's publish time); carried unchanged through any retries. Don't set manually. required: false - default: "0" + default: "" type: string jobs: @@ -357,24 +358,27 @@ jobs: # fresh workflow_dispatch, not a longer timeout. This must be a fresh dispatch and # NOT a job rerun — rerunning would stack a second `github-pages` artifact under # this same run and break deploy-pages' artifact lookup ("Multiple artifacts named - # github-pages"). Capped at 2 retries so a genuine outage still ends up red instead - # of looping forever. + # github-pages"). Only tag/release deploys carry a retry-until budget (set by the + # re-dispatch job below, from the release's publish time, carried unchanged through + # each retry) — that's the same-SHA dedup wedge this targets, which a push/PR/manual + # deploy can't hit (those are always first-of-SHA). No budget → no auto-retry, just + # the red check. - name: Retry via a fresh dispatch if the origin is wedged - if: failure() && steps.verify.outcome == 'failure' + if: failure() && steps.verify.outcome == 'failure' && inputs.retry-until != '' env: GH_TOKEN: ${{ github.token }} REPO: ${{ github.repository }} WF: ${{ inputs.dispatch-workflow }} DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} - RETRY: ${{ inputs.verify-retry }} + RETRY_UNTIL: ${{ inputs.retry-until }} run: | set -euo pipefail - if [ "$RETRY" -ge 2 ]; then - echo "::error::origin still wedged after $RETRY retries — giving up, manual re-dispatch needed" + if [ "$(date +%s)" -ge "$RETRY_UNTIL" ]; then + echo "::error::origin still wedged past the retry deadline ($(date -d "@$RETRY_UNTIL" -Is)) — giving up, manual re-dispatch needed" exit 0 fi - gh workflow run "$WF" --repo "$REPO" --ref "$DEFAULT_BRANCH" -f verify-retry="$((RETRY + 1))" - echo "::notice::origin wedged — dispatched a fresh retry ($((RETRY + 1))/2)" + gh workflow run "$WF" --repo "$REPO" --ref "$DEFAULT_BRANCH" -f retry-until="$RETRY_UNTIL" + echo "::notice::origin wedged — dispatched a fresh retry (budget until $(date -d "@$RETRY_UNTIL" -Is))" # A tag can't deploy inline (same SHA as the default-branch push → Pages drops it # unless the event is workflow_dispatch). Wait for the release, then re-dispatch the @@ -397,10 +401,16 @@ jobs: set -euo pipefail # ci.yml's release job attaches the tag's docs.zip in parallel; the dispatched # run gathers the tag from that asset, so don't dispatch until it exists. + published_at="" for _ in $(seq 1 36); do - gh release view "$TAG" --repo "$REPO" >/dev/null 2>&1 && break + published_at=$(gh release view "$TAG" --repo "$REPO" --json publishedAt -q .publishedAt 2>/dev/null) && break sleep 5 done - gh release view "$TAG" --repo "$REPO" >/dev/null 2>&1 \ + [ -n "$published_at" ] \ || { echo "::error::release $TAG not found after 3 min — did the release job fail?"; exit 1; } - gh workflow run "$WF" --repo "$REPO" --ref "$DEFAULT_BRANCH" + # Retry budget anchored on the release's publish time, not the commit's — a tag + # can be cut long after its commit was merged (this repo's own release flow + # allows it), so the commit's own timestamp would wrongly disqualify the exact + # same-SHA dedup wedge (deploy-pages#383) this budget exists to retry through. + retry_until=$(( $(date -d "$published_at" +%s) + 900 )) + gh workflow run "$WF" --repo "$REPO" --ref "$DEFAULT_BRANCH" -f retry-until="$retry_until"