From 0b7501238b1ec6232819fbe1e5ae52d07714d1ce Mon Sep 17 00:00:00 2001 From: Matt McKay Date: Tue, 7 Jul 2026 16:19:05 +1000 Subject: [PATCH 1/3] Fix linkcheck false positives (release-asset input, accept 200) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit lecture-dp's link checker mirrored the live site with `wget --accept html`, which fetches only HTML — so lychee reported every local asset (_static/*, _images/*) as "file not found" — and used `--accept 403,503`, which drops 200 so every working link was flagged. Check the published HTML release asset instead — a permanent tarball of the full site (HTML + assets), per QuantEcon/meta#282 lesson 2 and matching lecture-python-programming. Restore `--accept 200,403,503`, add `--root-dir` for root-relative links, and scan only top-level `_site/*.html` so the Sphinx theme partials under _static/ (raw Jinja, not real links) are skipped. Mirrors the fix in QuantEcon/continuous_time_mcs#215. See QuantEcon/meta#282 Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/linkcheck.yml | 49 +++++++++++++++++++++------------ 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/.github/workflows/linkcheck.yml b/.github/workflows/linkcheck.yml index 0f890d6..7233fbc 100644 --- a/.github/workflows/linkcheck.yml +++ b/.github/workflows/linkcheck.yml @@ -9,27 +9,37 @@ jobs: linkcheck: runs-on: ubuntu-latest permissions: + contents: read # read the latest release asset issues: write # required for peter-evans/create-issue-from-file steps: - # This repo deploys GitHub Pages via the artifact workflow (OIDC), so - # there is no gh-pages branch to check out — mirror the live site instead. - - name: Mirror published site - id: mirror + # This repo deploys GitHub Pages via the artifact workflow (no gh-pages + # branch). Check links against the published HTML release asset — a + # permanent tarball of the full site (HTML plus _static/_images/etc.) — + # so lychee resolves local asset and root-relative links without + # depending on the live site being reachable. + - name: Download and extract latest release HTML + id: fetch + env: + GH_TOKEN: ${{ github.token }} run: | - wget --quiet --recursive --no-parent --no-host-directories \ - --accept html https://dp.quantecon.org/ || true - count=$(find . -name '*.html' | wc -l | tr -d ' ') - echo "Mirrored $count HTML file(s) from https://dp.quantecon.org/" + asset_url=$(gh api repos/${{ github.repository }}/releases/latest \ + --jq '.assets[] | select(.name | endswith(".tar.gz")) | .browser_download_url' | head -1) + mkdir -p _site + if [ -n "$asset_url" ]; then + curl -sSL "$asset_url" | tar -xz -C _site || true + fi + count=$(find _site -name '*.html' | wc -l | tr -d ' ') + echo "Extracted $count HTML file(s) from the latest release asset" if [ "$count" -eq 0 ]; then - # Mirror failed (site down or wget broke). Don't run lychee against - # an empty mirror and report green — surface the failure instead. + # No usable release asset — don't run lychee against an empty tree + # and report green; surface the failure instead. echo "ok=false" >> "$GITHUB_OUTPUT" mkdir -p lychee { - echo "## Link Checker — mirror failed" + echo "## Link Checker — release asset missing" echo - echo "\`wget\` mirrored **0 HTML files** from https://dp.quantecon.org/, so the link check did not run." - echo "The live site may be down, or the mirror step may be broken — please investigate." + echo "Could not download or extract an HTML \`.tar.gz\` from the latest release, so the link check did not run." + echo "Check that the most recent \`publish-*\` release has the \`lecture-dp-html-*.tar.gz\` asset attached." } > lychee/out.md else echo "ok=true" >> "$GITHUB_OUTPUT" @@ -37,15 +47,20 @@ jobs: - name: Link Checker id: lychee - if: steps.mirror.outputs.ok == 'true' + if: steps.fetch.outputs.ok == 'true' uses: lycheeverse/lychee-action@v2 with: fail: false - args: --accept 403,503 '**/*.html' + # Check only the top-level lecture pages — not theme partials under + # _static/ (e.g. webpack-macros.html holds raw Jinja `{{ pathto(...) }}` + # that isn't a real link). 200 must stay in --accept or every working + # link is flagged; --root-dir resolves root-relative links (e.g. + # /_notebooks/*.ipynb, /_pdf/*.pdf) against the extracted site root. + args: --accept 200,403,503 --root-dir ${{ github.workspace }}/_site '_site/*.html' - # Open an issue if the mirror failed, or if lychee found broken links. + # Open an issue if the release asset was missing, or lychee found broken links. - name: Create Issue From File - if: steps.mirror.outputs.ok != 'true' || steps.lychee.outputs.exit_code != 0 + if: steps.fetch.outputs.ok != 'true' || steps.lychee.outputs.exit_code != 0 uses: peter-evans/create-issue-from-file@v6 with: title: Link Checker Report From 6571982204c4e50abd0d630f2d44e8b11207f91b Mon Sep 17 00:00:00 2001 From: Matt McKay Date: Tue, 7 Jul 2026 16:22:38 +1000 Subject: [PATCH 2/3] Make release-asset fetch non-fatal Under `set -eo pipefail`, a transient `gh api` failure aborted the fetch step before the empty-input guard ran, so no 'release asset missing' report was created. Add `|| true` (and silence stderr) so the guard still fires. Mirrors QuantEcon/continuous_time_mcs#215 (Copilot review follow-up). Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/linkcheck.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/linkcheck.yml b/.github/workflows/linkcheck.yml index 7233fbc..5a2a5ab 100644 --- a/.github/workflows/linkcheck.yml +++ b/.github/workflows/linkcheck.yml @@ -22,8 +22,11 @@ jobs: env: GH_TOKEN: ${{ github.token }} run: | + # `|| true` + 2>/dev/null: a transient gh api error (rate-limit/auth) + # must not abort the step under `set -eo pipefail` — fall through to + # the empty-input guard below so it reports "release asset missing". asset_url=$(gh api repos/${{ github.repository }}/releases/latest \ - --jq '.assets[] | select(.name | endswith(".tar.gz")) | .browser_download_url' | head -1) + --jq '.assets[] | select(.name | endswith(".tar.gz")) | .browser_download_url' 2>/dev/null | head -1 || true) mkdir -p _site if [ -n "$asset_url" ]; then curl -sSL "$asset_url" | tar -xz -C _site || true From 57111c1892671bcb199b8ee761c30f7a4613ec87 Mon Sep 17 00:00:00 2001 From: Matt McKay Date: Tue, 7 Jul 2026 21:41:26 +1000 Subject: [PATCH 3/3] Harden release-asset extract against partial downloads (Copilot review) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A failed or truncated `curl | tar` was swallowed by `|| true`, potentially leaving a partial _site tree where count > 0 — so lychee would run against an incomplete site and re-introduce 'file not found' false positives. Use `curl -f` and an `if !` guard that clears _site on failure, so a bad download is treated as 'no usable asset' and reported via the guard. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/linkcheck.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/linkcheck.yml b/.github/workflows/linkcheck.yml index 5a2a5ab..521ac6b 100644 --- a/.github/workflows/linkcheck.yml +++ b/.github/workflows/linkcheck.yml @@ -29,7 +29,14 @@ jobs: --jq '.assets[] | select(.name | endswith(".tar.gz")) | .browser_download_url' 2>/dev/null | head -1 || true) mkdir -p _site if [ -n "$asset_url" ]; then - curl -sSL "$asset_url" | tar -xz -C _site || true + # `curl -f` fails on HTTP errors; an `if !` guard catches a failed + # or partial download/extract and discards the incomplete tree, so + # lychee never runs against a half-extracted site (which would + # reintroduce "file not found" false positives). + if ! curl -fsSL "$asset_url" | tar -xz -C _site; then + echo "::warning::could not download/extract $asset_url — treating as no usable asset" + rm -rf _site && mkdir -p _site + fi fi count=$(find _site -name '*.html' | wc -l | tr -d ' ') echo "Extracted $count HTML file(s) from the latest release asset"