From 314c713ebcfaaf9a42bbbd8eecfdec67d4650962 Mon Sep 17 00:00:00 2001 From: Matt McKay Date: Wed, 5 Aug 2026 13:10:28 +1000 Subject: [PATCH 1/3] FIX: repair the GPU install stack and the build cache Two independent breakages have kept every workflow in this repo red. Neither was caused by any recent PR. 1. ci.yml installed torch from the cu128 nightly channel, which has been frozen since April 2026. pip resolved torch to 2.12.0.dev20260408 while the newest torchvision there pinned torch==2.12.0.dev20260407, a wheel since pruned, giving ResolutionImpossible. Nothing imports torchvision or torchaudio -- the only torch consumer is bayes_nonconj.md, using core tensor ops -- so both are dropped and torch now comes from the stable cu128 index. 2. cache.yml is the sole producer of the build-cache artifact that ci, collab, linkcheck and publish all download, and it has never produced one. It never installed torch or pyro-ppl despite bayes_nonconj.md importing both, so its Build HTML step failed during notebook execution. Both are added here. Also fixed, and load-bearing: - numpy<2. anaconda=2024.10 brings numba 0.60.0, which caps at numpy<2.1, and 9 lectures import numba. The unpinned `jax[cuda12-local]` upgrade resolved to jax 0.11.0 -> numpy 2.5.1 and broke all of them. Pinning jax to 0.7.1 (the last release accepting numpy>=1.26) is not sufficient on its own: "pymc<6" resolves to 5.28.5, which requires pytensor>=2.38.2, which requires numpy>=2.0. Verified with pip --dry-run: {arviz<1, pymc<6, kaleido<1} gives numpy 2.4.6, while adding numpy<2 backtracks to pymc 5.25.1 / pytensor 2.31.7 / numpy 1.26.4. An assertion after the install makes drift loud. - include-hidden-files on cache.yml's upload. _config.yml sets execute_notebooks: "cache", so the execution cache lives in _build/.jupyter_cache. upload-artifact has excluded hidden files by default since v4.4, so the artifact would have shipped without the cache it exists to carry. - kaleido<1 in collab.yml. That job runs in the Colab container, which none of the conda pins reach; it ships plotly 5.24.1 and no kaleido, so back_prop.md's `!pip install kaleido` pulls an incompatible 1.x. - The three ci.yml report uploads shared the artifact name execution-reports, so the second collided with the first exactly when a build failed and the report was most needed. jax moves from cuda12-local to cuda12 (bundled CUDA), matching what lecture-jax runs green on this same AMI. Note .github/runs-on.yml pins ami-09baf66e396fa7cfd, which is lecture-jax's pre-CUDA-13 image -- the siblings moved to CUDA 13 in November 2025 and we did not, so cu130 and jax[cuda13] would be wrong here until the AMI is bumped. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/cache.yml | 19 ++++++++++++++++--- .github/workflows/ci.yml | 21 +++++++++++++++------ .github/workflows/collab.yml | 5 +++++ .github/workflows/publish.yml | 11 +++++++++-- 4 files changed, 45 insertions(+), 11 deletions(-) diff --git a/.github/workflows/cache.yml b/.github/workflows/cache.yml index e617bc8..1e12f91 100644 --- a/.github/workflows/cache.yml +++ b/.github/workflows/cache.yml @@ -23,8 +23,17 @@ jobs: - name: Install JAX, Numpyro shell: bash -l {0} run: | - pip install --upgrade "jax[cuda12-local]" - pip install numpyro + # CUDA 12 stack. .github/runs-on.yml pins ami-09baf66e396fa7cfd, a CUDA 12 + # AMI, so do not move to cu130 / jax[cuda13] without bumping the AMI first. + # torch and pyro-ppl were never installed here even though bayes_nonconj.md + # imports both -- this workflow has never produced a successful build. + pip install torch --index-url https://download.pytorch.org/whl/cu128 + pip install pyro-ppl + pip install "jax[cuda12]==0.7.1" + pip install numpyro + # See the equivalent block in ci.yml for why numpy<2 is load-bearing. + pip install "numpy<2" "arviz<1" "pymc<6" "kaleido<1" + python -c "import numpy, numba; assert numpy.__version__.startswith('1.'), numpy.__version__" python scripts/test-jax-install.py - name: Check nvidia drivers shell: bash -l {0} @@ -44,4 +53,8 @@ jobs: uses: actions/upload-artifact@v6 with: name: build-cache - path: _build \ No newline at end of file + path: _build + # _config.yml sets execute_notebooks: "cache", so the execution cache + # lives in _build/.jupyter_cache. upload-artifact excludes hidden files + # by default, which would ship a "cache" containing no cache. + include-hidden-files: true diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b17ca3f..27ac22e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,10 +23,19 @@ jobs: - name: Install JAX, Numpyro shell: bash -l {0} run: | - pip install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu128 + # CUDA 12 stack. .github/runs-on.yml pins ami-09baf66e396fa7cfd, a CUDA 12 + # AMI, so do not move to cu130 / jax[cuda13] without bumping the AMI first. + pip install torch --index-url https://download.pytorch.org/whl/cu128 pip install pyro-ppl - pip install --upgrade "jax[cuda12-local]" - pip install numpyro + pip install "jax[cuda12]==0.7.1" + pip install numpyro + # numpy<2 is load-bearing. anaconda=2024.10 brings numba 0.60.0, which caps + # at numpy<2.1, and 9 lectures import numba. "pymc<6" on its own resolves to + # pymc 5.28.5 -> pytensor>=2.38.2 -> numpy>=2.0, i.e. numpy 2.4.x. Pinning + # numpy<2 makes pip backtrack to pymc 5.25.1 / pytensor 2.31.7 instead. + # These also make the notebooks' own unpinned `!pip install` cells no-ops. + pip install "numpy<2" "arviz<1" "pymc<6" "kaleido<1" + python -c "import numpy, numba; assert numpy.__version__.startswith('1.'), numpy.__version__" python scripts/test-jax-install.py # Check nvidia drivers - name: nvidia Drivers @@ -56,7 +65,7 @@ jobs: uses: actions/upload-artifact@v6 if: failure() with: - name: execution-reports + name: execution-reports-notebooks path: _build/jupyter/reports - name: Build PDF from LaTeX shell: bash -l {0} @@ -68,7 +77,7 @@ jobs: uses: actions/upload-artifact@v6 if: failure() with: - name: execution-reports + name: execution-reports-latex path: _build/latex/reports # Final Build of HTML - name: Build HTML @@ -79,7 +88,7 @@ jobs: uses: actions/upload-artifact@v6 if: failure() with: - name: execution-reports + name: execution-reports-html path: _build/html/reports - name: Preview Deploy to Netlify uses: nwtgck/actions-netlify@v3 diff --git a/.github/workflows/collab.yml b/.github/workflows/collab.yml index a3f0fe5..8870252 100644 --- a/.github/workflows/collab.yml +++ b/.github/workflows/collab.yml @@ -37,6 +37,11 @@ jobs: shell: bash -l {0} run: | pip install jupyter-book==1.0.3 quantecon-book-theme==0.8.2 sphinx-tojupyter==0.3.0 sphinxext-rediraffe==0.2.7 sphinxcontrib-youtube==1.3.0 sphinx-togglebutton==0.3.2 arviz sphinx-proof sphinx-exercise sphinx-reredirects + # The Colab image ships plotly 5.24.1 and no kaleido, so back_prop.md's + # `!pip install kaleido` cell pulls kaleido 1.x, which is incompatible. + # Pinning here makes that cell a no-op. Unlike the conda workflows, this + # container is a coherent numpy 2.0.2 stack, so it needs no numpy pin. + pip install "kaleido<1" # Build of HTML (Execution Testing) - name: Build HTML shell: bash -l {0} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index cae7931..b73f1f6 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -25,8 +25,15 @@ jobs: - name: Install JAX, Numpyro shell: bash -l {0} run: | - pip install --upgrade "jax[cuda12-local]" - pip install numpyro + # CUDA 12 stack. .github/runs-on.yml pins ami-09baf66e396fa7cfd, a CUDA 12 + # AMI, so do not move to cu130 / jax[cuda13] without bumping the AMI first. + pip install torch --index-url https://download.pytorch.org/whl/cu128 + pip install pyro-ppl + pip install "jax[cuda12]==0.7.1" + pip install numpyro + # See the equivalent block in ci.yml for why numpy<2 is load-bearing. + pip install "numpy<2" "arviz<1" "pymc<6" "kaleido<1" + python -c "import numpy, numba; assert numpy.__version__.startswith('1.'), numpy.__version__" python scripts/test-jax-install.py - name: Check nvidia drivers shell: bash -l {0} From 3e2e873b9d2db405d15ba53d2179942ce6fcf402 Mon Sep 17 00:00:00 2001 From: Matt McKay Date: Wed, 5 Aug 2026 14:25:37 +1000 Subject: [PATCH 2/3] FIX: hold the conda stack when pre-installing arviz and pymc The first smoke test (run 30974637413) got further than anything in a year -- torch, pyro-ppl, jax 0.7.1 and numpyro all installed cleanly, and numpy stayed at 1.26.4 -- then died in the pin step. "numpy<2" was too weak a constraint. arviz pulls xarray, current xarray wants a pandas newer than the conda-provided 2.2.2, pandas 3.x requires numpy>=2, and with numpy pinned below 2 pip had only pandas left to move. It walked 3.0.5 down to 2.1.0, reached an sdist, and the build failed: the numpy Cython headers require Cython>=3 and pandas 2.1.0 pins Cython<3. Fixed by constraining the conda-provided stack instead of just numpy, so pip backtracks on the arviz/xarray side where wheels exist. Verified with pip --dry-run --only-binary=:all: that holding numpy/pandas/scipy at the runner's actual versions (1.26.4 / 2.2.2 / 1.13.1, read out of the failed run's log) resolves wheel-only to arviz 0.23.4, pymc 5.25.1, pytensor 2.31.7, xarray-einstats 0.9.1, kaleido 0.2.1, leaving all three conda packages untouched. The constraints file is generated from the live environment rather than hardcoded, so bumping anaconda in environment.yml cannot silently invalidate it, and it skips any package that is absent rather than failing. --only-binary=:all: turns a repeat of this failure into an immediate clear error instead of a five-minute compile that ends in a Cython traceback. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/cache.yml | 6 ++++-- .github/workflows/ci.yml | 18 ++++++++++++------ .github/workflows/publish.yml | 6 ++++-- 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/.github/workflows/cache.yml b/.github/workflows/cache.yml index 1e12f91..28591c9 100644 --- a/.github/workflows/cache.yml +++ b/.github/workflows/cache.yml @@ -31,8 +31,10 @@ jobs: pip install pyro-ppl pip install "jax[cuda12]==0.7.1" pip install numpyro - # See the equivalent block in ci.yml for why numpy<2 is load-bearing. - pip install "numpy<2" "arviz<1" "pymc<6" "kaleido<1" + # See the equivalent block in ci.yml for why the conda stack must be held. + python -c "import importlib.metadata as md; d={x.metadata['Name'].lower():x.version for x in md.distributions()}; print('\n'.join(f'{p}=={d[p]}' for p in ('numpy','pandas','scipy') if p in d))" > /tmp/conda-pins.txt + cat /tmp/conda-pins.txt + pip install --only-binary=:all: -c /tmp/conda-pins.txt "arviz<1" "pymc<6" "kaleido<1" python -c "import numpy, numba; assert numpy.__version__.startswith('1.'), numpy.__version__" python scripts/test-jax-install.py - name: Check nvidia drivers diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 27ac22e..94cb2e1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,12 +29,18 @@ jobs: pip install pyro-ppl pip install "jax[cuda12]==0.7.1" pip install numpyro - # numpy<2 is load-bearing. anaconda=2024.10 brings numba 0.60.0, which caps - # at numpy<2.1, and 9 lectures import numba. "pymc<6" on its own resolves to - # pymc 5.28.5 -> pytensor>=2.38.2 -> numpy>=2.0, i.e. numpy 2.4.x. Pinning - # numpy<2 makes pip backtrack to pymc 5.25.1 / pytensor 2.31.7 instead. - # These also make the notebooks' own unpinned `!pip install` cells no-ops. - pip install "numpy<2" "arviz<1" "pymc<6" "kaleido<1" + # arviz and pymc are pre-installed at versions the conda stack can carry, so + # the notebooks' own unpinned `!pip install arviz pymc` cells become no-ops. + # They must not be allowed to move the conda-provided scientific stack: left + # free, xarray drags pandas forward, pandas 3.x requires numpy>=2, and pip + # backtracks through pandas until it reaches an sdist that fails to build + # (the numpy Cython headers need Cython>=3). numpy>=2 would also break numba + # 0.60.0, which caps at numpy<2.1 and is imported by 9 lectures. + # The constraints file is derived from the env rather than hardcoded so that + # bumping anaconda in environment.yml does not silently invalidate it. + python -c "import importlib.metadata as md; d={x.metadata['Name'].lower():x.version for x in md.distributions()}; print('\n'.join(f'{p}=={d[p]}' for p in ('numpy','pandas','scipy') if p in d))" > /tmp/conda-pins.txt + cat /tmp/conda-pins.txt + pip install --only-binary=:all: -c /tmp/conda-pins.txt "arviz<1" "pymc<6" "kaleido<1" python -c "import numpy, numba; assert numpy.__version__.startswith('1.'), numpy.__version__" python scripts/test-jax-install.py # Check nvidia drivers diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index b73f1f6..edf9283 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -31,8 +31,10 @@ jobs: pip install pyro-ppl pip install "jax[cuda12]==0.7.1" pip install numpyro - # See the equivalent block in ci.yml for why numpy<2 is load-bearing. - pip install "numpy<2" "arviz<1" "pymc<6" "kaleido<1" + # See the equivalent block in ci.yml for why the conda stack must be held. + python -c "import importlib.metadata as md; d={x.metadata['Name'].lower():x.version for x in md.distributions()}; print('\n'.join(f'{p}=={d[p]}' for p in ('numpy','pandas','scipy') if p in d))" > /tmp/conda-pins.txt + cat /tmp/conda-pins.txt + pip install --only-binary=:all: -c /tmp/conda-pins.txt "arviz<1" "pymc<6" "kaleido<1" python -c "import numpy, numba; assert numpy.__version__.startswith('1.'), numpy.__version__" python scripts/test-jax-install.py - name: Check nvidia drivers From 704441f3b7e4e07362a7758d615752b8186ccce3 Mon Sep 17 00:00:00 2001 From: Matt McKay Date: Wed, 5 Aug 2026 16:05:25 +1000 Subject: [PATCH 3/3] FIX: pin prettytable<3.18 -- the actual cause of the build failure Run 30975106893 executed 24 of 25 lectures successfully, including the 70-minute bayes_nonconj and back_prop's kaleido image export. Exactly one failed, and the execution report -- the first ever retrieved from this repo -- gives the cause: AttributeError: module 'wcwidth' has no attribute 'width' prettytable/prettytable.py:3080 in _str_block_width prettytable declares "Requires-Dist: wcwidth" with no lower bound. pip therefore accepts the wcwidth 0.2.x that anaconda ships as already satisfying it, and installs the latest prettytable. prettytable 3.18.0 switched from wcswidth() to wcwidth.width(), which does not exist before wcwidth 0.3.0. Verified the boundary against the wheels: 3.17.0 uses wcswidth(), 3.18.0 uses wcwidth.width(); wcwidth 0.2.14 exports (wcwidth, wcswidth, list_versions) and 0.3.0 adds width(). That is an upstream packaging bug, and it fires on any machine whose wcwidth predates 0.3.0 -- which is why prob_matrix.md and prob_meaning.md have been able to break without either lecture changing. Pinning prettytable rather than upgrading wcwidth deliberately: prettytable is used by two lectures, while wcwidth is shared with prompt_toolkit and IPython. This keeps the blast radius at the package that is actually wrong. Pre-installing it here also makes the notebooks' own `!pip install prettytable` a no-op. Verified the full pin set still resolves wheel-only with the conda stack held. Not addressed here, and deliberately: linearmodels and interpolation are still installed unpinned at notebook time, where the constraints file does not reach them. Both are fine against today's releases. Narrowing that gap is a separate change; over-broad pinning is what broke run 30974637413. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/cache.yml | 2 +- .github/workflows/ci.yml | 7 ++++++- .github/workflows/publish.yml | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cache.yml b/.github/workflows/cache.yml index 28591c9..3cbf454 100644 --- a/.github/workflows/cache.yml +++ b/.github/workflows/cache.yml @@ -34,7 +34,7 @@ jobs: # See the equivalent block in ci.yml for why the conda stack must be held. python -c "import importlib.metadata as md; d={x.metadata['Name'].lower():x.version for x in md.distributions()}; print('\n'.join(f'{p}=={d[p]}' for p in ('numpy','pandas','scipy') if p in d))" > /tmp/conda-pins.txt cat /tmp/conda-pins.txt - pip install --only-binary=:all: -c /tmp/conda-pins.txt "arviz<1" "pymc<6" "kaleido<1" + pip install --only-binary=:all: -c /tmp/conda-pins.txt "arviz<1" "pymc<6" "kaleido<1" "prettytable<3.18" python -c "import numpy, numba; assert numpy.__version__.startswith('1.'), numpy.__version__" python scripts/test-jax-install.py - name: Check nvidia drivers diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 94cb2e1..b709608 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,9 +38,14 @@ jobs: # 0.60.0, which caps at numpy<2.1 and is imported by 9 lectures. # The constraints file is derived from the env rather than hardcoded so that # bumping anaconda in environment.yml does not silently invalidate it. + # prettytable<3.18 is a real fix, not hygiene: prettytable declares a bare + # "wcwidth" with no floor, so pip accepts conda's wcwidth 0.2.x as satisfying + # it, but 3.18.0 calls wcwidth.width(), which only exists from wcwidth 0.3.0. + # That AttributeError is what broke prob_matrix.md in run 30975106893. + # 3.17.0 still uses wcswidth(), which conda's wcwidth has. python -c "import importlib.metadata as md; d={x.metadata['Name'].lower():x.version for x in md.distributions()}; print('\n'.join(f'{p}=={d[p]}' for p in ('numpy','pandas','scipy') if p in d))" > /tmp/conda-pins.txt cat /tmp/conda-pins.txt - pip install --only-binary=:all: -c /tmp/conda-pins.txt "arviz<1" "pymc<6" "kaleido<1" + pip install --only-binary=:all: -c /tmp/conda-pins.txt "arviz<1" "pymc<6" "kaleido<1" "prettytable<3.18" python -c "import numpy, numba; assert numpy.__version__.startswith('1.'), numpy.__version__" python scripts/test-jax-install.py # Check nvidia drivers diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index edf9283..0e39387 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -34,7 +34,7 @@ jobs: # See the equivalent block in ci.yml for why the conda stack must be held. python -c "import importlib.metadata as md; d={x.metadata['Name'].lower():x.version for x in md.distributions()}; print('\n'.join(f'{p}=={d[p]}' for p in ('numpy','pandas','scipy') if p in d))" > /tmp/conda-pins.txt cat /tmp/conda-pins.txt - pip install --only-binary=:all: -c /tmp/conda-pins.txt "arviz<1" "pymc<6" "kaleido<1" + pip install --only-binary=:all: -c /tmp/conda-pins.txt "arviz<1" "pymc<6" "kaleido<1" "prettytable<3.18" python -c "import numpy, numba; assert numpy.__version__.startswith('1.'), numpy.__version__" python scripts/test-jax-install.py - name: Check nvidia drivers