From 499f8132c03efc98112e42949925f45b969f3cb0 Mon Sep 17 00:00:00 2001 From: lmoresi Date: Fri, 17 Apr 2026 12:30:58 -0700 Subject: [PATCH 1/4] Make pixi.toml static; derive PETSC_ARCH via activation script Previously ./uw petsc switch rewrote PETSC_ARCH values in pixi.toml via sed, which (1) made every switch a tracked file change, (2) forced contributors on different PETSc versions to carry diverging pixi.toml in their working tree, and (3) did not reliably propagate to pixi's activated environment because pixi caches resolved state. Replace the five hardcoded PETSC_ARCH = "petsc-*-uw-*" entries (in features amr [osx-arm64 and linux-64], amr-mpich, amr-openmpi, and amr-debug) with per-feature activation scripts that source the new petsc-custom/activate-petsc-arch.sh. The script reads the active version from petsc-custom/.petsc-version and combines it with the active $PIXI_ENVIRONMENT_NAME (and uname on Darwin/Linux for the plain amr env) to construct PETSC_ARCH = petsc-{ver}-uw-{mpi}{suffix}. Non-AMR environments match no case and leave PETSC_ARCH unset, so the script is safe as a no-op for default/runtime/dev/mpich/openmpi environments. Switching PETSc versions now only updates .petsc-version (one untracked, gitignored file); pixi.toml is not modified. Underworld development team with AI support from Claude Code --- petsc-custom/activate-petsc-arch.sh | 33 +++++++++++++++++++++++++++++ pixi.toml | 21 ++++++++++-------- 2 files changed, 45 insertions(+), 9 deletions(-) create mode 100755 petsc-custom/activate-petsc-arch.sh diff --git a/petsc-custom/activate-petsc-arch.sh b/petsc-custom/activate-petsc-arch.sh new file mode 100755 index 000000000..61d0afe6d --- /dev/null +++ b/petsc-custom/activate-petsc-arch.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# activate-petsc-arch.sh — sourced by pixi on environment activation. +# +# Dynamically sets PETSC_ARCH for AMR environments (custom PETSc builds) +# by combining the active pixi environment name with the PETSc version +# pinned in petsc-custom/.petsc-version. +# +# This replaces hardcoded PETSC_ARCH entries in pixi.toml so that +# switching PETSc versions (via ./uw petsc switch ) does not require +# editing tracked configuration. + +_uw_mpi="" +_uw_suffix="" +case "${PIXI_ENVIRONMENT_NAME:-}" in + amr|amr-runtime|amr-dev) + case "$(uname -s)" in + Darwin) _uw_mpi="openmpi" ;; + *) _uw_mpi="mpich" ;; + esac ;; + amr-mpich|amr-mpich-dev) _uw_mpi="mpich" ;; + amr-openmpi|amr-openmpi-dev) _uw_mpi="openmpi" ;; + amr-debug) _uw_mpi="openmpi"; _uw_suffix="-debug" ;; +esac + +if [ -n "$_uw_mpi" ]; then + _uw_ver="4" + _uw_ver_file="${PIXI_PROJECT_ROOT:-.}/petsc-custom/.petsc-version" + [ -f "$_uw_ver_file" ] && _uw_ver=$(cat "$_uw_ver_file" 2>/dev/null || echo "4") + + export PETSC_ARCH="petsc-${_uw_ver}-uw-${_uw_mpi}${_uw_suffix}" +fi + +unset _uw_mpi _uw_suffix _uw_ver _uw_ver_file diff --git a/pixi.toml b/pixi.toml index e72f7f827..b8d6796cc 100644 --- a/pixi.toml +++ b/pixi.toml @@ -165,6 +165,9 @@ cmake = ">=3.31,<4" make = ">=4.4,<5" mpi4py = ">=4,<5" +[feature.amr.activation] +scripts = ["petsc-custom/activate-petsc-arch.sh"] + [feature.amr.activation.env] PETSC_DIR = "$PIXI_PROJECT_ROOT/petsc-custom/petsc" @@ -173,16 +176,10 @@ openmpi = ">=5.0,<6" hdf5 = { version = ">=1.14,<2", build = "*openmpi*" } h5py = { version = ">=3.12,<4", build = "*openmpi*" } -[feature.amr.target.osx-arm64.activation.env] -PETSC_ARCH = "petsc-4-uw-openmpi" - [feature.amr.target.linux-64.dependencies] hdf5 = { version = ">=1.14,<2", build = "*mpich*" } h5py = { version = ">=3.12,<4", build = "*mpich*" } -[feature.amr.target.linux-64.activation.env] -PETSC_ARCH = "petsc-4-uw-mpich" - [feature.amr.tasks] petsc-local-build = { cmd = "./build-petsc.sh", cwd = "petsc-custom" } petsc-local-clean = { cmd = "./build-petsc.sh clean", cwd = "petsc-custom" } @@ -202,9 +199,11 @@ mpi4py = ">=4,<5" hdf5 = { version = ">=1.14,<2", build = "*mpich*" } h5py = { version = ">=3.12,<4", build = "*mpich*" } +[feature.amr-mpich.activation] +scripts = ["petsc-custom/activate-petsc-arch.sh"] + [feature.amr-mpich.activation.env] PETSC_DIR = "$PIXI_PROJECT_ROOT/petsc-custom/petsc" -PETSC_ARCH = "petsc-4-uw-mpich" [feature.amr-mpich.tasks] petsc-local-build = { cmd = "./build-petsc.sh", cwd = "petsc-custom" } @@ -221,9 +220,11 @@ mpi4py = ">=4,<5" hdf5 = { version = ">=1.14,<2", build = "*openmpi*" } h5py = { version = ">=3.12,<4", build = "*openmpi*" } +[feature.amr-openmpi.activation] +scripts = ["petsc-custom/activate-petsc-arch.sh"] + [feature.amr-openmpi.activation.env] PETSC_DIR = "$PIXI_PROJECT_ROOT/petsc-custom/petsc" -PETSC_ARCH = "petsc-4-uw-openmpi" [feature.amr-openmpi.tasks] petsc-local-build = { cmd = "./build-petsc.sh", cwd = "petsc-custom" } @@ -246,9 +247,11 @@ petsc-local-clean = { cmd = "./build-petsc.sh clean", cwd = "petsc-custom" } # "--COPTFLAGS=-g -O0" "--CXXOPTFLAGS=-g -O0" "--FOPTFLAGS=-g -O0" # make PETSC_DIR=$(pwd) PETSC_ARCH=petsc-4-uw-openmpi-debug all +[feature.amr-debug.activation] +scripts = ["petsc-custom/activate-petsc-arch.sh"] + [feature.amr-debug.activation.env] PETSC_DIR = "$PIXI_PROJECT_ROOT/petsc-custom/petsc" -PETSC_ARCH = "petsc-4-uw-openmpi-debug" # ============================================ # HPC CLUSTER FEATURE From 9b7c359a6dab42c8f05134245184733bd2bf3b43 Mon Sep 17 00:00:00 2001 From: lmoresi Date: Fri, 17 Apr 2026 12:31:05 -0700 Subject: [PATCH 2/4] Nuke stale build artifacts and petsc4py on ./uw petsc switch ./uw build has target-change detection (build/.petsc_target) that cleans build/lib.* when petsc4py.get_config() reports a new arch. After ./uw petsc switch, though, the still-installed petsc4py is linked to the OLD arch and continues to report it, so the check does not fire and pip/Cython silently reuse stale wheels against the new PETSc. Force the full rebuild path at switch time: remove build/lib.*, build/temp.*, build/bdist.*, the .petsc_target marker, purge pip's wheel cache, and uninstall petsc4py (AMR envs only). The next ./uw build then rebuilds petsc4py against the new PETSc and recompiles every Cython extension. Underworld development team with AI support from Claude Code --- uw | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/uw b/uw index 42bcb2247..d94c5f192 100755 --- a/uw +++ b/uw @@ -293,8 +293,23 @@ run_petsc_cmd() { exit 1 fi $PIXI run -e "$(get_env)" "$SCRIPT_DIR/petsc-custom/build-petsc.sh" checkout "$arg" + + # Nuke stale build artifacts and petsc4py. The next ./uw build's + # target-change detection can't fire because the still-installed + # petsc4py reports the OLD arch, so we force a full rebuild here. + echo "" + echo "Clearing stale build artifacts for clean rebuild..." + rm -rf "$SCRIPT_DIR"/build/lib.* "$SCRIPT_DIR"/build/temp.* "$SCRIPT_DIR"/build/bdist.* 2>/dev/null + rm -f "$SCRIPT_DIR/build/.petsc_target" + $PIXI run -e "$(get_env)" pip cache purge 2>/dev/null || true + if is_amr_env "$(get_env)"; then + echo "Uninstalling old petsc4py (linked to previous PETSc build)..." + $PIXI run -e "$(get_env)" pip uninstall -y petsc4py 2>/dev/null || true + fi + echo "" - echo "To complete the switch, rebuild:" + echo "To complete the switch, rebuild PETSc then underworld3:" + echo " ./uw petsc build" echo " ./uw build" ;; active) From 61675b59e856d3fe5fe40b1a50549a355021efec Mon Sep 17 00:00:00 2001 From: lmoresi Date: Fri, 17 Apr 2026 13:16:54 -0700 Subject: [PATCH 3/4] setup.py: prefer PETSC_DIR/PETSC_ARCH env vars over petsc4py.get_config() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When pixi activation exports PETSC_DIR and PETSC_ARCH for a custom build, setup.py was ignoring them and reading petsc4py.get_config() instead. That reports the arch petsc4py was BUILT against — so immediately after a PETSc version switch, setup.py would silently link the rebuild against the previous PETSc, not the new one. Re-order the PETSc discovery into an explicit priority chain: 1. PETSC_DIR / PETSC_ARCH from the environment (pixi activation, shell export, HPC module load) 2. petsc4py.get_config() — authoritative match for installed petsc4py 3. conda `petsc` pip package 4. CONDA_PREFIX/lib/.../site-packages/petsc fallback The previous code only consulted env vars as a last-resort branch of the conda fallback, which was exactly backwards — env vars are the most specific signal and should win. Underworld development team with AI support from Claude Code --- setup.py | 54 +++++++++++++++++++++++++----------------------------- 1 file changed, 25 insertions(+), 29 deletions(-) diff --git a/setup.py b/setup.py index 45c98a680..840c84ab2 100644 --- a/setup.py +++ b/setup.py @@ -76,35 +76,34 @@ def configure(): LIBRARY_DIRS = [] LIBRARIES = [] - PETSC_DIR = "" - PETSC_ARCH = "" - - # try get PETSC_DIR from petsc pip installation - try: - import petsc - - PETSC_DIR = petsc.get_petsc_dir() - except: - pass - - # PETSc import os - if not os.path.exists(PETSC_DIR): - print(f"PETSC_INFO from petsc4py - {petsc4py.get_config()}") - PETSC_DIR = petsc4py.get_config()["PETSC_DIR"] - PETSC_ARCH = petsc4py.get_config()["PETSC_ARCH"] - - # It is preferable to use the petsc4py paths to the - # petsc libraries for consistency but the pip installation - # of PETSc sometimes points to the temporary setup up path - - if not os.path.exists(PETSC_DIR): - print(f"PETSC_DIR {PETSC_DIR} is bad - trying another ...") + PETSC_DIR = "" + PETSC_ARCH = "" - if os.environ.get("CONDA_PREFIX") and not os.environ.get("PETSC_DIR"): + # Priority 1: Environment variables (set by pixi activation for custom builds) + if os.environ.get("PETSC_DIR") and os.path.exists(os.environ["PETSC_DIR"]): + PETSC_DIR = os.environ["PETSC_DIR"] + PETSC_ARCH = os.environ.get("PETSC_ARCH", "") + + # Priority 2: petsc4py configuration (matches the installed petsc4py) + if not PETSC_DIR or not os.path.exists(PETSC_DIR): + config = petsc4py.get_config() + PETSC_DIR = config["PETSC_DIR"] + PETSC_ARCH = config.get("PETSC_ARCH", "") + + # Priority 3: conda petsc package + if not PETSC_DIR or not os.path.exists(PETSC_DIR): + try: + import petsc + PETSC_DIR = petsc.get_petsc_dir() + except ImportError: + pass + + # Priority 4: conda prefix fallback + if not PETSC_DIR or not os.path.exists(PETSC_DIR): + if os.environ.get("CONDA_PREFIX"): import sys - py_version = f"{sys.version_info.major}.{sys.version_info.minor}" PETSC_DIR = os.path.join( os.environ["CONDA_PREFIX"], @@ -112,10 +111,7 @@ def configure(): "python" + py_version, "site-packages", "petsc", - ) # symlink to latest python - PETSC_ARCH = os.environ.get("PETSC_ARCH", "") - else: - PETSC_DIR = os.environ["PETSC_DIR"] + ) PETSC_ARCH = os.environ.get("PETSC_ARCH", "") print(f"Using PETSc:") From 15f8e1d8ed54e2dcef34422b13ad931b5f7b7139 Mon Sep 17 00:00:00 2001 From: lmoresi Date: Fri, 17 Apr 2026 13:59:51 -0700 Subject: [PATCH 4/4] Address Copilot review feedback on PR #120 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - activate-petsc-arch.sh: platform-aware MPI for amr-debug (was hardcoded openmpi, wrong on linux-64 where feature.amr pulls mpich). Version discovery now matches ./uw petsc_version_short: strip whitespace from .petsc-version, default to 324 (not "4", which would produce a legacy arch name that no current build tree uses). - pixi.toml: update the amr-debug setup comment to refer to "$PETSC_ARCH" (now exported by the activation script) instead of the stale hardcoded "petsc-4-uw-openmpi-debug" string. - uw: drop the global `pip cache purge` on ./uw petsc switch — it clears every project's pip wheel cache, which is overkill. The two installs that matter already avoid the cache: run_build passes --no-cache-dir to `pip install .`, and petsc4py's local-path install now does the same. Underworld development team with AI support from Claude Code --- petsc-custom/activate-petsc-arch.sh | 26 +++++++++++++++++--------- pixi.toml | 11 ++++++++--- uw | 6 ++++-- 3 files changed, 29 insertions(+), 14 deletions(-) diff --git a/petsc-custom/activate-petsc-arch.sh b/petsc-custom/activate-petsc-arch.sh index 61d0afe6d..16347481d 100755 --- a/petsc-custom/activate-petsc-arch.sh +++ b/petsc-custom/activate-petsc-arch.sh @@ -11,23 +11,31 @@ _uw_mpi="" _uw_suffix="" + +# Default MPI for AMR envs is platform-dependent and mirrors the conda deps +# pinned in pixi.toml: openmpi on macOS (feature.amr.target.osx-arm64), +# mpich on Linux (feature.amr.target.linux-64). +_uw_default_mpi="mpich" +[ "$(uname -s)" = "Darwin" ] && _uw_default_mpi="openmpi" + case "${PIXI_ENVIRONMENT_NAME:-}" in - amr|amr-runtime|amr-dev) - case "$(uname -s)" in - Darwin) _uw_mpi="openmpi" ;; - *) _uw_mpi="mpich" ;; - esac ;; + amr|amr-runtime|amr-dev) _uw_mpi="$_uw_default_mpi" ;; amr-mpich|amr-mpich-dev) _uw_mpi="mpich" ;; amr-openmpi|amr-openmpi-dev) _uw_mpi="openmpi" ;; - amr-debug) _uw_mpi="openmpi"; _uw_suffix="-debug" ;; + amr-debug) _uw_mpi="$_uw_default_mpi"; _uw_suffix="-debug" ;; esac if [ -n "$_uw_mpi" ]; then - _uw_ver="4" + # Match the version-discovery logic in ./uw (petsc_version_short): + # prefer .petsc-version, strip whitespace, fall back to 324. + _uw_ver="324" _uw_ver_file="${PIXI_PROJECT_ROOT:-.}/petsc-custom/.petsc-version" - [ -f "$_uw_ver_file" ] && _uw_ver=$(cat "$_uw_ver_file" 2>/dev/null || echo "4") + if [ -f "$_uw_ver_file" ]; then + _uw_read=$(tr -d '[:space:]' < "$_uw_ver_file" 2>/dev/null) + [ -n "$_uw_read" ] && _uw_ver="$_uw_read" + fi export PETSC_ARCH="petsc-${_uw_ver}-uw-${_uw_mpi}${_uw_suffix}" fi -unset _uw_mpi _uw_suffix _uw_ver _uw_ver_file +unset _uw_mpi _uw_suffix _uw_default_mpi _uw_ver _uw_ver_file _uw_read diff --git a/pixi.toml b/pixi.toml index b8d6796cc..2bc8f4c60 100644 --- a/pixi.toml +++ b/pixi.toml @@ -237,15 +237,20 @@ petsc-local-clean = { cmd = "./build-petsc.sh clean", cwd = "petsc-custom" } # Uses the same conda deps as amr but points to a separate PETSC_ARCH # so the optimised build is untouched. # -# Setup: +# PETSC_ARCH is set dynamically by petsc-custom/activate-petsc-arch.sh +# and resolves to: petsc--uw--debug +# where comes from petsc-custom/.petsc-version and follows +# the platform default (openmpi on macOS, mpich on Linux). +# +# Setup (inside a pixi shell -e amr-debug, so $PETSC_ARCH is already set): # cd petsc-custom/petsc # pixi run -e amr-debug python3 ./configure \ -# --with-petsc-arch=petsc-4-uw-openmpi-debug --with-debugging=1 \ +# --with-petsc-arch="$PETSC_ARCH" --with-debugging=1 \ # --with-mpi-dir="$CONDA_PREFIX" --with-hdf5=0 \ # --download-mpich=0 --download-openmpi=0 --download-mpi4py=0 \ # --with-petsc4py=0 --with-x=0 --with-pragmatic=0 --with-slepc=0 \ # "--COPTFLAGS=-g -O0" "--CXXOPTFLAGS=-g -O0" "--FOPTFLAGS=-g -O0" -# make PETSC_DIR=$(pwd) PETSC_ARCH=petsc-4-uw-openmpi-debug all +# make PETSC_DIR=$(pwd) PETSC_ARCH="$PETSC_ARCH" all [feature.amr-debug.activation] scripts = ["petsc-custom/activate-petsc-arch.sh"] diff --git a/uw b/uw index d94c5f192..eebb537fe 100755 --- a/uw +++ b/uw @@ -159,7 +159,7 @@ run_build() { # Step 2: Check/build petsc4py if ! petsc4py_installed "$env"; then echo " Installing petsc4py for $env..." - $PIXI run -e "$env" pip install "$PETSC_CUSTOM/src/binding/petsc4py" --no-build-isolation || { + $PIXI run -e "$env" pip install "$PETSC_CUSTOM/src/binding/petsc4py" --no-build-isolation --no-cache-dir || { echo -e "${YELLOW}petsc4py build failed${NC}" exit 1 } @@ -297,11 +297,13 @@ run_petsc_cmd() { # Nuke stale build artifacts and petsc4py. The next ./uw build's # target-change detection can't fire because the still-installed # petsc4py reports the OLD arch, so we force a full rebuild here. + # (run_build already passes --no-cache-dir to pip install . so we + # don't need a global pip cache purge — only petsc4py's local-path + # install needs its own --no-cache-dir.) echo "" echo "Clearing stale build artifacts for clean rebuild..." rm -rf "$SCRIPT_DIR"/build/lib.* "$SCRIPT_DIR"/build/temp.* "$SCRIPT_DIR"/build/bdist.* 2>/dev/null rm -f "$SCRIPT_DIR/build/.petsc_target" - $PIXI run -e "$(get_env)" pip cache purge 2>/dev/null || true if is_amr_env "$(get_env)"; then echo "Uninstalling old petsc4py (linked to previous PETSc build)..." $PIXI run -e "$(get_env)" pip uninstall -y petsc4py 2>/dev/null || true