Skip to content

Add PETSc version switching infrastructure #117

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
lmoresi merged 1 commit into from
Apr 17, 2026
Merged

Add PETSc version switching infrastructure #117

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -255,4 +255,5 @@ docs/beginner/tutorials/html5/*.html
.pixi-env
# PETSc build (directory in main repo, symlink in worktrees)
petsc-custom/petsc
petsc-custom/.petsc-version
Untitled*.ipynb
148 changes: 132 additions & 16 deletions petsc-custom/build-petsc.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,29 @@ set -e

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PETSC_DIR="${SCRIPT_DIR}/petsc"
VERSION_FILE="${SCRIPT_DIR}/.petsc-version"

# ── PETSc version detection ──────────────────────────────────────────────────
# Read version from .petsc-version file, or detect from the git checkout.
# Returns short form like "324" for v3.24.x or "325" for v3.25.x.

petsc_version_short() {
local ver=""
if [ -f "${VERSION_FILE}" ]; then
ver=$(cat "${VERSION_FILE}" | tr -d '[:space:]')
fi
if [ -z "${ver}" ] && [ -d "${PETSC_DIR}/.git" ]; then
# Detect from git tag: v3.24.2 → 324
local tag
tag=$(cd "${PETSC_DIR}" && git describe --tags --abbrev=0 2>/dev/null || echo "")
if [ -n "${tag}" ]; then
ver=$(echo "${tag}" | sed -E 's/^v([0-9]+)\.([0-9]+).*/\1\2/')
fi
fi
echo "${ver:-324}" # default to 324 if detection fails
}

PETSC_VER=$(petsc_version_short)

# ── Cluster detection ─────────────────────────────────────────────────────────
detect_cluster() {
Expand Down Expand Up @@ -91,7 +114,7 @@ EOF
}

MPI_IMPL=$(_detect_local_mpi)
PETSC_ARCH="petsc-4-uw-${MPI_IMPL}"
PETSC_ARCH="petsc-${PETSC_VER}-uw-${MPI_IMPL}"
;;

kaiju)
Expand All @@ -107,7 +130,7 @@ EOF
fi
MPI_DIR="$(dirname "$(dirname "$(which mpicc)")")"
MPI_IMPL="openmpi"
PETSC_ARCH="petsc-4-uw-openmpi"
PETSC_ARCH="petsc-${PETSC_VER}-uw-openmpi"
;;

gadi)
Expand All @@ -128,7 +151,7 @@ EOF
fi
MPI_DIR="$(dirname "$(dirname "$(which mpicc)")")"
MPI_IMPL="openmpi"
PETSC_ARCH="petsc-4-uw-openmpi"
PETSC_ARCH="petsc-${PETSC_VER}-uw-openmpi"
;;

*)
Expand Down Expand Up @@ -387,27 +410,118 @@ clean_all() {
fi
}

checkout_version() {
# Checkout a specific PETSc version tag and update .petsc-version
local tag="$1"
if [ -z "${tag}" ]; then
echo "Usage: $0 checkout <version-tag>"
echo " e.g.: $0 checkout v3.25.0"
echo ""
echo "Available tags:"
cd "${PETSC_DIR}" && git tag --list 'v3.2[0-9]*' | sort -V | tail -10
exit 1
fi

cd "${PETSC_DIR}"

# Fetch latest tags
git fetch --tags 2>/dev/null
Comment on lines +425 to +428

Copilot AI Apr 17, 2026

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

checkout_version() assumes ${PETSC_DIR} already exists and is a git repo (cd "${PETSC_DIR}"), so ./uw petsc switch ... will hard-fail on a fresh clone before any PETSc build/clone has happened. Consider checking for ${PETSC_DIR}/.git (or ${PETSC_DIR}/configure) up front and either running clone_petsc automatically or printing a clear instruction to run the build/clone step first.

Copilot uses AI. Check for mistakes.

if ! git rev-parse "${tag}" &>/dev/null; then
echo "Error: tag '${tag}' not found."
echo "Available tags:"
git tag --list 'v3.2[0-9]*' | sort -V | tail -10
exit 1
fi

echo "Checking out PETSc ${tag}..."
git checkout "${tag}"

# Update .petsc-version file
local ver_short
ver_short=$(echo "${tag}" | sed -E 's/^v([0-9]+)\.([0-9]+).*/\1\2/')
echo "${ver_short}" > "${VERSION_FILE}"
echo "Active PETSc version: ${ver_short} (${tag})"
Comment on lines +437 to +444

Copilot AI Apr 17, 2026

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

checkout_version() runs git checkout <tag> but the PETSc working tree is typically left dirty after apply_patches() (it uses git apply without committing). In that common case, git checkout will fail and version switching won’t work. Consider making checkout resilient by detecting a dirty tree and either stashing/resetting (with an explicit warning) or forcing checkout and then re-running apply_patches() for the new tag.

Copilot uses AI. Check for mistakes.
echo " Stored in: ${VERSION_FILE}"

# Check if a build exists for this version
local new_arch="petsc-${ver_short}-uw-${MPI_IMPL}"
if [ -d "${PETSC_DIR}/${new_arch}/lib" ]; then
echo " Existing build found: ${new_arch}"
echo " Run: ./uw build (to rebuild petsc4py + UW3)"
else
echo " No build for ${new_arch} yet."
echo " Run: pixi run -e <env> ./petsc-custom/build-petsc.sh"
fi
}

list_versions() {
echo "PETSc version builds:"
echo ""
local active_ver
active_ver=$(petsc_version_short)
for arch_dir in "${PETSC_DIR}"/petsc-*-uw-*/; do
[ -d "${arch_dir}" ] || continue
local arch_name=$(basename "${arch_dir}")
local marker=""
local style=""

# Classify: versioned (petsc-324-uw-*) vs legacy (petsc-4-uw-*)
if echo "${arch_name}" | grep -qE '^petsc-[0-9]{3,}-uw-'; then
# Versioned arch: petsc-324-uw-openmpi → 324
local arch_ver=$(echo "${arch_name}" | sed -E 's/^petsc-([0-9]+)-uw-.*/\1/')
if [ "${arch_ver}" = "${active_ver}" ]; then
marker=" (active)"
fi
else
style=" [legacy]"
fi

local has_lib=""
if [ -f "${arch_dir}/lib/libpetsc.dylib" ] || [ -f "${arch_dir}/lib/libpetsc.so" ]; then
has_lib=" [built]"
else
has_lib=" [incomplete]"
fi
echo " ${arch_name}${has_lib}${style}${marker}"
done
echo ""
if [ -d "${PETSC_DIR}/.git" ]; then
local tag
tag=$(cd "${PETSC_DIR}" && git describe --tags --abbrev=0 2>/dev/null || echo "unknown")
echo "Source checkout: ${tag}"
fi
echo "Active version: ${active_ver} (from ${VERSION_FILE})"
}

show_help() {
echo "Usage: $0 [command]"
echo ""
echo "Cluster: ${CLUSTER} (override: export UW_CLUSTER=local|kaiju|gadi)"
echo "PETSC_ARCH: ${PETSC_ARCH}"
echo "PETSC_ARCH: ${PETSC_ARCH} (version: ${PETSC_VER})"
echo ""
echo "Commands:"
echo " (none) Full build: clone, patch, configure, build"
[ "${CLUSTER}" = "local" ] && echo " (local: also runs petsc4py separately)"
echo " clone Clone PETSc repository"
echo " patch Apply UW3 patches to PETSc source"
echo " configure Configure PETSc with AMR tools"
echo " build Build PETSc"
echo " test Run PETSc tests"
echo " petsc4py Build and install petsc4py (local only)"
echo " clean Remove build for current arch (${PETSC_ARCH})"
echo " clean-all Remove entire PETSc directory (all builds)"
echo " help Show this help"
echo " (none) Full build: clone, patch, configure, build"
[ "${CLUSTER}" = "local" ] && echo " (local: also runs petsc4py separately)"
echo " clone Clone PETSc repository"
echo " checkout V Checkout PETSc version tag (e.g. v3.25.0) and set active"
echo " patch Apply UW3 patches to PETSc source"
echo " configure Configure PETSc with AMR tools"
echo " build Build PETSc"
echo " test Run PETSc tests"
echo " petsc4py Build and install petsc4py (local only)"
echo " versions List available PETSc builds"
echo " clean Remove build for current arch (${PETSC_ARCH})"
echo " clean-all Remove entire PETSc directory (all builds)"
echo " help Show this help"
if [ "${CLUSTER}" = "local" ]; then
echo ""
echo "MPICH and OpenMPI builds co-exist. To build both:"
echo "Version switching:"
echo " $0 checkout v3.25.0 # switch source to 3.25.0"
echo " $0 # build (creates petsc-325-uw-openmpi)"
echo " $0 checkout v3.24.2 # switch back (existing build reused)"
echo ""
echo "MPI variants co-exist. To build both:"
echo " pixi run -e amr ./petsc-custom/build-petsc.sh"
echo " pixi run -e amr-openmpi ./petsc-custom/build-petsc.sh"
fi
Expand All @@ -434,11 +548,13 @@ case "${1:-all}" in
echo "=========================================="
;;
clone) clone_petsc ;;
checkout) checkout_version "$2" ;;
patch) apply_patches ;;
configure) configure_petsc ;;
build) build_petsc ;;
test) test_petsc ;;
petsc4py) build_petsc4py ;;
versions) list_versions ;;
clean) clean_petsc ;;
clean-all) clean_all ;;
help|--help|-h) show_help ;;
Expand Down
90 changes: 82 additions & 8 deletions uw
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,24 +42,57 @@ get_env() {
fi
}

petsc_version_short() {
# Read active PETSc version from .petsc-version, or detect from checkout.
local ver_file="$SCRIPT_DIR/petsc-custom/.petsc-version"
if [ -f "$ver_file" ]; then
cat "$ver_file" | tr -d '[:space:]'
return
fi
# Detect from git tag
if [ -d "$PETSC_CUSTOM/.git" ]; then
local tag
tag=$(cd "$PETSC_CUSTOM" && git describe --tags --abbrev=0 2>/dev/null || echo "")
if [ -n "$tag" ]; then
echo "$tag" | sed -E 's/^v([0-9]+)\.([0-9]+).*/\1\2/'
return
fi
fi
echo "324" # default
}

petsc_arch_for_env() {
# Derive the PETSC_ARCH from the environment name.
# Primary AMR envs (amr, amr-runtime, amr-dev) use platform-default MPI.
# Override envs (amr-mpich*, amr-openmpi*) force a specific MPI.
# Derive the PETSC_ARCH from the environment name and active PETSc version.
# Format: petsc-{version}-uw-{mpi} (e.g. petsc-324-uw-openmpi)
local env="$1"
local ver=$(petsc_version_short)
local mpi=""
Comment on lines 64 to +69

Copilot AI Apr 17, 2026

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

petsc_arch_for_env() now selects a versioned arch (e.g. petsc-325-uw-openmpi), but this wrapper never exports/overrides PETSC_ARCH (and PETSC_DIR) when running pixi run for the AMR build steps (notably the pip install of petsc4py). Since the pixi environments still set legacy PETSC_ARCH=petsc-4-uw-*, switching versions can leave petsc4py/UW3 building against the wrong arch (or failing to find the new one). Consider explicitly setting PETSC_ARCH=$(petsc_arch_for_env "$env") (and PETSC_DIR=$PETSC_CUSTOM) in the environment for relevant pixi run invocations so the selected version is actually used.

Copilot uses AI. Check for mistakes.

case "$env" in
amr-mpich*) echo "petsc-4-uw-mpich" ;;
amr-openmpi*) echo "petsc-4-uw-openmpi" ;;
amr-mpich*) mpi="mpich" ;;
amr-openmpi*) mpi="openmpi" ;;
amr*)
# Platform-default MPI
if [[ "$(uname -s)" == "Darwin" ]]; then
echo "petsc-4-uw-openmpi"
mpi="openmpi"
else
echo "petsc-4-uw-mpich"
mpi="mpich"
fi
;;
*) echo "" ;; # conda-forge, no custom PETSc
*) echo ""; return ;; # conda-forge, no custom PETSc
esac

# Check for versioned arch first, fall back to legacy unversioned
local versioned="petsc-${ver}-uw-${mpi}"
local legacy="petsc-4-uw-${mpi}"
if [ -d "$PETSC_CUSTOM/${versioned}" ]; then
echo "${versioned}"
elif [ -d "$PETSC_CUSTOM/${legacy}" ]; then
echo "${legacy}"
else
# No build exists yet — return the versioned name for new builds
echo "${versioned}"
fi
}

petsc_built() {
Expand Down Expand Up @@ -221,6 +254,43 @@ run_build() {
echo " ./uw doctor (check configuration)"
}

run_petsc_cmd() {
local petsc_cmd="${1:-versions}"
local arg="$2"
case "$petsc_cmd" in
versions|list)
$PIXI run -e "$(get_env)" "$SCRIPT_DIR/petsc-custom/build-petsc.sh" versions
;;
switch)
if [ -z "$arg" ]; then
echo "Usage: ./uw petsc switch <version-tag>"
echo " e.g.: ./uw petsc switch v3.25.0"
exit 1
fi
$PIXI run -e "$(get_env)" "$SCRIPT_DIR/petsc-custom/build-petsc.sh" checkout "$arg"
echo ""
echo "To complete the switch, rebuild:"
echo " ./uw build"
;;
active)
local ver=$(petsc_version_short)
local arch=$(petsc_arch_for_env "$(get_env)")
echo "Active PETSc: ${ver} (arch: ${arch})"
;;
build)
$PIXI run -e "$(get_env)" "$SCRIPT_DIR/petsc-custom/build-petsc.sh"
;;
*)
echo "Usage: ./uw petsc [versions|switch <tag>|active|build]"
echo ""
echo " versions List available PETSc builds"
echo " switch <tag> Checkout a PETSc version (e.g. v3.25.0)"
echo " active Show active PETSc version and arch"
echo " build Build PETSc for the active version"
;;
esac
}

# Standalone diagnostics (works even if underworld3 not built)
run_doctor() {
local env=$(get_env)
Expand Down Expand Up @@ -1458,6 +1528,10 @@ case "${1:-}" in
rm -f "$SCRIPT_DIR/build/.petsc_target"
echo -e "${GREEN}Done${NC}. Run './uw build' to rebuild."
;;
petsc)
# PETSc version management: ./uw petsc [versions|switch|active]
run_petsc_cmd "$2" "$3"
;;
doctor)
run_doctor
;;
Expand Down
Loading