From f7d783dfe1b436732b8cc88b3242153320fe960a Mon Sep 17 00:00:00 2001 From: Giannis95 Date: Fri, 3 Jul 2026 14:36:41 +0100 Subject: [PATCH 1/5] Add Cumuluns 01_build_script which prefecthes the required submodules --- .../01_fetch_submodules_login-node.sh | 347 ++++++++++++++++++ 1 file changed, 347 insertions(+) create mode 100644 scripts/cumulus_build_scripts/01_fetch_submodules_login-node.sh diff --git a/scripts/cumulus_build_scripts/01_fetch_submodules_login-node.sh b/scripts/cumulus_build_scripts/01_fetch_submodules_login-node.sh new file mode 100644 index 0000000..62a2770 --- /dev/null +++ b/scripts/cumulus_build_scripts/01_fetch_submodules_login-node.sh @@ -0,0 +1,347 @@ +#!/bin/bash +# ============================================================================= +# 01_fetch_submodules_loginnode.sh +# +# Pre-fetch all submodules and external package tarballs that the +# MOOSE/MFEM build needs. Need to run on the login node (internet access) +# +# Hashes and URLs are read automatically from the checked-out repo, +# so this script works with any MOOSE version. +# +# USAGE : bash 01_fetch_submodules_loginnode.sh . +# (from any directory — it finds MOOSE via $HOME/moose) +# PS ./01_fetch_submodules_loginnode.sh won't work because it needs sudo access +# ============================================================================= + +set -e +set -u + +MOOSE_DIR="$HOME/moose" +PKGS_DIR="$HOME/petsc_packages" #directory where 02 point at + +# Commands for better readable and pretty output formatting + +info() { echo "[INFO] $*"; } +warn() { echo "[WARN] $*"; } +error() { echo "[ERROR] $*"; exit 1; } +step() { + echo "" + echo "=== $* ===" + echo "==================================================" +} + +# prechecks +step "Pre-flight checks" + +command -v git >/dev/null 2>&1 || error "git not found. Load it: module load git" + +[ -d "$MOOSE_DIR" ] || error "MOOSE not found at $MOOSE_DIR" +[ -f "$MOOSE_DIR/.gitmodules" ] || error "Not a git repo: $MOOSE_DIR" + +if [ -n "${SLURM_JOB_ID:-}" ]; then + warn "SLURM_JOB_ID is set — you appear to be on a compute node." + warn "This script needs internet. Continue anyway? [y/N]" + read -r ans; [[ "$ans" =~ ^[Yy]$ ]] || exit 1 +fi + +info "Checking internet connectivity..." +curl -s --max-time 5 https://github.com > /dev/null 2>&1 \ + || error "Cannot reach github.com — run this on a login node." +info "Internet OK." + +mkdir -p "$PKGS_DIR" + +# download resume and skips if already exists (function) +download() { + local url="$1" out="$2" + if [ -f "$out" ] && [ -s "$out" ] && file "$out" | grep -qE "gzip|bzip2|XZ|Zip|tar"; then + info "Already present: $(basename "$out") ($(du -sh "$out" | cut -f1))" + return 0 + fi + info "Downloading $(basename "$out") from $url ..." + if command -v wget >/dev/null 2>&1; then + wget -q --show-progress -c "$url" -O "$out" \ + || { warn "wget failed, retrying with curl..."; curl -L -C - -o "$out" "$url"; } + else + curl -L -C - -o "$out" "$url" + fi + # Validate — reject HTML error pages + if ! file "$out" | grep -qE "gzip|bzip2|XZ|Zip|tar"; then + rm -f "$out" + error "Download failed or returned non-tarball content for: $url" + fi + info " ✓ $(du -sh "$out" | cut -f1)" +} + +# get hash of a submodule from git (function) +submodule_hash() { + # Usage: submodule_hash + git -C "$1" ls-tree HEAD "$2" 2>/dev/null | awk '{print $3}' +} + +# clone + checkout a submodule that has update=none function +clone_pinned() { + local path="$1" url="$2" hash="$3" name="$4" + if [ -f "$path/CMakeLists.txt" ]; then + local current + current=$(git -C "$path" rev-parse HEAD 2>/dev/null || echo "unknown") + if [ "$current" = "$hash" ]; then + info "$name already at correct commit $hash — skipping." + return 0 + else + warn "$name checked out at $current, expected $hash — re-cloning." + fi + fi + info "Cloning $name @ $hash ..." + rm -rf "$path" + git clone "$url" "$path" + cd "$path" + git fetch --all --quiet + git checkout "$hash" + git submodule update --init --recursive + info "$name ✓" +} + + +# SECTION 1 — PETSc submodule + +step "Section 1: PETSc submodule" + +cd "$MOOSE_DIR" +git submodule update --init petsc +info "PETSc submodule ✓" + +# SECTION 2 — Determine PETSc external package URLs from configure.py +# in the from the moose directory +# Here PETSc configure in "offline dry-run" mode on the login node. +# It exits with code 10 and prints the URLs it needs — then automagically +# download them automatically. + +step "Section 2: PETSc external package tarballs (auto-detected URLs)" + +# Load python3 (may already be available or need a module) +command -v python3 >/dev/null 2>&1 || error "python3 not found — load it: module load Python" + +info "Running PETSc configure in dry-run / download-dir mode to detect URLs..." +info "This will exit with code 10 (normal) and print the list of packages." +echo "" + +CONFIGURE_OUT=$( + cd "$MOOSE_DIR/petsc" + python3 ./configure \ + --with-packages-download-dir="$PKGS_DIR" \ + --with-debugging=no \ + --with-mpi=1 \ + --with-shared-libraries=1 \ + --with-cxx-dialect=C++17 \ + --with-fortran-bindings=0 \ + --with-sowing=0 \ + --download-fblaslapack=1 \ + --download-hypre=1 \ + --download-metis=1 \ + --download-parmetis=1 \ + --download-mumps=1 \ + --download-scalapack=1 \ + --download-superlu_dist=1 \ + --download-slepc=1 \ + --download-strumpack=1 \ + --download-hdf5=1 \ + --download-libceed=1 \ + --download-openblas=1 \ + --download-hpddm=1 \ + --download-ptscotch=0 \ + --download-kokkos=0 \ + --download-kokkos-kernels=0 \ + --download-umpire=0 \ + 2>&1 || true # exit 10 is expected +) + +echo "$CONFIGURE_OUT" | grep -v "^Executing\|^stdout\|^$" | head -60 || true +echo "" + +# Parse URLs from the output and download each one +# PETSc output format: "pkgname ['url1', 'url2', ...]" +info "Parsing and downloading packages listed by PETSc configure..." + +# Parse URLs from PETSc output. +# Output format (one package per line): +# pkgname ['git clone https://...', 'https://.../foo.tar.gz'] +# extract only https:// tarball URLs (not git clone lines). +while IFS= read -r line; do + # Match lines that start with a word then a space then [' + if echo "$line" | grep -qE "^[a-zA-Z_][a-zA-Z0-9_]* \['" ; then + pkg=$(echo "$line" | awk '{print $1}') + + # Pull out every https URL ending in .tar.gz or .tgz + # Use grep -oE (extended, no Perl) for broader compatibility + urls=$(echo "$line" | grep -oE "https://[^']+\.(tar\.gz|tgz)" || true) + + if [ -z "$urls" ]; then + warn "No tarball URL found for $pkg — may need manual download" + continue + fi + + # Prefer ANL mirror if available, otherwise take the first URL + url=$(echo "$urls" | grep "cels.anl.gov" | head -1) + [ -z "$url" ] && url=$(echo "$urls" | head -1) + + # Safety: skip if URL is still empty or doesn't look like a URL + if [ -z "$url" ] || ! echo "$url" | grep -q "^https://"; then + warn "Skipping $pkg — could not extract a valid URL from: $line" + continue + fi + + # Filename is the last path component of the URL + fname=$(basename "$url") + + # Safety: skip if fname is empty or looks like a directory + if [ -z "$fname" ] || [ -d "$PKGS_DIR/$fname" ]; then + warn "Skipping $pkg — could not determine output filename from URL: $url" + continue + fi + + info "Package: $pkg -> $fname" + download "$url" "$PKGS_DIR/$fname" + fi +done <<< "$CONFIGURE_OUT" + +info "All PETSc tarballs downloaded into $PKGS_DIR" +echo "" +ls -lh "$PKGS_DIR" + +# SECTION 3 — libMesh submodules (eigen, netgen) +# netgen on cumulus module goes up to version GCCcore-12.3.0 +# Here version >13.0.0 is needed aparently +step "Section 3: libMesh submodules" + +cd "$MOOSE_DIR" +git submodule update --init --recursive libmesh +info "libMesh submodules ✓" + + +# SECTION 4 — WASP +step "Section 4: WASP submodule" + +cd "$MOOSE_DIR" +git submodule update --init --recursive framework/contrib/wasp 2>/dev/null \ + || info "WASP may be handled internally by update_and_rebuild_wasp.sh — continuing." + +# SECTION 5 — Conduit (update = none in .gitmodules — must clone manually) +step "Section 5: Conduit" + +CONDUIT_HASH=$(submodule_hash "$MOOSE_DIR" "framework/contrib/conduit") +[ -n "$CONDUIT_HASH" ] || error "Could not read Conduit hash from git ls-tree." +info "Pinned Conduit commit: $CONDUIT_HASH" + +# Resolve relative URL from .gitmodules (../../LLNL/conduit.git) +MOOSE_REMOTE=$(git -C "$MOOSE_DIR" remote get-url origin) +CONDUIT_URL="https://github.com/LLNL/conduit.git" +info "Conduit URL: $CONDUIT_URL" + +clone_pinned \ + "$MOOSE_DIR/framework/contrib/conduit" \ + "$CONDUIT_URL" \ + "$CONDUIT_HASH" \ + "Conduit" + +# SECTION 6 — MFEM (update = none in .gitmodules — must clone manually) +step "Section 6: MFEM" + +MFEM_HASH=$(submodule_hash "$MOOSE_DIR" "framework/contrib/mfem") +[ -n "$MFEM_HASH" ] || error "Could not read MFEM hash from git ls-tree." +info "Pinned MFEM commit: $MFEM_HASH" +MFEM_URL="https://github.com/mfem/mfem.git" + +clone_pinned \ + "$MOOSE_DIR/framework/contrib/mfem" \ + "$MFEM_URL" \ + "$MFEM_HASH" \ + "MFEM" + +# SECTION 7 — GSLIB (MFEM fetches this at build time; pre-clone to avoid +# the compute-node network error during make) +step "Section 7: GSLIB (required by MFEM)" + +# Read the tag/commit MFEM's CMakeLists expects +MFEM_CMAKE="$MOOSE_DIR/framework/contrib/mfem/CMakeLists.txt" +GSLIB_TAG=$(grep -A5 "gslib\|GSLIB" "$MFEM_CMAKE" 2>/dev/null \ + | grep -iE "GIT_TAG|VERSION" | head -1 \ + | grep -oP '[0-9]+\.[0-9]+\.[0-9]+' || echo "") + +GSLIB_DIR="$PKGS_DIR/gslib" +GSLIB_URL="https://github.com/Nek5000/gslib.git" + +if [ -d "$GSLIB_DIR/.git" ]; then + info "GSLIB already cloned at $GSLIB_DIR" + cd "$GSLIB_DIR" + git fetch --all --quiet +else + info "Cloning GSLIB into $GSLIB_DIR ..." + git clone "$GSLIB_URL" "$GSLIB_DIR" + cd "$GSLIB_DIR" +fi + +if [ -n "$GSLIB_TAG" ]; then + info "Checking out GSLIB tag/version: $GSLIB_TAG" + git checkout "v${GSLIB_TAG}" 2>/dev/null \ + || git checkout "$GSLIB_TAG" 2>/dev/null \ + || warn "Could not checkout tag $GSLIB_TAG — using default branch" +else + warn "Could not auto-detect GSLIB version from MFEM CMakeLists — using latest main" + git checkout main 2>/dev/null || git checkout master 2>/dev/null || true +fi +info "GSLIB ✓ (stored at $GSLIB_DIR)" + + +# SECTION 8 — HYPRE hash verification +# (PETSc configure already handled the download above, but we double-check +# the exact commit hash that MFEM's FindHYPRE.cmake requires, if any) + +step "Section 8: Verify HYPRE hash" + +HYPRE_FILE=$(ls "$PKGS_DIR"/*hypre* 2>/dev/null \ + || ls "$PKGS_DIR"/85b7795* 2>/dev/null \ + || true | head -1) +if [ -n "$HYPRE_FILE" ] && [ -s "$HYPRE_FILE" ]; then + info "HYPRE tarball present: $(basename "$HYPRE_FILE") ($(du -sh "$HYPRE_FILE" | cut -f1))" +else + # PETSc configure should have got it; if not, grab it explicitly + warn "HYPRE tarball not detected — fetching manually..." + HYPRE_HASH=$(grep -r "hypre" "$MOOSE_DIR/petsc/config/BuildSystem/config/packages/HYPRE.py" \ + 2>/dev/null | grep -oP '[0-9a-f]{40}' | head -1 || echo "") + if [ -n "$HYPRE_HASH" ]; then + info "HYPRE hash from PETSc config: $HYPRE_HASH" + download \ + "https://github.com/hypre-space/hypre/archive/${HYPRE_HASH}.tar.gz" \ + "$PKGS_DIR/${HYPRE_HASH}.tar.gz" + else + warn "Could not auto-detect HYPRE hash — check $PKGS_DIR manually." + fi +fi + + +# FINAL SUMMARY +# Print out summary with relevant info for 02_build_moose_computenode.sh +#Also sanity check + +echo "" +echo "============================================================" +echo " Pre-fetch complete!" +echo "============================================================" +echo "" +echo " Tarballs staging dir : $PKGS_DIR" +echo " Tarball count : $(ls "$PKGS_DIR"/*.tar.gz 2>/dev/null | wc -l)" +du -sh "$PKGS_DIR" +echo "" +echo " Submodule status:" +git -C "$MOOSE_DIR" submodule status petsc libmesh \ + framework/contrib/conduit framework/contrib/mfem 2>/dev/null || true +echo "" +echo " GSLIB cached at : $GSLIB_DIR" +echo "" +echo " Next step:" +echo " Request a compute node:" +echo " srun --pty --time=08:00:00 --ntasks=8 --mem=32G bash" +echo " Then run:" +echo " bash 02_build_moose_computenode.sh" +echo "============================================================" From a19d9a6c097a7ef2e538eb6a024a98400fbf0914 Mon Sep 17 00:00:00 2001 From: Giannis95 Date: Fri, 3 Jul 2026 16:39:18 +0100 Subject: [PATCH 2/5] Clean up SHELL_SHFMT and symbols. Update step() output function --- .../01_fetch_submodules_login-node.sh | 44 +++++++++---------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/scripts/cumulus_build_scripts/01_fetch_submodules_login-node.sh b/scripts/cumulus_build_scripts/01_fetch_submodules_login-node.sh index 62a2770..d8d4d09 100644 --- a/scripts/cumulus_build_scripts/01_fetch_submodules_login-node.sh +++ b/scripts/cumulus_build_scripts/01_fetch_submodules_login-node.sh @@ -10,7 +10,7 @@ # # USAGE : bash 01_fetch_submodules_loginnode.sh . # (from any directory — it finds MOOSE via $HOME/moose) -# PS ./01_fetch_submodules_loginnode.sh won't work because it needs sudo access +# PS ./01_fetch_submodules_loginnode.sh won't work because it needs sudo rights # ============================================================================= set -e @@ -20,10 +20,12 @@ MOOSE_DIR="$HOME/moose" PKGS_DIR="$HOME/petsc_packages" #directory where 02 point at # Commands for better readable and pretty output formatting - -info() { echo "[INFO] $*"; } -warn() { echo "[WARN] $*"; } -error() { echo "[ERROR] $*"; exit 1; } +info() { echo "[INFO] $*"; } +warn() { echo "[WARN] $*"; } +error() { + echo "[ERROR] $*" + exit 1 +} step() { echo "" echo "=== $* ===" @@ -35,18 +37,19 @@ step "Pre-flight checks" command -v git >/dev/null 2>&1 || error "git not found. Load it: module load git" -[ -d "$MOOSE_DIR" ] || error "MOOSE not found at $MOOSE_DIR" +[ -d "$MOOSE_DIR" ] || error "MOOSE not found at $MOOSE_DIR" [ -f "$MOOSE_DIR/.gitmodules" ] || error "Not a git repo: $MOOSE_DIR" if [ -n "${SLURM_JOB_ID:-}" ]; then - warn "SLURM_JOB_ID is set — you appear to be on a compute node." + warn "SLURM_JOB_ID is set - you appear to be on a compute node." warn "This script needs internet. Continue anyway? [y/N]" - read -r ans; [[ "$ans" =~ ^[Yy]$ ]] || exit 1 + read -r ans + [[ "$ans" =~ ^[Yy]$ ]] || exit 1 fi info "Checking internet connectivity..." -curl -s --max-time 5 https://github.com > /dev/null 2>&1 \ - || error "Cannot reach github.com — run this on a login node." +curl -s --max-time 5 https://github.com >/dev/null 2>&1 \ + || error "Cannot reach github.com - run this on a login node." info "Internet OK." mkdir -p "$PKGS_DIR" @@ -58,19 +61,18 @@ download() { info "Already present: $(basename "$out") ($(du -sh "$out" | cut -f1))" return 0 fi - info "Downloading $(basename "$out") from $url ..." + info "Downloading $(basename "$out") ..." if command -v wget >/dev/null 2>&1; then wget -q --show-progress -c "$url" -O "$out" \ - || { warn "wget failed, retrying with curl..."; curl -L -C - -o "$out" "$url"; } + || { warn "wget failed, retrying with curl..."; curl -L -o "$out" "$url"; } else - curl -L -C - -o "$out" "$url" + curl -L -o "$out" "$url" fi - # Validate — reject HTML error pages if ! file "$out" | grep -qE "gzip|bzip2|XZ|Zip|tar"; then rm -f "$out" error "Download failed or returned non-tarball content for: $url" fi - info " ✓ $(du -sh "$out" | cut -f1)" + info " OK: $(du -sh "$out" | cut -f1)" } # get hash of a submodule from git (function) @@ -86,10 +88,10 @@ clone_pinned() { local current current=$(git -C "$path" rev-parse HEAD 2>/dev/null || echo "unknown") if [ "$current" = "$hash" ]; then - info "$name already at correct commit $hash — skipping." + info "$name already at correct commit $hash - skipping." return 0 else - warn "$name checked out at $current, expected $hash — re-cloning." + warn "$name checked out at $current, expected $hash - re-cloning." fi fi info "Cloning $name @ $hash ..." @@ -99,7 +101,7 @@ clone_pinned() { git fetch --all --quiet git checkout "$hash" git submodule update --init --recursive - info "$name ✓" + info "$name cloned OK" } @@ -109,7 +111,7 @@ step "Section 1: PETSc submodule" cd "$MOOSE_DIR" git submodule update --init petsc -info "PETSc submodule ✓" +info "PETSc submodule OK" # SECTION 2 — Determine PETSc external package URLs from configure.py # in the from the moose directory @@ -159,8 +161,6 @@ CONFIGURE_OUT=$( echo "$CONFIGURE_OUT" | grep -v "^Executing\|^stdout\|^$" | head -60 || true echo "" -# Parse URLs from the output and download each one -# PETSc output format: "pkgname ['url1', 'url2', ...]" info "Parsing and downloading packages listed by PETSc configure..." # Parse URLs from PETSc output. @@ -194,7 +194,7 @@ while IFS= read -r line; do # Filename is the last path component of the URL fname=$(basename "$url") - # Safety: skip if fname is empty or looks like a directory + # Safety: skip if filenname is empty or looks like a directory if [ -z "$fname" ] || [ -d "$PKGS_DIR/$fname" ]; then warn "Skipping $pkg — could not determine output filename from URL: $url" continue From e55edc12688b2df075123c440eb1de98dc98d892 Mon Sep 17 00:00:00 2001 From: Giannis95 Date: Fri, 3 Jul 2026 16:57:45 +0100 Subject: [PATCH 3/5] fixed trailling whitespaces, general shfmt changes, and deleted unused variables. --- .../01_fetch_submodules_login-node.sh | 94 +++++++++---------- 1 file changed, 43 insertions(+), 51 deletions(-) diff --git a/scripts/cumulus_build_scripts/01_fetch_submodules_login-node.sh b/scripts/cumulus_build_scripts/01_fetch_submodules_login-node.sh index d8d4d09..88a8f4d 100644 --- a/scripts/cumulus_build_scripts/01_fetch_submodules_login-node.sh +++ b/scripts/cumulus_build_scripts/01_fetch_submodules_login-node.sh @@ -8,7 +8,7 @@ # Hashes and URLs are read automatically from the checked-out repo, # so this script works with any MOOSE version. # -# USAGE : bash 01_fetch_submodules_loginnode.sh . +# USAGE : bash 01_fetch_submodules_loginnode.sh # (from any directory — it finds MOOSE via $HOME/moose) # PS ./01_fetch_submodules_loginnode.sh won't work because it needs sudo rights # ============================================================================= @@ -20,8 +20,8 @@ MOOSE_DIR="$HOME/moose" PKGS_DIR="$HOME/petsc_packages" #directory where 02 point at # Commands for better readable and pretty output formatting -info() { echo "[INFO] $*"; } -warn() { echo "[WARN] $*"; } +info() { echo "[INFO] $*"; } +warn() { echo "[WARN] $*"; } error() { echo "[ERROR] $*" exit 1 @@ -32,7 +32,7 @@ step() { echo "==================================================" } -# prechecks +# prechecks step "Pre-flight checks" command -v git >/dev/null 2>&1 || error "git not found. Load it: module load git" @@ -104,7 +104,6 @@ clone_pinned() { info "$name cloned OK" } - # SECTION 1 — PETSc submodule step "Section 1: PETSc submodule" @@ -129,33 +128,33 @@ info "This will exit with code 10 (normal) and print the list of packages." echo "" CONFIGURE_OUT=$( - cd "$MOOSE_DIR/petsc" - python3 ./configure \ - --with-packages-download-dir="$PKGS_DIR" \ - --with-debugging=no \ - --with-mpi=1 \ - --with-shared-libraries=1 \ - --with-cxx-dialect=C++17 \ - --with-fortran-bindings=0 \ - --with-sowing=0 \ - --download-fblaslapack=1 \ - --download-hypre=1 \ - --download-metis=1 \ - --download-parmetis=1 \ - --download-mumps=1 \ - --download-scalapack=1 \ - --download-superlu_dist=1 \ - --download-slepc=1 \ - --download-strumpack=1 \ - --download-hdf5=1 \ - --download-libceed=1 \ - --download-openblas=1 \ - --download-hpddm=1 \ - --download-ptscotch=0 \ - --download-kokkos=0 \ - --download-kokkos-kernels=0 \ - --download-umpire=0 \ - 2>&1 || true # exit 10 is expected + cd "$MOOSE_DIR/petsc" + python3 ./configure \ + --with-packages-download-dir="$PKGS_DIR" \ + --with-debugging=no \ + --with-mpi=1 \ + --with-shared-libraries=1 \ + --with-cxx-dialect=C++17 \ + --with-fortran-bindings=0 \ + --with-sowing=0 \ + --download-fblaslapack=1 \ + --download-hypre=1 \ + --download-metis=1 \ + --download-parmetis=1 \ + --download-mumps=1 \ + --download-scalapack=1 \ + --download-superlu_dist=1 \ + --download-slepc=1 \ + --download-strumpack=1 \ + --download-hdf5=1 \ + --download-libceed=1 \ + --download-openblas=1 \ + --download-hpddm=1 \ + --download-ptscotch=0 \ + --download-kokkos=0 \ + --download-kokkos-kernels=0 \ + --download-umpire=0 \ + 2>&1 || true # exit 10 is expected ) echo "$CONFIGURE_OUT" | grep -v "^Executing\|^stdout\|^$" | head -60 || true @@ -169,7 +168,7 @@ info "Parsing and downloading packages listed by PETSc configure..." # extract only https:// tarball URLs (not git clone lines). while IFS= read -r line; do # Match lines that start with a word then a space then [' - if echo "$line" | grep -qE "^[a-zA-Z_][a-zA-Z0-9_]* \['" ; then + if echo "$line" | grep -qE "^[a-zA-Z_][a-zA-Z0-9_]* \['"; then pkg=$(echo "$line" | awk '{print $1}') # Pull out every https URL ending in .tar.gz or .tgz @@ -203,21 +202,20 @@ while IFS= read -r line; do info "Package: $pkg -> $fname" download "$url" "$PKGS_DIR/$fname" fi -done <<< "$CONFIGURE_OUT" +done <<<"$CONFIGURE_OUT" info "All PETSc tarballs downloaded into $PKGS_DIR" echo "" ls -lh "$PKGS_DIR" # SECTION 3 — libMesh submodules (eigen, netgen) -# netgen on cumulus module goes up to version GCCcore-12.3.0 +# netgen on cumulus module goes up to version GCCcore-12.3.0 # Here version >13.0.0 is needed aparently step "Section 3: libMesh submodules" cd "$MOOSE_DIR" git submodule update --init --recursive libmesh -info "libMesh submodules ✓" - +info "libMesh submodules OK" # SECTION 4 — WASP step "Section 4: WASP submodule" @@ -234,9 +232,7 @@ CONDUIT_HASH=$(submodule_hash "$MOOSE_DIR" "framework/contrib/conduit") info "Pinned Conduit commit: $CONDUIT_HASH" # Resolve relative URL from .gitmodules (../../LLNL/conduit.git) -MOOSE_REMOTE=$(git -C "$MOOSE_DIR" remote get-url origin) CONDUIT_URL="https://github.com/LLNL/conduit.git" -info "Conduit URL: $CONDUIT_URL" clone_pinned \ "$MOOSE_DIR/framework/contrib/conduit" \ @@ -265,8 +261,8 @@ step "Section 7: GSLIB (required by MFEM)" # Read the tag/commit MFEM's CMakeLists expects MFEM_CMAKE="$MOOSE_DIR/framework/contrib/mfem/CMakeLists.txt" GSLIB_TAG=$(grep -A5 "gslib\|GSLIB" "$MFEM_CMAKE" 2>/dev/null \ - | grep -iE "GIT_TAG|VERSION" | head -1 \ - | grep -oP '[0-9]+\.[0-9]+\.[0-9]+' || echo "") + | grep -iE "GIT_TAG|VERSION" | head -1 \ + | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' || echo "") GSLIB_DIR="$PKGS_DIR/gslib" GSLIB_URL="https://github.com/Nek5000/gslib.git" @@ -290,8 +286,7 @@ else warn "Could not auto-detect GSLIB version from MFEM CMakeLists — using latest main" git checkout main 2>/dev/null || git checkout master 2>/dev/null || true fi -info "GSLIB ✓ (stored at $GSLIB_DIR)" - +info "GSLIB OK (stored at $GSLIB_DIR)" # SECTION 8 — HYPRE hash verification # (PETSc configure already handled the download above, but we double-check @@ -299,16 +294,14 @@ info "GSLIB ✓ (stored at $GSLIB_DIR)" step "Section 8: Verify HYPRE hash" -HYPRE_FILE=$(ls "$PKGS_DIR"/*hypre* 2>/dev/null \ - || ls "$PKGS_DIR"/85b7795* 2>/dev/null \ - || true | head -1) +HYPRE_FILE=$(find "$PKGS_DIR" -maxdepth 1 -name "*hypre*" -type f 2>/dev/null | head -1) if [ -n "$HYPRE_FILE" ] && [ -s "$HYPRE_FILE" ]; then info "HYPRE tarball present: $(basename "$HYPRE_FILE") ($(du -sh "$HYPRE_FILE" | cut -f1))" else # PETSc configure should have got it; if not, grab it explicitly warn "HYPRE tarball not detected — fetching manually..." HYPRE_HASH=$(grep -r "hypre" "$MOOSE_DIR/petsc/config/BuildSystem/config/packages/HYPRE.py" \ - 2>/dev/null | grep -oP '[0-9a-f]{40}' | head -1 || echo "") + 2>/dev/null | grep -oE '[0-9a-f]{40}' | head -1 || echo "") if [ -n "$HYPRE_HASH" ]; then info "HYPRE hash from PETSc config: $HYPRE_HASH" download \ @@ -319,10 +312,9 @@ else fi fi - -# FINAL SUMMARY +# FINAL SUMMARY # Print out summary with relevant info for 02_build_moose_computenode.sh -#Also sanity check +# Also sanity check echo "" echo "============================================================" @@ -330,7 +322,7 @@ echo " Pre-fetch complete!" echo "============================================================" echo "" echo " Tarballs staging dir : $PKGS_DIR" -echo " Tarball count : $(ls "$PKGS_DIR"/*.tar.gz 2>/dev/null | wc -l)" +echo " Tarball count : $(find "$PKGS_DIR" -maxdepth 1 -name '*.tar.gz' | wc -l)" du -sh "$PKGS_DIR" echo "" echo " Submodule status:" From 2bbefd96a17bdcb160953e2dd8f7e5acf0231fc0 Mon Sep 17 00:00:00 2001 From: Giannis95 Date: Fri, 3 Jul 2026 17:07:04 +0100 Subject: [PATCH 4/5] fix: make .sh script executable chmod=+ --- .../cumulus_build_scripts/01_fetch_submodules_login-node.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/scripts/cumulus_build_scripts/01_fetch_submodules_login-node.sh b/scripts/cumulus_build_scripts/01_fetch_submodules_login-node.sh index 88a8f4d..086e73d 100644 --- a/scripts/cumulus_build_scripts/01_fetch_submodules_login-node.sh +++ b/scripts/cumulus_build_scripts/01_fetch_submodules_login-node.sh @@ -22,10 +22,7 @@ PKGS_DIR="$HOME/petsc_packages" #directory where 02 point at # Commands for better readable and pretty output formatting info() { echo "[INFO] $*"; } warn() { echo "[WARN] $*"; } -error() { - echo "[ERROR] $*" - exit 1 -} +error() { echo "[ERROR] $*"; exit 1; } step() { echo "" echo "=== $* ===" From d4831c12428cf3f71bf9fd14a74c34cb3ba161f6 Mon Sep 17 00:00:00 2001 From: Giannis95 Date: Wed, 22 Jul 2026 09:05:01 +0100 Subject: [PATCH 5/5] Draft build-in script for compute node build --- .../02_mfem-moose_compute-node-build.sh | 279 ++++++++++++++++++ 1 file changed, 279 insertions(+) create mode 100644 scripts/cumulus_build_scripts/02_mfem-moose_compute-node-build.sh diff --git a/scripts/cumulus_build_scripts/02_mfem-moose_compute-node-build.sh b/scripts/cumulus_build_scripts/02_mfem-moose_compute-node-build.sh new file mode 100644 index 0000000..5c7a398 --- /dev/null +++ b/scripts/cumulus_build_scripts/02_mfem-moose_compute-node-build.sh @@ -0,0 +1,279 @@ +#!/bin/bash +# ============================================================================= +# 02_build_moose_computenode.sh +# +# Build PETSc, libMesh, WASP, Conduit and MFEM (with GSLIB) on a compute +# node, offline. All tarballs and submodules must have been +# pre-fetched by 01_fetch_submodules_loginnode.sh first. +# +# There are two ways to run this script: +# 1: Interactive (request a node first): +# srun --pty --time=08:00:00 --ntasks=8 --mem=32G bash +# bash 02_build_moose_computenode.sh +# +# 2:Batch job submission (submit the script to SLURM): +# sbatch 02_build_moose_computenode.sh +# +# SLURM DIRECTIVES - edit as needed: +#SBATCH --job-name=moose_build +#SBATCH --ntasks=8 +#SBATCH --mem=32G +#SBATCH --time=08:00:00 +#SBATCH --output=moose_build_%j.log +#SBATCH --error=moose_build_%j.err +# ============================================================================= + +set -e +set -u + +# ============================================================================= +# CONFIGURATION - edit paths here if your layout differs +# ============================================================================= +MOOSE_DIR="$HOME/moose" +PKGS_DIR="$HOME/petsc_packages" # tarballs downloaded by script 01 +PETSC_DIR="$MOOSE_DIR/petsc" +PETSC_ARCH="arch-moose" +GSLIB_DIR="$PKGS_DIR/gslib" # pre-cloned by script 01 +NPROCS="${SLURM_NTASKS:-8}" + +# Output helpers +info() { echo "[INFO] $*"; } +warn() { echo "[WARN] $*"; } +error() { + echo "[ERROR] $*" + exit 1 +} +step() { + echo "" + echo "=== $* ===" + echo "==================================================" +} +ok() { echo "[OK] $*"; } + +check() { + local desc="$1" test_cmd="$2" + if eval "$test_cmd" >/dev/null 2>&1; then + ok "$desc" + else + echo "[MISSING] $desc" + ERRORS=$((ERRORS + 1)) + fi +} + +# SECTION 0 - Pre-build sanity checks +# Confirm is is running on a compute node and all inputs from script 01 are pointed correctly +step "Pre-flight checks" + +info "Running on: $(hostname)" + +# Refuse to run on a login node +if echo "$(hostname)" | grep -qi "login"; then + error "Must run on a COMPUTE node, not a login node. + Request one: srun --pty --time=08:00:00 --ntasks=8 --mem=32G bash" +fi + +# Compute nodes have no internet - warn only if internet unexpectedly present +if curl -s --max-time 3 https://github.com >/dev/null 2>&1; then + warn "This node has internet access (unusual for Cumulus compute nodes)." +fi + +ERRORS=0 + +# MOOSE repo +check "MOOSE directory ($MOOSE_DIR)" "[ -d '$MOOSE_DIR' ]" +check "MOOSE configure script" "[ -f '$MOOSE_DIR/configure' ]" +check "update_and_rebuild_mfem.sh" "[ -f '$MOOSE_DIR/scripts/update_and_rebuild_mfem.sh' ]" + +# PETSc submodule +check "PETSc submodule ($PETSC_DIR)" "[ -d '$PETSC_DIR' ]" +check "PETSc configure script" "[ -f '$PETSC_DIR/configure' ]" + +# Key tarballs in PKGS_DIR - check by glob pattern +check "HDF5 tarball" "ls '$PKGS_DIR'/hdf5*.tar.gz 2>/dev/null | head -1 | grep -q ." +check "MUMPS tarball" "ls '$PKGS_DIR'/MUMPS*.tar.gz 2>/dev/null | head -1 | grep -q ." +check "ScaLAPACK tarball" "ls '$PKGS_DIR'/scalapack*.tar.gz 2>/dev/null | head -1 | grep -q ." +check "SuperLU_dist tarball" "ls '$PKGS_DIR'/superlu*.tar.gz 2>/dev/null | head -1 | grep -q ." +check "STRUMPACK tarball" "ls '$PKGS_DIR'/v8.0.0.tar.gz '$PKGS_DIR'/STRUMPACK*.tar.gz 2>/dev/null | head -1 | grep -q ." +# HYPRE tarball: PETSc names it after the commit hash (40 hex chars), not *hypre* +# Accept either a friendly name or the hash-named file (starts with hex digits) +check_hypre() { + find "$PKGS_DIR" -maxdepth 1 -type f \( \ + -name "*hypre*" -o -name "*HYPRE*" \ + \) 2>/dev/null | head -1 | grep -q . && return 0 + # Hash-named: 40 lowercase hex chars + .tar.gz + find "$PKGS_DIR" -maxdepth 1 -type f -name "*.tar.gz" 2>/dev/null | + grep -qE "/[0-9a-f]{40}\.tar\.gz$" +} +check "HYPRE tarball" "check_hypre" + +# Submodules that needed manual checkout in script 01 +check "libMesh submodule" "[ -f '$MOOSE_DIR/libmesh/configure.ac' ]" +# Conduit: check for CMakeLists.txt OR src/ directory (both indicate a valid checkout) +check "Conduit checked out" "[ -f '$MOOSE_DIR/framework/contrib/conduit/CMakeLists.txt' ] || [ -d '$MOOSE_DIR/framework/contrib/conduit/src' ]" +check "MFEM checked out" "[ -f '$MOOSE_DIR/framework/contrib/mfem/CMakeLists.txt' ]" +check "GSLIB pre-cloned" "[ -d '$GSLIB_DIR/.git' ]" + +[ "$ERRORS" -eq 0 ] || error "$ERRORS pre-flight check(s) failed. + Run 01_fetch_submodules_loginnode.sh on the login node first." + +info "All pre-flight checks passed." + +# ============================================================================= +# SECTION 1 - Load modules +# Same toolchain used to pre-fetch in script 01 +# ============================================================================= +step "Section 1: Loading modules" + +module purge +module load GCC/13.3.0 +module load OpenMPI/5.0.3-GCC-13.3.0 +module load CMake/3.29.3-GCCcore-13.3.0 +module load flex/2.6.4-GCCcore-13.3.0 +module load libtirpc/1.3.5-GCCcore-13.3.0 +module load git/2.45.1-GCCcore-13.3.0 + +export CC=mpicc +export CXX=mpicxx +export FC=mpif90 +export F90=mpif90 +export F77=mpif77 +export PETSC_DIR="$PETSC_DIR" +export PETSC_ARCH="$PETSC_ARCH" + +# MOOSE_JOBS controls parallelism inside the rebuild scripts +export MOOSE_JOBS="$NPROCS" +export METHODS=opt +export METHOD=opt + +# Verify all required tools are reachable after module load +for cmd in mpicc mpicxx mpif90 cmake flex git python3; do + command -v "$cmd" >/dev/null 2>&1 || error "$cmd not found after module load." +done + +info "Modules loaded:" +module list 2>&1 | tail -n +2 || true +info "Compiler : $(mpicc --version | head -1)" +info "CMake : $(cmake --version | head -1)" + +# SECTION 2 - PETSc +# Locate HDF5 tarball automatically by glob so the filename does not need +# to be hardcoded here. +# ptscotch, kokkos, kokkos-kernels and umpire are disabled - see build log. +step "Section 2: Building PETSc" + +HDF5_TARBALL=$(ls "$PKGS_DIR"/hdf5*.tar.gz 2>/dev/null | head -1 || true) +[ -n "$HDF5_TARBALL" ] || error "HDF5 tarball not found in $PKGS_DIR" +info "HDF5 tarball: $HDF5_TARBALL" + +cd "$MOOSE_DIR" +./scripts/update_and_rebuild_petsc.sh \ + --skip-submodule-update \ + --with-packages-download-dir="$PKGS_DIR" \ + --download-ptscotch=0 \ + --download-kokkos=0 \ + --download-kokkos-kernels=0 \ + --download-umpire=0 \ + --download-hdf5="$HDF5_TARBALL" + +ok "PETSc" + +# SECTION 3 - libMesh +step "Section 3: Building libMesh" + +cd "$MOOSE_DIR" +./scripts/update_and_rebuild_libmesh.sh + +ok "libMesh" + +# SECTION 4 - WASP +step "Section 4: Building WASP" + +cd "$MOOSE_DIR" +./scripts/update_and_rebuild_wasp.sh + +ok "WASP" + +# SECTION 5 - Conduit +step "Section 5: Building Conduit" + +cd "$MOOSE_DIR" +./scripts/update_and_rebuild_conduit.sh + +ok "Conduit" + +# SECTION 6 - MFEM with GSLIB via update_and_rebuild_mfem.sh 9 (tricky part) +# +# GSLIB was pre-cloned by script 01 into $PKGS_DIR/gslib to avoid the +# compute-node git clone failure that happens when MFEM_FETCH_GSLIB=YES +# tries to reach github.com during make. +# +# The script reads GSLIB_SOURCE_DIR to find an existing local clone instead +# of fetching from the internet. +# +# PETSc_CONFIG_CURRENT=YES is exported to skip the multipass PETSc link-test +# that hangs for hours on Lustre filesystems (see build log for details). + +step "Section 6: Building MFEM (with GSLIB)" + +# Verify PETSc actually built before proceeding +ARCH_DIR="$PETSC_DIR/$PETSC_ARCH" +[ -f "$ARCH_DIR/lib/libpetsc.so" ] \ + || error "PETSc library not found at $ARCH_DIR/lib/libpetsc.so - did Section 2 succeed?" + +# Point the MFEM build script at the pre-cloned GSLIB +export GSLIB_SOURCE_DIR="$GSLIB_DIR" + +# Skip the multipass PETSc link-test that hangs on Lustre (see build log) +export PETSc_CONFIG_CURRENT=YES + +cd "$MOOSE_DIR" +./scripts/update_and_rebuild_mfem.sh + +ok "MFEM" + +# SECTION 7 - MOOSE configure --with-mfem +step "Section 7: MOOSE configure --with-mfem" + +cd "$MOOSE_DIR" +./configure --with-mfem + +ok "MOOSE configure" + +# SECTION 8 - Build MOOSE modules/combined +step "Section 8: Building MOOSE (modules/combined)" + +cd "$MOOSE_DIR/modules/combined" +make METHOD=opt -j"$NPROCS" + +ok "MOOSE modules/combined" + +# SECTION 9 - Build tests +step "Section 9: Building MOOSE tests" + +cd "$MOOSE_DIR/test" +make METHOD=opt -j"$NPROCS" + +ok "MOOSE tests" + +# FINAL SUMMARY +echo "" +echo "============================================================" +echo " MOOSE/MFEM build complete on $(hostname)" +echo " $(date)" +echo "============================================================" +echo "" +echo " Key install paths:" +echo " PETSc : $ARCH_DIR" +echo " libMesh : $MOOSE_DIR/libmesh/installed" +echo " MFEM : $MOOSE_DIR/framework/contrib/mfem/installed" +echo " MOOSE : $MOOSE_DIR/modules/combined/combined-opt" +echo "" +echo " To run the MFEM tests:" +echo " cd $MOOSE_DIR/test" +echo " ./run_tests --opt -j$NPROCS --re mfem" +echo "" +echo " To use MOOSE in a job script, load:" +echo " module load GCC/13.3.0 OpenMPI/5.0.3-GCC-13.3.0" +echo " export PETSC_DIR=$PETSC_DIR" +echo " export PETSC_ARCH=$PETSC_ARCH" +echo "============================================================" \ No newline at end of file