Add spack build testing

.github/actions/report_nightly_test_failure/action.yml

@@ -0,0 +1,91 @@
+name: Report nightly test failure
+description: >
+  Create or update a GitHub issue when a nightly test fails. The calling
+  workflow/job must grant `permissions: issues: write` for this to work.
+
+inputs:
+  title:
+    description: Title of the failure issue
+    required: true
+
+  body:
+    description: Initial body of the failure issue
+    required: true
+
+  labels:
+    description: Comma-separated list of issue labels
+    required: true
+
+outputs:
+  issue-number:
+    description: Number of the created or updated issue
+    value: ${{ steps.report.outputs.issue-number }}
+
+runs:
+  using: composite
+  steps:
+    - name: Create or update issue
+      id: report
+      uses: actions/github-script@v7
+      env:
+        ISSUE_TITLE: ${{ inputs.title }}
+        ISSUE_BODY: ${{ inputs.body }}
+        ISSUE_LABELS: ${{ inputs.labels }}
+      with:
+        script: |
+          const title = process.env.ISSUE_TITLE;
+          const body = process.env.ISSUE_BODY;
+          const labels = process.env.ISSUE_LABELS.split(",").map((label) => label.trim());
+
+          const runUrl =
+            `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
+
+          const { data: existingIssues } = await github.rest.issues.listForRepo({
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            state: "open",
+            labels: labels.join(","),
+            per_page: 100,
+          });
+
+          const existing = existingIssues.find((issue) => issue.title === title);
+
+          const timestamp = new Date().toISOString();
+
+          const note = [
+            `- \`${timestamp}\`: failed run [#${context.runNumber}](${runUrl}) `,
+            `(commit ${context.sha.substring(0, 8)}, ref \`${context.ref}\`)`,
+          ].join("");
+
+          if (existing) {
+            await github.rest.issues.createComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: existing.number,
+              body: note,
+            });
+
+            core.info(`Added a comment to existing issue #${existing.number}`);
+            core.setOutput("issue-number", existing.number);
+          } else {
+            const issueBody = [
+              body,
+              "",
+              note,
+              "",
+              "This issue is filed/updated automatically. Please close it once the underlying problem is fixed; " +
+                "a new one will be filed automatically if it fails again.",
+            ].join("\n");
+
+            const created = await github.rest.issues.create({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              title,
+              body: issueBody,
+              labels,
+            });
+
+            core.info(`Filed new issue #${created.data.number}`);
+            core.setOutput("issue-number", created.data.number);
+          }
+

.github/workflows/nightly_tests.yml

@@ -85,6 +85,9 @@ jobs:
     needs: gcc13_assertions_test
     runs-on: ubuntu-latest
     if: success() || failure()
+    permissions:
+      contents: read
+      issues: write
     steps:
       - uses: actions/checkout@v7
         with:
@@ -93,6 +96,17 @@ jobs:
         with:
           junit-report-base-name: gcc13_assertions_test_report
           retention-days: 2
+      - name: Report failure
+        if: always() && needs.gcc13_assertions_test.result != 'success'
+        uses: ./.github/actions/report_nightly_test_failure
+        with:
+          title: "Nightly test failure: gcc13_assertions"
+          body: >
+            The `gcc13_assertions` nightly job failed. See the
+            [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id
+            }})
+            for details.
+          labels: "github_actions,type: bug report"
 
   clang18_build:
     runs-on: ubuntu-latest
@@ -242,6 +256,9 @@ jobs:
     needs: clang18_test
     runs-on: ubuntu-latest
     if: success() || failure()
+    permissions:
+      contents: read
+      issues: write
     steps:
       - uses: actions/checkout@v7
         with:
@@ -250,6 +267,17 @@ jobs:
         with:
           junit-report-base-name: clang18_test_report
           retention-days: 2
+      - name: Report failure
+        if: always() && needs.clang18_test.result != 'success'
+        uses: ./.github/actions/report_nightly_test_failure
+        with:
+          title: "Nightly test failure: clang18"
+          body: >
+            The `clang18` nightly job failed. See the
+            [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id
+            }})
+            for details.
+          labels: "github_actions,type: bug report"
 
   clang18_build_oldest_supported_dependencies:
     runs-on: ubuntu-latest
@@ -403,6 +431,9 @@ jobs:
     needs: gcc13_no_optional_dependencies_test
     runs-on: ubuntu-latest
     if: success() || failure()
+    permissions:
+      contents: read
+      issues: write
     steps:
       - uses: actions/checkout@v7
         with:
@@ -411,6 +442,17 @@ jobs:
         with:
           junit-report-base-name: gcc13_no_optional_dependencies_test_report
           retention-days: 2
+      - name: Report failure
+        if: always() && needs.gcc13_no_optional_dependencies_test.result != 'success'
+        uses: ./.github/actions/report_nightly_test_failure
+        with:
+          title: "Nightly test failure: gcc13_no_optional_dependencies"
+          body: >
+            The `gcc13_no_optional_dependencies` nightly job failed. See the
+            [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id
+            }})
+            for details.
+          labels: "github_actions,type: bug report"
 
   gcc13_asan_build:
     runs-on: ubuntu-latest
@@ -490,6 +532,9 @@ jobs:
     needs: gcc13_asan_test
     runs-on: ubuntu-latest
     if: success() || failure()
+    permissions:
+      contents: read
+      issues: write
     steps:
       - uses: actions/checkout@v7
         with:
@@ -498,6 +543,17 @@ jobs:
         with:
           junit-report-base-name: gcc13_asan_test_report
           retention-days: 2
+      - name: Report failure
+        if: always() && needs.gcc13_asan_test.result != 'success'
+        uses: ./.github/actions/report_nightly_test_failure
+        with:
+          title: "Nightly test failure: gcc13_asan"
+          body: >
+            The `gcc13_asan` nightly job failed. See the
+            [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id
+            }})
+            for details.
+          labels: "github_actions,type: bug report"
 
   clang18_performance_tests_build:
     runs-on: ubuntu-latest

.github/workflows/spack.yml

@@ -0,0 +1,192 @@
+name: Spack build test
+
+# Builds the latest official 4C release with Spack
+# (https://packages.spack.io/package.html?name=4c-multiphysics) and runs a
+# smoke test against it
+on:
+  schedule:
+    # Weekly, avoiding the top of the hour as recommended by GitHub
+    - cron: "17 5 * * 0"
+      timezone: "Europe/Berlin"
+  workflow_dispatch:
+  # Also run for PRs that change this workflow.
+  pull_request:
+    paths:
+      - ".github/workflows/spack.yml"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  spack-build:
+    name: Build and test 4C (${{ matrix.os.name }}, ${{ matrix.variant.name }})
+    runs-on: ${{ matrix.os.name }}
+    # deal.II and the "full" variant build many extra dependencies from source
+    timeout-minutes: 240
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          # gcc@14 hits an internal compiler error on 4C's tensor templates;
+          # pin the compiler per OS to one that is known to work.
+          - name: ubuntu-24.04
+            compiler: gcc@13
+            compiler_bin: /usr/bin/g++-13
+          - name: ubuntu-26.04
+            compiler: gcc@15
+            compiler_bin: /usr/bin/g++-15
+        # add macOS support as soon as 4C (and an updated spack repo entry) support it
+        variant:
+          # Optional Spack variants (see `spack info 4c-multiphysics`).
+          - name: minimal
+            spec_args: ""
+          - name: full
+            spec_args: "+vtk +gmsh +mirco +arborx +fftw +backtrace"
+          - name: dealii
+            spec_args: "+dealii"
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@v2
+        with:
+          egress-policy: audit
+
+      - name: Set up Spack
+        uses: spack/setup-spack@v3
+        with:
+          # Pinned release for reproducibility. Bump periodically
+          spack_ref: v1.2.2
+          # `4c-multiphysics` is not part of any tagged spack-packages release
+          # yet, so pin to a `develop` commit that contains it instead.
+          # Pinned to the commit that added the 4C package (eaed8e6): a later
+          # commit (6c4f647) removed trilinos@16.2.1, which the package
+          # requires but was not yet updated so far, breaking concretization.
+          # TODO: switch `packages_ref` to a release tag (e.g. v2026.09.0) once
+          # one ships a `4c-multiphysics` package that no longer needs this.
+          packages_ref: eaed8e6d6c80bf7fcaeea0de4b00e19256bb846f
+          buildcache: true
+          color: true
+
+      - name: Find compilers
+        run: |
+          # Register only this one compiler: a broad `spack compiler find`
+          # also picks up system clang/LLVM and registers it as an external
+          # `libllvm`, which then breaks mesa (missing llvm-config).
+          spack compiler find ${{ matrix.os.compiler_bin }}
+
+      - name: Determine the latest 4C release known to Spack
+        id: version
+        run: |
+          # `spack versions --safe` prints a column-aligned, indented table
+          # (via colify), so tokenize on whitespace before filtering.
+          # 4C uses calendar versioning (YYYY.MINOR.PATCH), which also
+          # excludes non-release tokens such as "main".
+          LATEST_VERSION=$(spack --color=never versions --safe 4c-multiphysics \
+            | tr -s '[:space:]' '\n' \
+            | grep -E '^[0-9]{4}\.[0-9]+\.[0-9]+$' \
+            | sort --version-sort --reverse | head -n1)
+          if [ -z "$LATEST_VERSION" ]; then
+            echo "Could not determine the latest 4C release from Spack." >&2
+            exit 1
+          fi
+          echo "Latest 4C release known to Spack: $LATEST_VERSION"
+          echo "version=$LATEST_VERSION" >> "$GITHUB_OUTPUT"
+          echo "### Spack build test" >> "$GITHUB_STEP_SUMMARY"
+          echo "- OS: \`${{ matrix.os.name }}\`" >> "$GITHUB_STEP_SUMMARY"
+          echo "- Variant: \`${{ matrix.variant.name }}\` (\`${{ matrix.variant.spec_args }}\`)" >> "$GITHUB_STEP_SUMMARY"
+          echo "- 4C version: \`$LATEST_VERSION\`" >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Show the concretized spec
+        run: spack spec -l "4c-multiphysics@${{ steps.version.outputs.version }} ${{ matrix.variant.spec_args
+          }} %${{ matrix.os.compiler }}"
+
+      - name: Install 4C
+        run: |
+          spack install --fail-fast --verbose "4c-multiphysics@${{ steps.version.outputs.version }} ${{ matrix.variant.spec_args }} %${{ matrix.os.compiler }}"
+          spack find --long --variants 4c-multiphysics
+
+      - name: Checkout tutorial input files matching the installed release
+        uses: actions/checkout@v7
+        with:
+          ref: v${{ steps.version.outputs.version }}
+          sparse-checkout: |
+            tests/tutorials/poisson
+          sparse-checkout-cone-mode: false
+          path: 4C-src
+
+      - name: Smoke test - run 4C on a tutorial input file
+        shell: spack-bash {0}
+        run: |
+          spack load 4c-multiphysics
+
+          mkdir -p "$GITHUB_WORKSPACE/spack_smoke_test"
+          cd "$GITHUB_WORKSPACE/spack_smoke_test"
+          # 4C prints its version banner at the start of every run.
+          4C "$GITHUB_WORKSPACE/4C-src/tests/tutorials/poisson/tutorial_poisson_thermo.4C.yaml" spack_smoke_test_output
+
+          # Verify output was actually produced, not just a zero exit code.
+          if ! ls spack_smoke_test_output*.vtu >/dev/null 2>&1 && \
+             ! ls spack_smoke_test_output*.pvd >/dev/null 2>&1; then
+            echo "No output files were produced by the smoke test run." >&2
+            exit 1
+          fi
+
+      - name: Collect Spack build logs on failure
+        if: failure()
+        run: |
+          mkdir -p "$GITHUB_WORKSPACE/spack_build_logs"
+          # Build logs live under Spack's stage root, not the install prefix
+          # (which may not exist on failure). `cp --parents` keeps each
+          # package's stage subdirectory to avoid name clashes.
+          STAGE_DIR="$(spack location --stages 2>/dev/null || true)"
+          if [ -n "$STAGE_DIR" ] && [ -d "$STAGE_DIR" ]; then
+            (cd "$STAGE_DIR" && find . -name "*.txt" -exec cp --parents {} "$GITHUB_WORKSPACE/spack_build_logs/" \;)
+          else
+            echo "Spack stage directory not found; no build logs to collect." >&2
+          fi
+          spack debug report > "$GITHUB_WORKSPACE/spack_build_logs/spack_debug_report.txt" 2>&1 || true
+
+      - name: Upload smoke test output
+        if: success() || failure()
+        uses: actions/upload-artifact@v7
+        with:
+          name: spack_smoke_test_output-${{ matrix.os.name }}-${{ matrix.variant.name }}
+          path: |
+            ${{ github.workspace }}/spack_smoke_test
+            ${{ github.workspace }}/spack_build_logs
+          retention-days: 7
+
+  report-failure:
+    name: File an issue on failure
+    needs: spack-build
+    # Only file issues for unattended (scheduled) runs
+    if: failure() && github.event_name == 'schedule'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          sparse-checkout: .github
+      - name: Report failure
+        uses: ./.github/actions/report_nightly_test_failure
+        with:
+          title: "Scheduled Spack build test is failing"
+          body: >
+            The scheduled **Spack build test** workflow failed.
+
+            This workflow builds the latest official 4C release via the
+            [`4c-multiphysics` Spack package](https://packages.spack.io/package.html?name=4c-multiphysics)
+            across several OS/variant combinations and runs a smoke test against the result.
+
+            A failure here typically indicates one of:
+
+            - The `4c-multiphysics` Spack package (or one of its dependencies) needs to be updated for
+            the latest 4C release.
+
+            - A regression in 4C's build system that only shows up outside of the in-repo Docker-based
+            CI.
+
+            - A transient upstream Spack/buildcache issue.
+          labels: "github_actions,type: bug report"