CI infrastructure improvements: heartbeat, dump checks, timeouts, logging (#15950)

* Change Heartbeat default interval from 5s to 60s

The 5-second interval generates excessive log output during CI test runs.
A 60-second default reduces noise while still providing periodic status
updates for diagnosing runner hangs and disk space issues.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* Pass 60s heartbeat interval explicitly in CI workflow

Linux test steps were relying on the default interval. Now that the
default changed to 60s this is technically a no-op, but passing it
explicitly makes the intent clear and matches the Windows steps which
already specified 60s.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* Move dump file check from final results to individual test jobs

The dump file check was in specialized-test-runner.yml and checked
artifacts/all-logs after downloading from sub-jobs. Move it to
run-tests.yml where it checks testresults/ directly in the same job,
matching the actual --crashdump/--hangdump output directory.

This catches crashes/timeouts earlier and in the correct job context.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* Include timeout values in specialized test runsheets

Add testSessionTimeout and testHangTimeout MSBuild properties to the
JSON emitted by SpecializedTestRunsheetBuilderBase.targets, allowing
per-project timeout overrides to flow through the runsheet to the
CI workflow.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* Clarify PR filtering log message for specialized test workflows

The old message 'filtering to single test project' was ambiguous.
Make it clear this is a sanity-check behavior for pull_request events.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* Clamp heartbeat interval to >= 1s to prevent tight loops

Addresses review feedback: reject zero or negative interval values
that would cause a tight loop or crash in Task.Delay.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* Narrow dump file check to hang dumps only

Crash dumps (e.g. *_crash.dmp) can be produced during process cleanup
even when all tests passed — this is benign. Only fail on hang dump
files (*hangdump*) which indicate the test runner timed out and tests
may not have completed.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

---------

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: radical

.github/workflows/run-tests.yml

@@ -333,8 +333,8 @@ jobs:
           GITHUB_PR_HEAD_SHA: ${{ inputs.requiresCliArchive && github.event.pull_request.head.sha || '' }}
           GH_TOKEN: ${{ inputs.requiresCliArchive && github.token || '' }}
         run: |
-          # Start heartbeat monitor in background
-          ${{ github.workspace }}/${{ env.DOTNET_SCRIPT }} ${{ github.workspace }}/tools/scripts/Heartbeat.cs &
+          # Start heartbeat monitor in background (60s interval to reduce log noise)
+          ${{ github.workspace }}/${{ env.DOTNET_SCRIPT }} ${{ github.workspace }}/tools/scripts/Heartbeat.cs 60 &
           HEARTBEAT_PID=$!
 
           # Run tests
@@ -425,8 +425,8 @@ jobs:
           GITHUB_PR_HEAD_SHA: ${{ inputs.requiresCliArchive && github.event.pull_request.head.sha || '' }}
           GH_TOKEN: ${{ inputs.requiresCliArchive && github.token || '' }}
         run: |
-          # Start heartbeat monitor in background
-          ${{ env.DOTNET_SCRIPT }} ${{ github.workspace }}/tools/scripts/Heartbeat.cs &
+          # Start heartbeat monitor in background (60s interval to reduce log noise)
+          ${{ env.DOTNET_SCRIPT }} ${{ github.workspace }}/tools/scripts/Heartbeat.cs 60 &
           HEARTBEAT_PID=$!
 
           # Run tests
@@ -561,6 +561,28 @@ jobs:
 
           Write-Host "✓ Test execution completed successfully (found $validFileCount valid .trx file(s))"
 
+      - name: Check for hang dump files
+        if: always()
+        shell: pwsh
+        run: |
+          $testResultsDir = "${{ github.workspace }}/testresults"
+          if (-not (Test-Path $testResultsDir)) {
+            Write-Host "No test results directory found, skipping dump file check"
+            return
+          }
+
+          # Only check for hang dump files — these indicate a test timed out.
+          # Crash dumps (e.g. *_crash.dmp) can be produced during process cleanup
+          # even when all tests passed, so we don't fail on those.
+          $hangDumpFiles = Get-ChildItem -Path $testResultsDir -Filter *hangdump* -Recurse -ErrorAction SilentlyContinue
+          if ($hangDumpFiles.Count -gt 0) {
+            Write-Host "::error::Hang dump file(s) detected — the test runner timed out:"
+            foreach ($df in $hangDumpFiles) { Write-Host "  - $($df.FullName)" }
+            exit 1
+          }
+
+          Write-Host "✓ No hang dump files detected"
+
       - name: Dump docker info
         if: ${{ always() && runner.os == 'Linux' }}
         run: |

.github/workflows/specialized-test-runner.yml

@@ -201,23 +201,6 @@ jobs:
           ${{ github.workspace }}/testresults
           --combined
 
-      - name: Fail if test runner crashed or timed out
-        if: steps.check_tests.outputs.tests_run == 'true'
-        shell: pwsh
-        run: |
-          $logDirectory = "${{ github.workspace }}/artifacts/all-logs"
-
-          # Check for dump files (.dmp) in any of the collected test artifacts.
-          # A .dmp file indicates a crash (--crashdump) or timeout (--hangdump).
-          $dumpFiles = Get-ChildItem -Path $logDirectory -Filter *.dmp -Recurse -ErrorAction SilentlyContinue
-          if ($dumpFiles.Count -gt 0) {
-            Write-Host "::error::Dump file(s) detected — the test runner crashed or timed out in one or more jobs:"
-            foreach ($df in $dumpFiles) { Write-Host "  - $($df.FullName)" }
-            exit 1
-          }
-
-          Write-Host "✓ No dump files detected"
-
       - name: Fail if any dependency failed
         # 'skipped' can be when a transitive dependency fails and the dependent job gets 'skipped'.
         # For example, one of setup_* jobs failing and the Integration test jobs getting 'skipped'

eng/AfterSolutionBuild.targets

@@ -74,7 +74,7 @@
         $isSpecializedWorkflow = ($testRunnerName -eq 'QuarantinedTestRunsheetBuilder' -or $testRunnerName -eq 'OuterloopTestRunsheetBuilder')
 
         if ($isPullRequest -and $isSpecializedWorkflow -and $combined.Count -gt 0) {
-            Write-Host "Running on pull_request event for specialized tests - filtering to single test project"
+            Write-Host "Running on pull_request event for specialized tests - picking a single test project as a sanity check"
             # Get the first unique project name and keep all OS entries for that project
             $firstProjectName = $combined[0].project
             $combined = @($combined | Where-Object { $_.project -eq $firstProjectName })

eng/SpecializedTestRunsheetBuilderBase.targets

@@ -155,9 +155,9 @@
       <!-- Replace \ with /, and then escape " with \", so we have a compliant JSON -->
       <_TestCommand>$([System.String]::Copy($(_TestCommand)).Replace("\", "/").Replace('&quot;', '\&quot;'))</_TestCommand>
 
-      <_TestRunsheetWindows>{ "label": "w: $(_TestRunsheet)", "project": "$(_TestRunsheet)", "os": "$(_GithubActionsRunnerWindows)", "command": "./eng/build.ps1 $(_TestCommand)", "requiresNugets": $(_RequiresNugets), "requiresTestSdk": $(_RequiresTestSdk), "requiresCliArchive": $(_RequiresCliArchive) }</_TestRunsheetWindows>
-      <_TestRunsheetLinux>{ "label": "l: $(_TestRunsheet)", "project": "$(_TestRunsheet)", "os": "$(_GithubActionsRunnerLinux)", "command": "./eng/build.sh $(_TestCommand)", "requiresNugets": $(_RequiresNugets), "requiresTestSdk": $(_RequiresTestSdk), "requiresCliArchive": $(_RequiresCliArchive) }</_TestRunsheetLinux>
-      <_TestRunsheetMacOS>{ "label": "m: $(_TestRunsheet)", "project": "$(_TestRunsheet)", "os": "$(_GithubActionsRunnerMacOS)", "command": "./eng/build.sh $(_TestCommand)", "requiresNugets": $(_RequiresNugets), "requiresTestSdk": $(_RequiresTestSdk), "requiresCliArchive": $(_RequiresCliArchive) }</_TestRunsheetMacOS>
+      <_TestRunsheetWindows>{ "label": "w: $(_TestRunsheet)", "project": "$(_TestRunsheet)", "os": "$(_GithubActionsRunnerWindows)", "command": "./eng/build.ps1 $(_TestCommand)", "requiresNugets": $(_RequiresNugets), "requiresTestSdk": $(_RequiresTestSdk), "requiresCliArchive": $(_RequiresCliArchive), "testSessionTimeout": "$(TestSessionTimeout)", "testHangTimeout": "$(TestHangTimeout)" }</_TestRunsheetWindows>
+      <_TestRunsheetLinux>{ "label": "l: $(_TestRunsheet)", "project": "$(_TestRunsheet)", "os": "$(_GithubActionsRunnerLinux)", "command": "./eng/build.sh $(_TestCommand)", "requiresNugets": $(_RequiresNugets), "requiresTestSdk": $(_RequiresTestSdk), "requiresCliArchive": $(_RequiresCliArchive), "testSessionTimeout": "$(TestSessionTimeout)", "testHangTimeout": "$(TestHangTimeout)" }</_TestRunsheetLinux>
+      <_TestRunsheetMacOS>{ "label": "m: $(_TestRunsheet)", "project": "$(_TestRunsheet)", "os": "$(_GithubActionsRunnerMacOS)", "command": "./eng/build.sh $(_TestCommand)", "requiresNugets": $(_RequiresNugets), "requiresTestSdk": $(_RequiresTestSdk), "requiresCliArchive": $(_RequiresCliArchive), "testSessionTimeout": "$(TestSessionTimeout)", "testHangTimeout": "$(TestHangTimeout)" }</_TestRunsheetMacOS>
     </PropertyGroup>
 
     <WriteLinesToFile

tools/scripts/Heartbeat.cs

@@ -3,7 +3,7 @@
 // diagnose runner hangs and disk space issues during tests.
 //
 // Usage: dotnet tools/scripts/Heartbeat.cs [interval-seconds]
-// Default interval: 5 seconds
+// Default interval: 60 seconds
 //
 // Example: dotnet tools/scripts/Heartbeat.cs 10
 
@@ -16,7 +16,12 @@
          RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "Windows" :
          throw new NotSupportedException("Unsupported OS platform");
 
-var intervalSeconds = args.Length > 0 && int.TryParse(args[0], out var parsed) ? parsed : 5;
+const int defaultIntervalSeconds = 60;
+var intervalSeconds = args.Length > 0 &&
+                      int.TryParse(args[0], out var parsed) &&
+                      parsed >= 1
+    ? parsed
+    : defaultIntervalSeconds;
 var cts = new CancellationTokenSource();
 
 Console.CancelKeyPress += (_, e) =>