diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 3a1ad851a..4cbaf0f49 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -72,9 +72,10 @@ workflows) before calling the matrix job. - `scripts/ci/validation-test-matrix.json` is the catalog: `platforms` (each with per-architecture target/artifact/1ES pool and the backends that platform - supports) and `triggers` (which OS/backend pairs each plan runs). The - `triggers` keys *are* the plan list — the resolver reads them at run time, so - adding a plan needs no script change. + supports), `triggers` (which OS/backend pairs each plan runs), and the + optional `backendDelayedStart` (per-backend job-start stagger, in seconds). + The `triggers` keys *are* the plan list — the resolver reads them at run time, + so adding a plan needs no script change. - `scripts/ci/resolve-validation-test-matrix.mjs` validates that catalog and expands a plan (currently `pr`, `nightly`, `weekly`, `enabled`) into GitHub Actions matrices. It rejects an invalid catalog before any specialized test @@ -95,7 +96,9 @@ the matrix `backend` id to the repository's existing backend suite. Ids that share a suite get their own case (`process-t1` and `process-t3` both run `WinProcessContainer-Tests.ps1`, which derives the tier it expects from the host's own `--probe`). A backend with no wired suite fails loudly rather than -reporting a false success. +reporting a false success. The Windows dispatcher points `TEMP` at +`$RUNNER_TEMP` before running a suite, so anything a test writes to the temp +directory is picked up by the job's log upload without per-file CI wiring. ### Individual components diff --git a/.github/workflows/Validation.Tests.Matrix.Job.yml b/.github/workflows/Validation.Tests.Matrix.Job.yml index 9a958d94a..817ebf880 100644 --- a/.github/workflows/Validation.Tests.Matrix.Job.yml +++ b/.github/workflows/Validation.Tests.Matrix.Job.yml @@ -38,7 +38,7 @@ jobs: fail-fast: false matrix: ${{ fromJSON(needs.resolve.outputs.windows) }} runs-on: [ self-hosted, "1ES.Pool=${{ matrix.pool }}", "JobId=mxc-e2e-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" ] - timeout-minutes: 60 + timeout-minutes: 180 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 @@ -48,6 +48,17 @@ jobs: name: ${{ matrix.artifact }} path: artifacts/bin + - name: Delayed start + if: matrix.startup_delay_seconds > 0 + shell: pwsh + run: | + # Spreads network-heavy setup across the pool's shared egress address + # instead of issuing every job's downloads at once. Configured by + # backendDelayedStart in scripts/ci/validation-test-matrix.json. + $seconds = ${{ matrix.startup_delay_seconds }} + Write-Host "Waiting $seconds second(s) before downloading prerequisites." + Start-Sleep -Seconds $seconds + - name: Prepare backend prerequisites timeout-minutes: 15 shell: pwsh @@ -59,8 +70,6 @@ jobs: if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - name: Run backend tests - # A backend that hangs (rather than failing) would otherwise burn the - # full job timeout. Fail fast enough to keep the log useful. timeout-minutes: 45 shell: pwsh run: | @@ -71,16 +80,25 @@ jobs: Tee-Object -FilePath (Join-Path $env:RUNNER_TEMP 'mxc-ci.log') -Append if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - - name: Upload failure logs - if: failure() || cancelled() + - name: Upload logs + if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a with: name: logs-${{ inputs.plan }}-${{ matrix.os }}-${{ matrix.architecture }}-${{ matrix.backend }}-${{ github.run_attempt }} + # The dispatcher points TEMP here, so suite logs and scratch trees are + # collected without enumerating each one. Exclude bulk binaries that + # tooling also drops in TEMP; they dwarf the logs and aid no triage. path: | - ${{ runner.temp }}/mxc-ci.log - ${{ runner.temp }}/mxc-wpc-tests/logs - ${{ runner.temp }}/WinProcessContainer-Tests.results.* - ${{ runner.temp }}/mxc_concurrent_oneshot + ${{ runner.temp }} + !${{ runner.temp }}/**/*.vhd + !${{ runner.temp }}/**/*.vhdx + !${{ runner.temp }}/**/*.tar + !${{ runner.temp }}/**/*.msi + !${{ runner.temp }}/**/*.msix + !${{ runner.temp }}/**/*.appx + !${{ runner.temp }}/**/*.zip + !${{ runner.temp }}/**/*.exe + !${{ runner.temp }}/**/*.dll if-no-files-found: ignore retention-days: 7 @@ -92,7 +110,7 @@ jobs: fail-fast: false matrix: ${{ fromJSON(needs.resolve.outputs.linux) }} runs-on: [ self-hosted, "1ES.Pool=${{ matrix.pool }}", "JobId=mxc-e2e-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" ] - timeout-minutes: 60 + timeout-minutes: 180 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 @@ -102,6 +120,18 @@ jobs: name: ${{ matrix.artifact }} path: artifacts/bin + - name: Delayed start + if: matrix.startup_delay_seconds > 0 + shell: bash + run: | + # Spreads network-heavy setup across the pool's shared egress address + # instead of issuing every job's downloads at once. Configured by + # backendDelayedStart in scripts/ci/validation-test-matrix.json. + set -euo pipefail + seconds=${{ matrix.startup_delay_seconds }} + echo "Waiting $seconds second(s) before downloading prerequisites." + sleep "$seconds" + - name: Prepare backend prerequisites timeout-minutes: 15 shell: bash @@ -126,8 +156,8 @@ jobs: tee -a "$RUNNER_TEMP/mxc-ci.log" fi - - name: Upload failure logs - if: failure() || cancelled() + - name: Upload logs + if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a with: name: logs-${{ inputs.plan }}-${{ matrix.os }}-${{ matrix.architecture }}-${{ matrix.backend }}-${{ github.run_attempt }} @@ -143,7 +173,7 @@ jobs: fail-fast: false matrix: ${{ fromJSON(needs.resolve.outputs.macos) }} runs-on: ${{ matrix.runner }} - timeout-minutes: 60 + timeout-minutes: 180 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 @@ -153,6 +183,18 @@ jobs: name: ${{ matrix.artifact }} path: artifacts/bin + - name: Delayed start + if: matrix.startup_delay_seconds > 0 + shell: bash + run: | + # Spreads network-heavy setup across the pool's shared egress address + # instead of issuing every job's downloads at once. Configured by + # backendDelayedStart in scripts/ci/validation-test-matrix.json. + set -euo pipefail + seconds=${{ matrix.startup_delay_seconds }} + echo "Waiting $seconds second(s) before starting tests." + sleep "$seconds" + - name: Run backend tests timeout-minutes: 60 shell: bash @@ -163,8 +205,8 @@ jobs: '${{ matrix.backend }}' "$GITHUB_WORKSPACE/artifacts/bin" 2>&1 | tee "$RUNNER_TEMP/mxc-ci.log" - - name: Upload failure logs - if: failure() || cancelled() + - name: Upload logs + if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a with: name: logs-${{ inputs.plan }}-${{ matrix.os }}-${{ matrix.architecture }}-${{ matrix.backend }}-${{ github.run_attempt }} diff --git a/docs/ci-validation-infrastructure.md b/docs/ci-validation-infrastructure.md index ec0e99486..821c5b67b 100644 --- a/docs/ci-validation-infrastructure.md +++ b/docs/ci-validation-infrastructure.md @@ -27,11 +27,11 @@ the individual local test scripts are documented in |------|------| | `.github/workflows/Validation.Tests.Scheduled.yml` | Scheduled entry point. Builds artifacts, then calls the matrix job. | | `.github/workflows/Validation.Tests.Matrix.Job.yml` | `workflow_call`-only. Resolves the plan and runs the per-family test jobs. | -| `scripts/ci/validation-test-matrix.json` | The matrix: OS versions, backends, triggers. | +| `scripts/ci/validation-test-matrix.json` | The matrix: OS versions, backends, triggers, job staggering. | | `scripts/ci/resolve-validation-test-matrix.mjs` | Matrix validator + plan expander. Emits the GitHub Actions matrices. | | `scripts/ci/prepare-windows-host.ps1` | Per-backend Windows host preparation / prerequisite assertions. | | `scripts/ci/prepare-linux-host.sh` | Per-backend Linux package install and service startup (distro-aware). | -| `tests/scripts/run_ci_backend_tests.ps1` | Windows dispatcher: backend id → existing backend suite. | +| `tests/scripts/run_ci_backend_tests.ps1` | Windows dispatcher: backend id → existing backend suite. Also points `TEMP` at `$RUNNER_TEMP` so logs get collected. | | `tests/scripts/run_ci_backend_tests.sh` | Linux/macOS dispatcher: backend id → existing backend suite. | ### Flow @@ -75,10 +75,9 @@ Build artifacts are kept for 1 day — they exist only to feed these jobs. | `macos` | GitHub-hosted `${{ matrix.runner }}` | Download artifact → `chmod +x` → `run_ci_backend_tests.sh `. No host-prep step. | Per-job display name: `, , ` (macOS omits -the architecture). Job timeout 60 min; host prep 15 min; the test step 45 min -(60 on macOS), so a hung backend fails while the log is still useful. On failure -or cancellation the job uploads `mxc-ci.log` plus the Process Container log -directories as `logs-----`, kept 7 days. +the architecture). Job timeout 180 min; host prep 15 min; the test step 45 min +(60 on macOS), so a hung backend fails while the log is still useful. Logs are +uploaded either way — see [Log collection](#log-collection). ## The catalog @@ -157,6 +156,39 @@ isolation-session, wslc; 25H2/24H2/23H2 × process-t3 + wslc) and 8 Linux (each of the four distros × bubblewrap + lxc). macOS resolves empty because Seatbelt has no wired suite. +### `backendDelayedStart` + +Optional. Staggers the start of jobs for a named backend instead of letting +them all begin at once: + +```json +"backendDelayedStart": [ + { "backend": "wslc", "seconds": 300 } +] +``` + +Every runner in a pool shares one egress address, so a backend whose setup +pulls down a large runtime or several container images concentrates all that +traffic into a burst the moment its jobs start together. Public registries +answer with rate limiting and stalled downloads. + +`seconds` is the gap between consecutive jobs of that backend, counted per +backend and following the resolved job order. With the entry above, four WSLC +jobs start at 0, 300, 600, and 900 seconds. + +The resolver puts the offset on every matrix entry as +`startup_delay_seconds` — `0` where no stagger applies — and the job sleeps +that long before its first network step. Job timeout is a flat 180 minutes, +with plenty of room for any wait you'd reasonably configure. + +Leave the section out (or empty) and every job starts as soon as its runner is +ready. A backend id that no plan schedules is accepted; it just never applies. + +Do keep in mind that the runner is held while it sleeps — Actions can't defer +allocating a matrix job, so the wait has to happen inside it. Use no more than +the contention calls for. This spreads simultaneous load and nothing else; a +single download that stalls on its own is unaffected. + ## Backend status Snapshot of what the matrix actually proves today. Update this table as backends @@ -167,8 +199,8 @@ get fixed or wired. | Process T1 | ✅ Good | Prerelease Windows only. Remaining failures are genuine MXC bugs or harness limitations. | | Process T3 | ✅ Good | Non-prerelease Windows builds only, until the testing suite is updated. | | Bubblewrap | ✅ Good | | -| LXC | ⚠️ Mostly good | Some networking tests fail on distros other than Ubuntu 24.04; host-vs-MXC cause not yet isolated. | -| WSLC | ⚠️ Mostly good | Can hit a download rate limit while updating WSL / pulling container images. Planned fix: split into several jobs spaced ~15 min apart. | +| LXC | ✅ Good | Some networking tests fail on distros other than Ubuntu 24.04; seems to be an issue with MXC. | +| WSLC | ✅ Good | Might have to retry hung jobs - this is an issue with overzealous agent reclaiming. | | IsolationSession | ⚠️ Blocked | `Feature_AgentSessionsBaseSupport` is not enabled on the pool image yet. | | Windows Sandbox | ⛔ Not scheduled | Dispatcher case is wired; no trigger entry yet. | | MicroVM | ⛔ Not working | Windows cold and warm starts hang; no Linux suite. The artifact payload is currently commented out in the build jobs. | @@ -204,13 +236,33 @@ message instead of surfacing later as an opaque backend error. - `bubblewrap` — installs `bwrap` (apt/dnf/yum/microdnf) and relaxes `kernel.apparmor_restrict_unprivileged_userns` (ephemeral CI hosts only). - `lxc` — installs the LXC stack, reloads the AppArmor profile, starts and waits - for `lxcbr0`, and prints network diagnostics. On RHEL-likes it first needs - EPEL, because Red Hat dropped LXC after RHEL 7 and ships no replacement. + for `lxcbr0`, enables bridge netfilter, and makes sure the bridge's NAT rule + is in place. On RHEL-likes it needs EPEL first, because Red Hat dropped LXC + after RHEL 7 and ships no replacement. - `microvm` — asserts the NanVix payload exists. - `hyperlight` — no-op. macOS has no preparation step. +## Log collection + +Every job uploads its logs whether it passed or failed, as +`logs-----`, kept 7 days. + +The catch is that `$env:TEMP` is not `$RUNNER_TEMP`. The Windows suites write their +scratch trees, transcripts, and results files under the user's temp directory +(`C:\Users\\AppData\Local\Temp`), but `upload-artifact` reads +`${{ runner.temp }}` (`C:\a\_work\_temp`). Anything left in the former is simply +never collected, which is why the artifact used to arrive nearly empty. + +So `run_ci_backend_tests.ps1` points `TEMP` and `TMP` at `$RUNNER_TEMP` before +it dispatches. Parameter defaults, `[System.IO.Path]::GetTempPath()`, and child +processes all read those variables, so everything temp-rooted lands in the +upload directory without CI having to know a single filename. + +Linux and macOS need none of this — those suites log to stdout, and the run +step tees that into `$RUNNER_TEMP/mxc-ci.log`. + ## Runbook Always finish with a local resolve, which runs the full catalog validation: @@ -282,6 +334,19 @@ Replace the explicit failure in the dispatcher with the suite invocation, add any host prerequisites, then add the OS/backend pair to a trigger. Always verify by testing it ahead of time. +### Stagger a backend's job starts + +Add or edit its `backendDelayedStart` entry in the catalog, then resolve +locally to confirm the offsets. Worth reaching for when a backend's setup is +network-heavy enough that concurrent jobs run into rate limits or stalled +downloads — and worth removing again once that pressure is gone. + +### Collect a new log file + +Have the suite write it under `$env:TEMP`. The dispatcher redirects that to the +upload directory, so nothing in CI needs to change. See +[Log collection](#log-collection). + ### Change the schedule Everything schedule-related lives in `Validation.Tests.Scheduled.yml`: the two diff --git a/scripts/ci/prepare-linux-host.sh b/scripts/ci/prepare-linux-host.sh index 16eafcfeb..f7afb2743 100644 --- a/scripts/ci/prepare-linux-host.sh +++ b/scripts/ci/prepare-linux-host.sh @@ -129,57 +129,56 @@ start_lxc_bridge() { ip addr show "$bridge" || true } -# Report the host-side state that container networking depends on. Purely -# diagnostic: never fails the job, so a networking problem still surfaces as -# the backend test failure rather than as a prerequisite error. -report_lxc_network_diagnostics() { +# Outbound container traffic leaves the bridge subnet with a private source +# address, so it needs a MASQUERADE rule to reach anything off-host. lxc-net +# normally installs one, but it skips its firewall setup when it believes +# another manager owns the ruleset, leaving a bridge that hands out leases the +# container cannot use. The symptom is a name-resolution failure inside the +# guest, which reads like a policy problem and is not one. +ensure_bridge_nat() { local bridge="${LXC_BRIDGE:-lxcbr0}" + local subnet - echo "--- LXC network diagnostics (host) ---" - - echo "net.ipv4.ip_forward: $(cat /proc/sys/net/ipv4/ip_forward 2>/dev/null || echo unknown)" - - echo "Bridge $bridge:" - ip -4 addr show "$bridge" 2>/dev/null | sed 's/^/ /' || echo " (absent)" - - echo "dnsmasq processes:" - pgrep -af dnsmasq 2>/dev/null | sed 's/^/ /' || echo " (none)" - - echo "NAT rules for the bridge subnet:" - sudo iptables -t nat -S POSTROUTING 2>/dev/null | grep -E '10\.0\.3|MASQUERADE' | - sed 's/^/ /' || echo " (none found)" - - echo "FORWARD policy and bridge rules:" - sudo iptables -S FORWARD 2>/dev/null | grep -E "policy|$bridge" | sed 's/^/ /' || - echo " (none found)" - - echo "Host /etc/resolv.conf nameservers:" - grep '^nameserver' /etc/resolv.conf 2>/dev/null | sed 's/^/ /' || echo " (none)" + subnet="$(ip -4 -o addr show "$bridge" 2>/dev/null | awk '{print $4}' | head -n 1)" + if [[ -z "$subnet" ]]; then + echo "WARNING: $bridge has no IPv4 subnet; skipping NAT setup." >&2 + return 0 + fi - echo "lxc-net configuration:" - grep -E '^(USE_LXC_BRIDGE|LXC_ADDR|LXC_NETMASK|LXC_DHCP_RANGE|LXC_DHCP_CONFILE)' \ - /etc/default/lxc-net 2>/dev/null | sed 's/^/ /' || echo " (no /etc/default/lxc-net)" + # Match on the source subnet rather than the rule text: lxc-net's own rule + # and ours are equivalent however they are spelled. + if sudo iptables -t nat -S POSTROUTING 2>/dev/null | + grep -q -- "-s ${subnet%%/*}"; then + echo "NAT for $subnet is already present." + return 0 + fi - # Prove the host itself can resolve the name the network test uses. If this - # fails, the container was never going to succeed. - if command -v getent >/dev/null 2>&1; then - echo "Host resolution of api.github.com:" - getent ahostsv4 api.github.com 2>/dev/null | head -n 2 | sed 's/^/ /' || - echo " FAILED - the host cannot resolve it either" + if sudo iptables -t nat -A POSTROUTING -s "$subnet" ! -d "$subnet" -j MASQUERADE; then + echo "Installed MASQUERADE for $subnet." + else + echo "WARNING: could not install MASQUERADE for $subnet; containers will not reach off-host destinations." >&2 fi +} - # Ask the bridge's own resolver, which is what a container is handed via - # DHCP. This isolates "dnsmasq is broken" from "the host is fine". - local bridge_ip - bridge_ip="$(ip -4 -o addr show "$bridge" 2>/dev/null | - awk '{print $4}' | cut -d/ -f1 | head -n 1)" - if [[ -n "$bridge_ip" ]] && command -v nslookup >/dev/null 2>&1; then - echo "Resolution via bridge resolver ($bridge_ip):" - nslookup api.github.com "$bridge_ip" 2>&1 | tail -n 4 | sed 's/^/ /' || - echo " FAILED - dnsmasq on $bridge is not answering" +# Container network policy is programmed as iptables rules reached from +# FORWARD, which only sees bridged traffic when br_netfilter is loaded and +# bridge-nf-call-iptables is enabled. Neither is guaranteed on a fresh image, +# and without them the backend refuses to report success for a policy it +# cannot enforce. +enable_bridge_netfilter() { + if ! sudo modprobe br_netfilter 2>/dev/null; then + echo "WARNING: could not load br_netfilter; bridged traffic may bypass iptables." >&2 fi - echo "--- end diagnostics ---" + local knob + for knob in bridge-nf-call-iptables bridge-nf-call-ip6tables; do + if [[ -e "/proc/sys/net/bridge/$knob" ]]; then + sudo sysctl -w "net.bridge.$knob=1" >/dev/null || + echo "WARNING: could not enable net.bridge.$knob." >&2 + else + echo "WARNING: /proc/sys/net/bridge/$knob is absent; container network policy cannot be enforced." >&2 + fi + done } chmod +x "$binary_directory/lxc-exec" @@ -203,7 +202,8 @@ case "$backend" in sudo apparmor_parser -rT /etc/apparmor.d/lxc* 2>/dev/null || true fi start_lxc_bridge - report_lxc_network_diagnostics + enable_bridge_netfilter + ensure_bridge_nat ;; microvm) for file in nanvixd.elf nanvix_rootfs.img python3.initrd bin/kernel.elf; do diff --git a/scripts/ci/resolve-validation-test-matrix.mjs b/scripts/ci/resolve-validation-test-matrix.mjs index 9c73382d2..565fcec1d 100644 --- a/scripts/ci/resolve-validation-test-matrix.mjs +++ b/scripts/ci/resolve-validation-test-matrix.mjs @@ -131,9 +131,41 @@ export function validateCatalog(catalog) { } } + validateBackendDelayedStart(catalog); + return { platforms, plans }; } +// backendDelayedStart is optional: an absent or empty section means every job +// starts as soon as its runner is ready. +function validateBackendDelayedStart(catalog) { + const delays = catalog.backendDelayedStart; + if (delays == null) { + return; + } + if (!Array.isArray(delays)) { + throw new Error('catalog backendDelayedStart must be an array'); + } + + const seen = new Set(); + for (const entry of delays) { + if (entry == null || typeof entry !== 'object' || Array.isArray(entry)) { + throw new Error('each backendDelayedStart entry must be an object'); + } + assertNonEmptyString(entry.backend, 'backendDelayedStart backend'); + if (seen.has(entry.backend)) { + throw new Error(`duplicate backendDelayedStart entry for ${entry.backend}`); + } + seen.add(entry.backend); + + if (!Number.isInteger(entry.seconds) || entry.seconds < 0) { + throw new Error( + `backendDelayedStart ${entry.backend} seconds must be a non-negative integer` + ); + } + } +} + export function expandPlan(catalog, plan) { const { platforms, plans } = validateCatalog(catalog); if (!plans.includes(plan)) { @@ -186,9 +218,51 @@ export function resolvePlan(catalog, plan) { suppressNonMacArm64(matrices); sortMatrices(matrices); + applyDelayedStart(matrices, catalog.backendDelayedStart); return matrices; } +// A backend named in backendDelayedStart has its jobs started at staggered +// offsets rather than all at once. This exists for backends whose setup +// downloads a large runtime or several container images: every runner in a +// pool shares one egress address, so simultaneous starts concentrate that +// traffic into a burst that draws rate limiting and stalled downloads. +// +// The entry's seconds value is the gap between consecutive jobs of that +// backend, counted independently per backend. Applied after sorting so the +// assignment follows the emitted order and stays reproducible. +function applyDelayedStart(matrices, delays) { + if (!Array.isArray(delays) || delays.length === 0) { + setDefaultDelays(matrices); + return; + } + + const stepSeconds = new Map(delays.map(entry => [entry.backend, entry.seconds])); + const scheduled = new Map(); + + for (const family of FAMILIES) { + for (const entry of matrices[family]) { + const step = stepSeconds.get(entry.backend); + if (step === undefined) { + entry.startup_delay_seconds = 0; + } else { + const position = scheduled.get(entry.backend) ?? 0; + entry.startup_delay_seconds = position * step; + scheduled.set(entry.backend, position + 1); + } + } + } +} + +// Every entry carries the field so the workflow's step condition is uniform. +function setDefaultDelays(matrices) { + for (const family of FAMILIES) { + for (const entry of matrices[family]) { + entry.startup_delay_seconds = 0; + } + } +} + // Windows and Linux ARM64 hosted VMs currently lack nested virtualization. // Keep their catalog entries intact for future enablement, but never emit them // until suitable test hosts are available. macOS remains ARM64-only. diff --git a/scripts/ci/validation-test-matrix.json b/scripts/ci/validation-test-matrix.json index bf7b2fc62..f0a2d46c3 100644 --- a/scripts/ci/validation-test-matrix.json +++ b/scripts/ci/validation-test-matrix.json @@ -325,6 +325,12 @@ } } ], + "backendDelayedStart": [ + { + "backend": "wslc", + "seconds": 45 + } + ], "triggers": { "pr": [], "nightly": [ diff --git a/tests/scripts/run_ci_backend_tests.ps1 b/tests/scripts/run_ci_backend_tests.ps1 index adcb916a9..3770f6390 100644 --- a/tests/scripts/run_ci_backend_tests.ps1 +++ b/tests/scripts/run_ci_backend_tests.ps1 @@ -28,7 +28,6 @@ param( [string]$Architecture ) -Set-StrictMode -Version Latest $ErrorActionPreference = 'Stop' $scriptRoot = Split-Path -Parent $MyInvocation.MyCommand.Path @@ -43,6 +42,27 @@ function Assert-File { } } +# The suites, and MXC itself, write logs and scratch trees under $env:TEMP, +# which is not the directory CI uploads from. Rather than enumerate every +# artifact and copy it afterwards, point TEMP at the upload directory for the +# duration of the run: parameter defaults, .NET GetTempPath(), and child +# processes all follow it, so anything temp-rooted lands where CI collects it. +# +# The suites' scratch-root guards compare against $env:TEMP, so they stay +# satisfied — this redirects what TEMP means rather than pointing a path +# outside it. +function Redirect-TempToRunnerTemp { + if (-not $env:RUNNER_TEMP) { + return + } + if (-not (Test-Path -LiteralPath $env:RUNNER_TEMP)) { + New-Item -ItemType Directory -Force -Path $env:RUNNER_TEMP | Out-Null + } + $env:TEMP = $env:RUNNER_TEMP + $env:TMP = $env:RUNNER_TEMP + Write-Host "Redirected TEMP to $env:RUNNER_TEMP so test logs are collected." +} + function Invoke-TestScript { param( [Parameter(Mandatory)][string]$Path, @@ -78,6 +98,8 @@ function Invoke-ProcessContainerTests { Copy-Item -LiteralPath $uiProbe -Destination (Join-Path $releaseDirectory 'wxc-ui-probe.exe') -Force $script = Join-Path $scriptRoot 'WinProcessContainer-Tests.ps1' + # -KeepArtifacts stops the harness deleting its scratch tree on a clean + # run, so a passing job still uploads its per-test logs and configs. # Skip build and Cargo phases because this job consumes a previously # built artifact; retain the host and containment behavior phases. $phases = @( @@ -97,12 +119,15 @@ function Invoke-ProcessContainerTests { -WxcRelease (Join-Path $releaseDirectory 'wxc-exec.exe') ` -UiProbeDebug (Join-Path $debugDirectory 'wxc-ui-probe.exe') ` -UiProbeRelease (Join-Path $releaseDirectory 'wxc-ui-probe.exe') ` + -KeepArtifacts ` -Phases $phases if ($LASTEXITCODE -ne 0) { throw "Process Container tests failed with exit code $LASTEXITCODE." } } +Redirect-TempToRunnerTemp + switch ($Backend) { 'process-t1' { Invoke-ProcessContainerTests diff --git a/tests/scripts/run_ci_backend_tests.sh b/tests/scripts/run_ci_backend_tests.sh index dd03f1245..4e156ba21 100644 --- a/tests/scripts/run_ci_backend_tests.sh +++ b/tests/scripts/run_ci_backend_tests.sh @@ -48,6 +48,9 @@ case "$backend" in mkdir -p "$release_directory" cp -a "$binary_directory/." "$release_directory/" chmod +x "$release_directory/lxc-exec" "$release_directory/unix-test-proxy" + # A skip here means a prerequisite disappeared on a runner provisioned + # to execute this suite, so turn it into a failure rather than a + # vacuously green gate. MXC_LXC_TESTS_REQUIRE_EXECUTION=1 bash "$script_root/run_lxc_all_tests.sh" ;; seatbelt) diff --git a/tests/scripts/run_wslc_all_tests.ps1 b/tests/scripts/run_wslc_all_tests.ps1 index d3c31b97e..4e4958f50 100644 --- a/tests/scripts/run_wslc_all_tests.ps1 +++ b/tests/scripts/run_wslc_all_tests.ps1 @@ -97,17 +97,6 @@ if (-not $SkipSetup) { } } -# Helper: StrictMode-safe property read; returns $null when the property (or the -# object) is absent. Lets the optional-config-field reads below work under the -# Set-StrictMode -Version Latest that run_ci_backend_tests.ps1 imposes. -function Get-JsonProperty { - param($Object, [Parameter(Mandatory)][string]$Name) - if ($null -eq $Object) { return $null } - $prop = $Object.PSObject.Properties[$Name] - if ($null -eq $prop) { return $null } - return $prop.Value -} - # Helper: run a single WSLC test config function Run-WslcTest { param( @@ -125,12 +114,9 @@ function Run-WslcTest { return @{ Name = $ConfigFile; Pass = $true; Skipped = $true; Reason = "File not found" } } - # Skip if the config references a tar file that doesn't exist locally. - # Read the chain defensively: this suite inherits Set-StrictMode -Version - # Latest from run_ci_backend_tests.ps1, under which touching a missing - # property is a terminating error, and most configs have no wslc.imageTarPath. + # Skip if the config references a tar file that doesn't exist locally $configJson = Get-Content $configPath -Raw | ConvertFrom-Json - $tarPath = Get-JsonProperty (Get-JsonProperty (Get-JsonProperty $configJson 'experimental') 'wslc') 'imageTarPath' + $tarPath = $configJson.experimental.wslc.imageTarPath if ($tarPath -and -not (Test-Path $tarPath)) { Write-Host " $ConfigFile ... " -NoNewline Write-Host "SKIP (tar not found: $tarPath)" -ForegroundColor Yellow @@ -183,7 +169,7 @@ function Run-WslcTest { # PostExitCheck runs after exit/output gates pass. Receives ($id, $output) # and must return truthy. Use for externally-observable state assertions. if ($pass -and $PostExitCheck) { - $containerId = Get-JsonProperty $configJson 'containerId' + $containerId = $configJson.containerId try { $checkResult = & $PostExitCheck $containerId $output if (-not $checkResult) {