From 09afd57351d2f3c5938c899d9f23ebce310b9b39 Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Mon, 17 Aug 2026 16:14:09 -0700 Subject: [PATCH 01/16] Bridge netfilter enabled --- .../Validation.Infrastructure.Tests.yml | 36 +++++++ scripts/ci/prepare-linux-host.sh | 24 +++++ scripts/ci/validation-test-matrix.json | 27 ++++- tests/scripts/run_ci_backend_tests.sh | 3 + tests/scripts/run_wslc_state_aware_tests.ps1 | 101 +++++++++++++----- 5 files changed, 162 insertions(+), 29 deletions(-) create mode 100644 .github/workflows/Validation.Infrastructure.Tests.yml diff --git a/.github/workflows/Validation.Infrastructure.Tests.yml b/.github/workflows/Validation.Infrastructure.Tests.yml new file mode 100644 index 000000000..1785d2515 --- /dev/null +++ b/.github/workflows/Validation.Infrastructure.Tests.yml @@ -0,0 +1,36 @@ +name: Validation Infrastructure Testing + +on: + push: + branches: + - user/emichlin/validation-infra-2 + +concurrency: + group: validation-infra-pr-tests-${{ github.ref }} + cancel-in-progress: true + +permissions: + actions: read + contents: read + +jobs: + dependency-feed-check: + uses: ./.github/workflows/Dependency.Feed.Check.Job.yml + + windows: + needs: dependency-feed-check + uses: ./.github/workflows/Build.Windows.Job.yml + + linux: + needs: dependency-feed-check + uses: ./.github/workflows/Build.Linux.Job.yml + + macos: + needs: dependency-feed-check + uses: ./.github/workflows/Build.MacOS.Job.yml + + test: + needs: [windows, linux, macos] + uses: ./.github/workflows/Validation.Tests.Matrix.Job.yml + with: + plan: enabled \ No newline at end of file diff --git a/scripts/ci/prepare-linux-host.sh b/scripts/ci/prepare-linux-host.sh index 16eafcfeb..062913762 100644 --- a/scripts/ci/prepare-linux-host.sh +++ b/scripts/ci/prepare-linux-host.sh @@ -129,6 +129,27 @@ start_lxc_bridge() { ip addr show "$bridge" || true } +# Container network policy is programmed as iptables rules reached from +# FORWARD, which only sees bridged traffic when br_netfilter is loaded and +# bridge-nf-call-iptables is enabled. Neither is guaranteed on a fresh image, +# and without them the backend refuses to report success for a policy it +# cannot enforce. +enable_bridge_netfilter() { + if ! sudo modprobe br_netfilter 2>/dev/null; then + echo "WARNING: could not load br_netfilter; bridged traffic may bypass iptables." >&2 + fi + + local knob + for knob in bridge-nf-call-iptables bridge-nf-call-ip6tables; do + if [[ -e "/proc/sys/net/bridge/$knob" ]]; then + sudo sysctl -w "net.bridge.$knob=1" >/dev/null || + echo "WARNING: could not enable net.bridge.$knob." >&2 + else + echo "WARNING: /proc/sys/net/bridge/$knob is absent; container network policy cannot be enforced." >&2 + fi + done +} + # Report the host-side state that container networking depends on. Purely # diagnostic: never fails the job, so a networking problem still surfaces as # the backend test failure rather than as a prerequisite error. @@ -139,6 +160,8 @@ report_lxc_network_diagnostics() { echo "net.ipv4.ip_forward: $(cat /proc/sys/net/ipv4/ip_forward 2>/dev/null || echo unknown)" + echo "bridge-nf-call-iptables: $(cat /proc/sys/net/bridge/bridge-nf-call-iptables 2>/dev/null || echo absent)" + echo "Bridge $bridge:" ip -4 addr show "$bridge" 2>/dev/null | sed 's/^/ /' || echo " (absent)" @@ -203,6 +226,7 @@ case "$backend" in sudo apparmor_parser -rT /etc/apparmor.d/lxc* 2>/dev/null || true fi start_lxc_bridge + enable_bridge_netfilter report_lxc_network_diagnostics ;; microvm) diff --git a/scripts/ci/validation-test-matrix.json b/scripts/ci/validation-test-matrix.json index bf7b2fc62..96256003a 100644 --- a/scripts/ci/validation-test-matrix.json +++ b/scripts/ci/validation-test-matrix.json @@ -387,6 +387,31 @@ } ], "weekly": [], - "enabled": [] + "enabled": [ + { + "os": "ubuntu-26.04", + "backends": [ + "lxc" + ] + }, + { + "os": "ubuntu-24.04", + "backends": [ + "lxc" + ] + }, + { + "os": "rhel-10", + "backends": [ + "lxc" + ] + }, + { + "os": "debian-13", + "backends": [ + "lxc" + ] + } + ] } } diff --git a/tests/scripts/run_ci_backend_tests.sh b/tests/scripts/run_ci_backend_tests.sh index dd03f1245..4e156ba21 100644 --- a/tests/scripts/run_ci_backend_tests.sh +++ b/tests/scripts/run_ci_backend_tests.sh @@ -48,6 +48,9 @@ case "$backend" in mkdir -p "$release_directory" cp -a "$binary_directory/." "$release_directory/" chmod +x "$release_directory/lxc-exec" "$release_directory/unix-test-proxy" + # A skip here means a prerequisite disappeared on a runner provisioned + # to execute this suite, so turn it into a failure rather than a + # vacuously green gate. MXC_LXC_TESTS_REQUIRE_EXECUTION=1 bash "$script_root/run_lxc_all_tests.sh" ;; seatbelt) diff --git a/tests/scripts/run_wslc_state_aware_tests.ps1 b/tests/scripts/run_wslc_state_aware_tests.ps1 index 2f2faa034..63967d9ac 100644 --- a/tests/scripts/run_wslc_state_aware_tests.ps1 +++ b/tests/scripts/run_wslc_state_aware_tests.ps1 @@ -283,6 +283,50 @@ function Envelope-Arm { '' } +# StrictMode-safe property read; returns $null when the object or property is +# absent. run_ci_backend_tests.ps1 imposes Set-StrictMode -Version Latest, under +# which touching a missing property on a PSCustomObject is a terminating error -- +# and an envelope carries exactly one of `result` / `error`, so the other arm is +# always missing. +function Get-JsonProperty { + param($Object, [Parameter(Mandatory)][string]$Name) + if ($null -eq $Object) { return $null } + $prop = $Object.PSObject.Properties[$Name] + if ($null -eq $prop) { return $null } + return $prop.Value +} + +# The envelope's `error` object, or $null when the envelope is absent/succeeded. +function Get-EnvelopeError { + param($Envelope) + Get-JsonProperty $Envelope 'error' +} + +# The envelope's error code, or '' / '' when absent. +function Get-EnvelopeErrorCode { + param($Envelope) + if ($null -eq $Envelope) { return '' } + $err = Get-JsonProperty $Envelope 'error' + if ($null -eq $err) { return '' } + $code = Get-JsonProperty $err 'code' + if ($null -eq $code) { return '' } + return [string]$code +} + +# A property of the envelope's `result` object, or $null when absent. +function Get-EnvelopeResultProperty { + param($Envelope, [Parameter(Mandatory)][string]$Name) + Get-JsonProperty (Get-JsonProperty $Envelope 'result') $Name +} + +# The envelope's `result.sandboxId` as a string, or $null when absent. +function Get-EnvelopeSandboxId { + param($Envelope) + $id = Get-EnvelopeResultProperty $Envelope 'sandboxId' + if ($null -eq $id) { return $null } + return [string]$id +} + # Is the daemon process currently running? function Test-DaemonRunning { $null -ne (Get-Process -Name $DaemonProcName -ErrorAction SilentlyContinue) @@ -345,7 +389,7 @@ function Provision-Sandbox { $r = Invoke-StateAware -ConfigFile $ConfigFile $envObj = Parse-Envelope -Stdout $r.Stdout if ((Envelope-Arm $envObj) -ne 'result') { return $null } - [string]$envObj.result.sandboxId + Get-EnvelopeSandboxId $envObj } # ---------------- Backend-availability probe ---------------- @@ -363,16 +407,17 @@ if ($script:PreExistingDaemon) { $probe = Invoke-StateAware -ConfigFile 'wslc_state_aware_provision.json' $probeEnv = Parse-Envelope -Stdout $probe.Stdout -if ($null -ne $probeEnv -and $probeEnv.error.code -eq 'backend_unavailable') { +$probeError = Get-EnvelopeError $probeEnv +if ((Get-EnvelopeErrorCode $probeEnv) -eq 'backend_unavailable') { Write-Host "SKIPPED: wxc-exec reports backend_unavailable (built without --features wslc, or no WSLC runtime)" -ForegroundColor Yellow exit 0 } -if ($null -ne $probeEnv -and $null -ne $probeEnv.result -and $null -ne $probeEnv.result.sandboxId) { - $probeSandboxId = [string]$probeEnv.result.sandboxId +$probeSandboxId = Get-EnvelopeSandboxId $probeEnv +if ($null -ne $probeSandboxId) { Write-Host "Backend probe: provisioned $probeSandboxId, deprovisioning ..." -ForegroundColor DarkGray $null = Invoke-StateAware -ConfigFile 'wslc_state_aware_deprovision.json' -SandboxId $probeSandboxId -} elseif ($null -ne $probeEnv -and $null -ne $probeEnv.error) { - Write-Host "WARN: probe provision errored (code=$($probeEnv.error.code)): $($probeEnv.error.message)" -ForegroundColor Yellow +} elseif ($null -ne $probeError) { + Write-Host "WARN: probe provision errored (code=$(Get-EnvelopeErrorCode $probeEnv)): $(Get-JsonProperty $probeError 'message')" -ForegroundColor Yellow Write-Host " Continuing -- individual tests will report specific failures." -ForegroundColor Yellow } @@ -386,7 +431,7 @@ try { $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_provision.json' $envObj = Assert-ResultEnvelope $r "provision" if ($envObj) { - $script:sandboxId = [string]$envObj.result.sandboxId + $script:sandboxId = Get-EnvelopeSandboxId $envObj Assert-True ($script:sandboxId -match '^wslc:[0-9a-f]{32}$') ` "sandbox_id matches wslc:<32-hex> ($script:sandboxId)" } @@ -399,7 +444,7 @@ try { $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_start.json' -SandboxId $script:sandboxId $envObj = Assert-ResultEnvelope $r "start" if ($envObj) { - Assert-True ($null -eq $envObj.result.metadata) "result.metadata absent (no start metadata in v1)" + Assert-True ($null -eq (Get-EnvelopeResultProperty $envObj 'metadata')) "result.metadata absent (no start metadata in v1)" } } } @@ -413,7 +458,7 @@ try { Assert-True ($r.Stdout -match 'wslc-state-aware-exec-marker') ` "stdout contains the script's output (relayed live, not enveloped)" $maybeEnv = Parse-Envelope -Stdout $r.Stdout - Assert-True ($null -eq $maybeEnv -or $null -eq $maybeEnv.error) ` + Assert-True ($null -eq (Get-EnvelopeError $maybeEnv)) ` "stdout is not a state-aware error envelope on success" } } @@ -500,7 +545,7 @@ try { $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_exec_rejected_filesystem.json' -SandboxId $script:sandboxId Assert-True ($r.ExitCode -ne 0) "exit code is non-zero (policy rejected)" $envObj = Parse-Envelope -Stdout $r.Stdout - $code = if ($envObj) { $envObj.error.code } else { '' } + $code = Get-EnvelopeErrorCode $envObj Assert-True ($code -eq 'policy_validation') "error.code is 'policy_validation' (got '$code')" } | Out-Null } @@ -530,7 +575,7 @@ try { $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_stop.json' -SandboxId $script:sandboxId Assert-True ($r.ExitCode -ne 0) "exit code is non-zero (stop on stale sandbox failed as expected)" $envObj = Parse-Envelope -Stdout $r.Stdout - $code = if ($envObj) { $envObj.error.code } else { '' } + $code = Get-EnvelopeErrorCode $envObj Assert-True ($code -eq 'not_provisioned') "error.code is 'not_provisioned' (got '$code')" } | Out-Null } @@ -559,7 +604,7 @@ try { $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_provision_with_filesystem.json' $envObj = Assert-ResultEnvelope $r "filesystem provision" if ($envObj) { - $script:fsSandboxId = [string]$envObj.result.sandboxId + $script:fsSandboxId = Get-EnvelopeSandboxId $envObj Assert-True ($script:fsSandboxId -match '^wslc:[0-9a-f]{32}$') ` "sandbox_id matches wslc:<32-hex> ($script:fsSandboxId)" } @@ -645,7 +690,7 @@ try { $netProvisionedOk = Run-StateAwareTest "C: provision (bridged network)" { $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_provision_bridged.json' $envObj = Assert-ResultEnvelope $r "bridged provision" - if ($envObj) { $script:netSandboxId = [string]$envObj.result.sandboxId } + if ($envObj) { $script:netSandboxId = Get-EnvelopeSandboxId $envObj } } $netStartedOk = $false @@ -696,7 +741,7 @@ Run-StateAwareTest "D: provision (deniedPaths nested under mount rejected)" { $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_provision_rejected_denied.json' Assert-True ($r.ExitCode -ne 0) "exit code is non-zero (policy rejected)" $envObj = Parse-Envelope -Stdout $r.Stdout - $code = if ($envObj) { $envObj.error.code } else { '' } + $code = Get-EnvelopeErrorCode $envObj Assert-True ($code -eq 'policy_validation') "error.code is 'policy_validation' (got '$code')" } | Out-Null @@ -704,7 +749,7 @@ Run-StateAwareTest "D: provision (host filtering rejected)" { $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_provision_rejected_hosts.json' Assert-True ($r.ExitCode -ne 0) "exit code is non-zero (policy rejected)" $envObj = Parse-Envelope -Stdout $r.Stdout - $code = if ($envObj) { $envObj.error.code } else { '' } + $code = Get-EnvelopeErrorCode $envObj Assert-True ($code -eq 'policy_validation') "error.code is 'policy_validation' (got '$code')" } | Out-Null @@ -712,7 +757,7 @@ Run-StateAwareTest "D: provision (proxy at provision rejected)" { $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_provision_rejected_proxy.json' Assert-True ($r.ExitCode -ne 0) "exit code is non-zero (policy rejected)" $envObj = Parse-Envelope -Stdout $r.Stdout - $code = if ($envObj) { $envObj.error.code } else { '' } + $code = Get-EnvelopeErrorCode $envObj Assert-True ($code -eq 'policy_validation') "error.code is 'policy_validation' (got '$code')" } | Out-Null @@ -721,7 +766,7 @@ Run-StateAwareTest "D: start (filesystem policy rejected)" { $r = Invoke-StateAware -Request $req Assert-True ($r.ExitCode -ne 0) "exit code is non-zero (policy rejected)" $envObj = Parse-Envelope -Stdout $r.Stdout - $code = if ($envObj) { $envObj.error.code } else { '' } + $code = Get-EnvelopeErrorCode $envObj Assert-True ($code -eq 'policy_validation') "error.code is 'policy_validation' (got '$code')" } | Out-Null @@ -735,7 +780,7 @@ Run-StateAwareTest "D: start (network.proxy rejected post-provision)" { $r = Invoke-StateAware -Request $req Assert-True ($r.ExitCode -ne 0) "exit code is non-zero (policy rejected)" $envObj = Parse-Envelope -Stdout $r.Stdout - $code = if ($envObj) { $envObj.error.code } else { '' } + $code = Get-EnvelopeErrorCode $envObj Assert-True ($code -eq 'policy_validation') "error.code is 'policy_validation' (got '$code')" } | Out-Null @@ -744,7 +789,7 @@ Run-StateAwareTest "D: stop (network.proxy rejected post-provision)" { $r = Invoke-StateAware -Request $req Assert-True ($r.ExitCode -ne 0) "exit code is non-zero (policy rejected)" $envObj = Parse-Envelope -Stdout $r.Stdout - $code = if ($envObj) { $envObj.error.code } else { '' } + $code = Get-EnvelopeErrorCode $envObj Assert-True ($code -eq 'policy_validation') "error.code is 'policy_validation' (got '$code')" } | Out-Null @@ -763,7 +808,7 @@ try { $reProvisionedOk = Run-StateAwareTest "E: provision (restart cycle)" { $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_provision.json' $envObj = Assert-ResultEnvelope $r "restart provision" - if ($envObj) { $script:reSandboxId = [string]$envObj.result.sandboxId } + if ($envObj) { $script:reSandboxId = Get-EnvelopeSandboxId $envObj } } $reStartedOk = $false @@ -801,7 +846,7 @@ try { $script:reRestartSucceeded = $true Write-Host " INFO: WSLc supports restart-after-stop on the same sandbox" -ForegroundColor DarkGray } elseif ($arm -eq 'error') { - Write-Host " INFO: WSLc does NOT support restart-after-stop (error.code=$($envObj.error.code)) -- documented limitation" -ForegroundColor DarkGray + Write-Host " INFO: WSLc does NOT support restart-after-stop (error.code=$(Get-EnvelopeErrorCode $envObj)) -- documented limitation" -ForegroundColor DarkGray } } | Out-Null @@ -843,7 +888,7 @@ try { $edgeProvisionedOk = Run-StateAwareTest "F: provision (exec-edge sandbox)" { $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_provision.json' $envObj = Assert-ResultEnvelope $r "edge provision" - if ($envObj) { $script:edgeSandboxId = [string]$envObj.result.sandboxId } + if ($envObj) { $script:edgeSandboxId = Get-EnvelopeSandboxId $envObj } } # F1: exec BEFORE start -> not_started (the daemon knows the id but the @@ -854,7 +899,7 @@ try { $r = Invoke-StateAware -Request $req Assert-True ($r.ExitCode -ne 0) "exit code is non-zero (exec before start rejected)" $envObj = Parse-Envelope -Stdout $r.Stdout - $code = if ($envObj) { $envObj.error.code } else { '' } + $code = Get-EnvelopeErrorCode $envObj Assert-True ($code -eq 'not_started') "error.code is 'not_started' (got '$code')" } | Out-Null } @@ -876,7 +921,7 @@ try { $r = Invoke-StateAware -Request $slow Assert-True ($r.ExitCode -ne 0) "timed-out exec exits non-zero" $envObj = Parse-Envelope -Stdout $r.Stdout - $code = if ($envObj) { $envObj.error.code } else { '' } + $code = Get-EnvelopeErrorCode $envObj Assert-True ($code -eq 'backend_error') "timeout maps to 'backend_error' (got '$code')" $after = @{ phase = 'exec'; sandboxId = $script:edgeSandboxId; process = @{ commandLine = 'echo survived-timeout'; timeout = 30000 } } @@ -912,7 +957,7 @@ try { $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_deprovision.json' -SandboxId $script:edgeSandboxId Assert-True ($r.ExitCode -ne 0) "second deprovision exits non-zero" $envObj = Parse-Envelope -Stdout $r.Stdout - $code = if ($envObj) { $envObj.error.code } else { '' } + $code = Get-EnvelopeErrorCode $envObj Assert-True ($code -eq 'not_provisioned') "error.code is 'not_provisioned' (got '$code')" } | Out-Null } @@ -937,12 +982,12 @@ try { $mcAProvOk = Run-StateAwareTest "G: provision sandbox A" { $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_provision.json' $envObj = Assert-ResultEnvelope $r "multi-container A provision" - if ($envObj) { $script:mcSandboxA = [string]$envObj.result.sandboxId } + if ($envObj) { $script:mcSandboxA = Get-EnvelopeSandboxId $envObj } } $mcBProvOk = Run-StateAwareTest "G: provision sandbox B" { $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_provision.json' $envObj = Assert-ResultEnvelope $r "multi-container B provision" - if ($envObj) { $script:mcSandboxB = [string]$envObj.result.sandboxId } + if ($envObj) { $script:mcSandboxB = Get-EnvelopeSandboxId $envObj } } if ($mcAProvOk -and $mcBProvOk) { @@ -1057,7 +1102,7 @@ try { $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_provision.json' $envObj = Assert-ResultEnvelope $r "post-teardown provision" if ($envObj) { - $script:recSandboxId = [string]$envObj.result.sandboxId + $script:recSandboxId = Get-EnvelopeSandboxId $envObj Assert-True ($script:recSandboxId -match '^wslc:[0-9a-f]{32}$') ` "sandbox_id matches wslc:<32-hex> ($script:recSandboxId)" } From 56a14028289d1515a97e6d182e9665062e1d71a5 Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Mon, 17 Aug 2026 16:46:47 -0700 Subject: [PATCH 02/16] adding host forwarding --- scripts/ci/prepare-linux-host.sh | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/scripts/ci/prepare-linux-host.sh b/scripts/ci/prepare-linux-host.sh index 062913762..bec5aac98 100644 --- a/scripts/ci/prepare-linux-host.sh +++ b/scripts/ci/prepare-linux-host.sh @@ -150,6 +150,22 @@ enable_bridge_netfilter() { done } +# Container-scoped rules hook egress (-i ), so a reply arrives in the +# opposite direction, matches nothing MXC installed, and falls through to the +# host's FORWARD policy. A DROP policy (Docker sets one) therefore breaks +# allowed destinations, and worse, makes the deny cases pass vacuously: a +# container with no working hook at all is equally unreachable. Forwarding by +# default leaves an MXC rule as the only thing that can block traffic. +allow_host_forwarding() { + local command + for command in iptables ip6tables; do + if command -v "$command" >/dev/null 2>&1; then + sudo "$command" -P FORWARD ACCEPT || + echo "WARNING: could not set the $command FORWARD policy to ACCEPT; deny-case tests may pass without enforcing anything." >&2 + fi + done +} + # Report the host-side state that container networking depends on. Purely # diagnostic: never fails the job, so a networking problem still surfaces as # the backend test failure rather than as a prerequisite error. @@ -172,8 +188,15 @@ report_lxc_network_diagnostics() { sudo iptables -t nat -S POSTROUTING 2>/dev/null | grep -E '10\.0\.3|MASQUERADE' | sed 's/^/ /' || echo " (none found)" - echo "FORWARD policy and bridge rules:" - sudo iptables -S FORWARD 2>/dev/null | grep -E "policy|$bridge" | sed 's/^/ /' || + # A vacuous pass depends on this policy, so print it verbatim. grep reads + # the whole ruleset, avoiding the SIGPIPE an early-closing head would send + # back to iptables under pipefail. + echo "FORWARD policy (must be ACCEPT, or deny cases prove nothing):" + sudo iptables -S FORWARD 2>/dev/null | grep '^-P' | sed 's/^/ /' || echo " (unknown)" + sudo ip6tables -S FORWARD 2>/dev/null | grep '^-P' | sed 's/^/ /' || echo " (unknown)" + + echo "FORWARD rules for the bridge:" + sudo iptables -S FORWARD 2>/dev/null | grep -E "$bridge" | sed 's/^/ /' || echo " (none found)" echo "Host /etc/resolv.conf nameservers:" @@ -227,6 +250,7 @@ case "$backend" in fi start_lxc_bridge enable_bridge_netfilter + allow_host_forwarding report_lxc_network_diagnostics ;; microvm) From a668c36134045ee0832a80dc2f418e445be4c9d4 Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Tue, 18 Aug 2026 10:49:32 -0700 Subject: [PATCH 03/16] NAT tests lxc --- scripts/ci/prepare-linux-host.sh | 77 +++++++++++++++++++++++++++++++- 1 file changed, 76 insertions(+), 1 deletion(-) diff --git a/scripts/ci/prepare-linux-host.sh b/scripts/ci/prepare-linux-host.sh index bec5aac98..29c20b093 100644 --- a/scripts/ci/prepare-linux-host.sh +++ b/scripts/ci/prepare-linux-host.sh @@ -129,6 +129,37 @@ start_lxc_bridge() { ip addr show "$bridge" || true } +# Outbound container traffic leaves the bridge subnet with a private source +# address, so it needs a MASQUERADE rule to reach anything off-host. lxc-net +# normally installs one, but it skips its firewall setup when it believes +# another manager owns the ruleset, leaving a bridge that hands out leases the +# container cannot use. The symptom is a name-resolution failure inside the +# guest, which reads like a policy problem and is not one. +ensure_bridge_nat() { + local bridge="${LXC_BRIDGE:-lxcbr0}" + local subnet + + subnet="$(ip -4 -o addr show "$bridge" 2>/dev/null | awk '{print $4}' | head -n 1)" + if [[ -z "$subnet" ]]; then + echo "WARNING: $bridge has no IPv4 subnet; skipping NAT setup." >&2 + return 0 + fi + + # Match on the source subnet rather than the rule text: lxc-net's own rule + # and ours are equivalent however they are spelled. + if sudo iptables -t nat -S POSTROUTING 2>/dev/null | + grep -q -- "-s ${subnet%%/*}"; then + echo "NAT for $subnet is already present." + return 0 + fi + + if sudo iptables -t nat -A POSTROUTING -s "$subnet" ! -d "$subnet" -j MASQUERADE; then + echo "Installed MASQUERADE for $subnet." + else + echo "WARNING: could not install MASQUERADE for $subnet; containers will not reach off-host destinations." >&2 + fi +} + # Container network policy is programmed as iptables rules reached from # FORWARD, which only sees bridged traffic when br_netfilter is loaded and # bridge-nf-call-iptables is enabled. Neither is guaranteed on a fresh image, @@ -185,7 +216,7 @@ report_lxc_network_diagnostics() { pgrep -af dnsmasq 2>/dev/null | sed 's/^/ /' || echo " (none)" echo "NAT rules for the bridge subnet:" - sudo iptables -t nat -S POSTROUTING 2>/dev/null | grep -E '10\.0\.3|MASQUERADE' | + sudo iptables -t nat -S POSTROUTING 2>/dev/null | grep -E 'MASQUERADE|SNAT' | sed 's/^/ /' || echo " (none found)" # A vacuous pass depends on this policy, so print it verbatim. grep reads @@ -228,6 +259,48 @@ report_lxc_network_diagnostics() { echo "--- end diagnostics ---" } +# Boot a throwaway container and confirm it gets an IPv4 lease and can reach +# off-host. Every host-side check can pass while a container still comes up +# IPv6-only, which surfaces as an unrelated-looking resolution failure much +# later in the suite. Purely diagnostic: never fails the job. +report_lxc_container_connectivity() { + local container="mxc-ci-netcheck-$$" + + echo "--- LXC container connectivity probe ---" + + if ! sudo lxc-create -n "$container" -t download -- \ + --dist alpine --release 3.23 --arch "$(dpkg --print-architecture 2>/dev/null || arch)" \ + >/dev/null 2>&1; then + echo " could not create the probe container; skipping." + echo "--- end probe ---" + return 0 + fi + + if sudo lxc-start -n "$container" >/dev/null 2>&1; then + # The suite's own containers wait about this long for an address. + sudo lxc-wait -n "$container" -s RUNNING -t 15 >/dev/null 2>&1 || true + sleep 5 + + echo " addresses:" + sudo lxc-info -n "$container" -iH 2>/dev/null | sed 's/^/ /' || echo " (none)" + + if sudo lxc-info -n "$container" -iH 2>/dev/null | grep -qE '^[0-9]+\.'; then + echo " IPv4: present" + else + echo " WARNING: no IPv4 address; DHCPv4 on the bridge is not working." >&2 + fi + + echo " resolution from inside the container:" + sudo lxc-attach -n "$container" -- getent hosts api.github.com 2>&1 | + sed 's/^/ /' || echo " FAILED - the container cannot resolve" + else + echo " probe container failed to start." + fi + + sudo lxc-destroy -n "$container" -f >/dev/null 2>&1 || true + echo "--- end probe ---" +} + chmod +x "$binary_directory/lxc-exec" case "$backend" in bubblewrap) @@ -251,7 +324,9 @@ case "$backend" in start_lxc_bridge enable_bridge_netfilter allow_host_forwarding + ensure_bridge_nat report_lxc_network_diagnostics + report_lxc_container_connectivity ;; microvm) for file in nanvixd.elf nanvix_rootfs.img python3.initrd bin/kernel.elf; do From f6a5af19b5c7c3f13f0580cc02745205beba106c Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Tue, 18 Aug 2026 12:13:14 -0700 Subject: [PATCH 04/16] Remove strict mode issues ; made LXC more honest with failures --- scripts/ci/prepare-linux-host.sh | 123 ------------------- scripts/ci/validation-test-matrix.json | 18 +++ tests/scripts/run_ci_backend_tests.ps1 | 1 - tests/scripts/run_wslc_all_tests.ps1 | 20 +-- tests/scripts/run_wslc_state_aware_tests.ps1 | 101 +++++---------- 5 files changed, 49 insertions(+), 214 deletions(-) diff --git a/scripts/ci/prepare-linux-host.sh b/scripts/ci/prepare-linux-host.sh index 29c20b093..f7afb2743 100644 --- a/scripts/ci/prepare-linux-host.sh +++ b/scripts/ci/prepare-linux-host.sh @@ -181,126 +181,6 @@ enable_bridge_netfilter() { done } -# Container-scoped rules hook egress (-i ), so a reply arrives in the -# opposite direction, matches nothing MXC installed, and falls through to the -# host's FORWARD policy. A DROP policy (Docker sets one) therefore breaks -# allowed destinations, and worse, makes the deny cases pass vacuously: a -# container with no working hook at all is equally unreachable. Forwarding by -# default leaves an MXC rule as the only thing that can block traffic. -allow_host_forwarding() { - local command - for command in iptables ip6tables; do - if command -v "$command" >/dev/null 2>&1; then - sudo "$command" -P FORWARD ACCEPT || - echo "WARNING: could not set the $command FORWARD policy to ACCEPT; deny-case tests may pass without enforcing anything." >&2 - fi - done -} - -# Report the host-side state that container networking depends on. Purely -# diagnostic: never fails the job, so a networking problem still surfaces as -# the backend test failure rather than as a prerequisite error. -report_lxc_network_diagnostics() { - local bridge="${LXC_BRIDGE:-lxcbr0}" - - echo "--- LXC network diagnostics (host) ---" - - echo "net.ipv4.ip_forward: $(cat /proc/sys/net/ipv4/ip_forward 2>/dev/null || echo unknown)" - - echo "bridge-nf-call-iptables: $(cat /proc/sys/net/bridge/bridge-nf-call-iptables 2>/dev/null || echo absent)" - - echo "Bridge $bridge:" - ip -4 addr show "$bridge" 2>/dev/null | sed 's/^/ /' || echo " (absent)" - - echo "dnsmasq processes:" - pgrep -af dnsmasq 2>/dev/null | sed 's/^/ /' || echo " (none)" - - echo "NAT rules for the bridge subnet:" - sudo iptables -t nat -S POSTROUTING 2>/dev/null | grep -E 'MASQUERADE|SNAT' | - sed 's/^/ /' || echo " (none found)" - - # A vacuous pass depends on this policy, so print it verbatim. grep reads - # the whole ruleset, avoiding the SIGPIPE an early-closing head would send - # back to iptables under pipefail. - echo "FORWARD policy (must be ACCEPT, or deny cases prove nothing):" - sudo iptables -S FORWARD 2>/dev/null | grep '^-P' | sed 's/^/ /' || echo " (unknown)" - sudo ip6tables -S FORWARD 2>/dev/null | grep '^-P' | sed 's/^/ /' || echo " (unknown)" - - echo "FORWARD rules for the bridge:" - sudo iptables -S FORWARD 2>/dev/null | grep -E "$bridge" | sed 's/^/ /' || - echo " (none found)" - - echo "Host /etc/resolv.conf nameservers:" - grep '^nameserver' /etc/resolv.conf 2>/dev/null | sed 's/^/ /' || echo " (none)" - - echo "lxc-net configuration:" - grep -E '^(USE_LXC_BRIDGE|LXC_ADDR|LXC_NETMASK|LXC_DHCP_RANGE|LXC_DHCP_CONFILE)' \ - /etc/default/lxc-net 2>/dev/null | sed 's/^/ /' || echo " (no /etc/default/lxc-net)" - - # Prove the host itself can resolve the name the network test uses. If this - # fails, the container was never going to succeed. - if command -v getent >/dev/null 2>&1; then - echo "Host resolution of api.github.com:" - getent ahostsv4 api.github.com 2>/dev/null | head -n 2 | sed 's/^/ /' || - echo " FAILED - the host cannot resolve it either" - fi - - # Ask the bridge's own resolver, which is what a container is handed via - # DHCP. This isolates "dnsmasq is broken" from "the host is fine". - local bridge_ip - bridge_ip="$(ip -4 -o addr show "$bridge" 2>/dev/null | - awk '{print $4}' | cut -d/ -f1 | head -n 1)" - if [[ -n "$bridge_ip" ]] && command -v nslookup >/dev/null 2>&1; then - echo "Resolution via bridge resolver ($bridge_ip):" - nslookup api.github.com "$bridge_ip" 2>&1 | tail -n 4 | sed 's/^/ /' || - echo " FAILED - dnsmasq on $bridge is not answering" - fi - - echo "--- end diagnostics ---" -} - -# Boot a throwaway container and confirm it gets an IPv4 lease and can reach -# off-host. Every host-side check can pass while a container still comes up -# IPv6-only, which surfaces as an unrelated-looking resolution failure much -# later in the suite. Purely diagnostic: never fails the job. -report_lxc_container_connectivity() { - local container="mxc-ci-netcheck-$$" - - echo "--- LXC container connectivity probe ---" - - if ! sudo lxc-create -n "$container" -t download -- \ - --dist alpine --release 3.23 --arch "$(dpkg --print-architecture 2>/dev/null || arch)" \ - >/dev/null 2>&1; then - echo " could not create the probe container; skipping." - echo "--- end probe ---" - return 0 - fi - - if sudo lxc-start -n "$container" >/dev/null 2>&1; then - # The suite's own containers wait about this long for an address. - sudo lxc-wait -n "$container" -s RUNNING -t 15 >/dev/null 2>&1 || true - sleep 5 - - echo " addresses:" - sudo lxc-info -n "$container" -iH 2>/dev/null | sed 's/^/ /' || echo " (none)" - - if sudo lxc-info -n "$container" -iH 2>/dev/null | grep -qE '^[0-9]+\.'; then - echo " IPv4: present" - else - echo " WARNING: no IPv4 address; DHCPv4 on the bridge is not working." >&2 - fi - - echo " resolution from inside the container:" - sudo lxc-attach -n "$container" -- getent hosts api.github.com 2>&1 | - sed 's/^/ /' || echo " FAILED - the container cannot resolve" - else - echo " probe container failed to start." - fi - - sudo lxc-destroy -n "$container" -f >/dev/null 2>&1 || true - echo "--- end probe ---" -} - chmod +x "$binary_directory/lxc-exec" case "$backend" in bubblewrap) @@ -323,10 +203,7 @@ case "$backend" in fi start_lxc_bridge enable_bridge_netfilter - allow_host_forwarding ensure_bridge_nat - report_lxc_network_diagnostics - report_lxc_container_connectivity ;; microvm) for file in nanvixd.elf nanvix_rootfs.img python3.initrd bin/kernel.elf; do diff --git a/scripts/ci/validation-test-matrix.json b/scripts/ci/validation-test-matrix.json index 96256003a..6b7fb8abc 100644 --- a/scripts/ci/validation-test-matrix.json +++ b/scripts/ci/validation-test-matrix.json @@ -411,6 +411,24 @@ "backends": [ "lxc" ] + }, + { + "os": "windows-25h2", + "backends": [ + "wslc" + ] + }, + { + "os": "windows-24h2", + "backends": [ + "wslc" + ] + }, + { + "os": "windows-23h2", + "backends": [ + "wslc" + ] } ] } diff --git a/tests/scripts/run_ci_backend_tests.ps1 b/tests/scripts/run_ci_backend_tests.ps1 index adcb916a9..f72a0b6a6 100644 --- a/tests/scripts/run_ci_backend_tests.ps1 +++ b/tests/scripts/run_ci_backend_tests.ps1 @@ -28,7 +28,6 @@ param( [string]$Architecture ) -Set-StrictMode -Version Latest $ErrorActionPreference = 'Stop' $scriptRoot = Split-Path -Parent $MyInvocation.MyCommand.Path diff --git a/tests/scripts/run_wslc_all_tests.ps1 b/tests/scripts/run_wslc_all_tests.ps1 index 694bdd0e0..9c2a6cb49 100644 --- a/tests/scripts/run_wslc_all_tests.ps1 +++ b/tests/scripts/run_wslc_all_tests.ps1 @@ -97,17 +97,6 @@ if (-not $SkipSetup) { } } -# Helper: StrictMode-safe property read; returns $null when the property (or the -# object) is absent. Lets the optional-config-field reads below work under the -# Set-StrictMode -Version Latest that run_ci_backend_tests.ps1 imposes. -function Get-JsonProperty { - param($Object, [Parameter(Mandatory)][string]$Name) - if ($null -eq $Object) { return $null } - $prop = $Object.PSObject.Properties[$Name] - if ($null -eq $prop) { return $null } - return $prop.Value -} - # Helper: run a single WSLC test config function Run-WslcTest { param( @@ -125,12 +114,9 @@ function Run-WslcTest { return @{ Name = $ConfigFile; Pass = $true; Skipped = $true; Reason = "File not found" } } - # Skip if the config references a tar file that doesn't exist locally. - # Read the chain defensively: this suite inherits Set-StrictMode -Version - # Latest from run_ci_backend_tests.ps1, under which touching a missing - # property is a terminating error, and most configs have no wslc.imageTarPath. + # Skip if the config references a tar file that doesn't exist locally $configJson = Get-Content $configPath -Raw | ConvertFrom-Json - $tarPath = Get-JsonProperty (Get-JsonProperty (Get-JsonProperty $configJson 'experimental') 'wslc') 'imageTarPath' + $tarPath = $configJson.experimental.wslc.imageTarPath if ($tarPath -and -not (Test-Path $tarPath)) { Write-Host " $ConfigFile ... " -NoNewline Write-Host "SKIP (tar not found: $tarPath)" -ForegroundColor Yellow @@ -183,7 +169,7 @@ function Run-WslcTest { # PostExitCheck runs after exit/output gates pass. Receives ($id, $output) # and must return truthy. Use for externally-observable state assertions. if ($pass -and $PostExitCheck) { - $containerId = Get-JsonProperty $configJson 'containerId' + $containerId = $configJson.containerId try { $checkResult = & $PostExitCheck $containerId $output if (-not $checkResult) { diff --git a/tests/scripts/run_wslc_state_aware_tests.ps1 b/tests/scripts/run_wslc_state_aware_tests.ps1 index 63967d9ac..2f2faa034 100644 --- a/tests/scripts/run_wslc_state_aware_tests.ps1 +++ b/tests/scripts/run_wslc_state_aware_tests.ps1 @@ -283,50 +283,6 @@ function Envelope-Arm { '' } -# StrictMode-safe property read; returns $null when the object or property is -# absent. run_ci_backend_tests.ps1 imposes Set-StrictMode -Version Latest, under -# which touching a missing property on a PSCustomObject is a terminating error -- -# and an envelope carries exactly one of `result` / `error`, so the other arm is -# always missing. -function Get-JsonProperty { - param($Object, [Parameter(Mandatory)][string]$Name) - if ($null -eq $Object) { return $null } - $prop = $Object.PSObject.Properties[$Name] - if ($null -eq $prop) { return $null } - return $prop.Value -} - -# The envelope's `error` object, or $null when the envelope is absent/succeeded. -function Get-EnvelopeError { - param($Envelope) - Get-JsonProperty $Envelope 'error' -} - -# The envelope's error code, or '' / '' when absent. -function Get-EnvelopeErrorCode { - param($Envelope) - if ($null -eq $Envelope) { return '' } - $err = Get-JsonProperty $Envelope 'error' - if ($null -eq $err) { return '' } - $code = Get-JsonProperty $err 'code' - if ($null -eq $code) { return '' } - return [string]$code -} - -# A property of the envelope's `result` object, or $null when absent. -function Get-EnvelopeResultProperty { - param($Envelope, [Parameter(Mandatory)][string]$Name) - Get-JsonProperty (Get-JsonProperty $Envelope 'result') $Name -} - -# The envelope's `result.sandboxId` as a string, or $null when absent. -function Get-EnvelopeSandboxId { - param($Envelope) - $id = Get-EnvelopeResultProperty $Envelope 'sandboxId' - if ($null -eq $id) { return $null } - return [string]$id -} - # Is the daemon process currently running? function Test-DaemonRunning { $null -ne (Get-Process -Name $DaemonProcName -ErrorAction SilentlyContinue) @@ -389,7 +345,7 @@ function Provision-Sandbox { $r = Invoke-StateAware -ConfigFile $ConfigFile $envObj = Parse-Envelope -Stdout $r.Stdout if ((Envelope-Arm $envObj) -ne 'result') { return $null } - Get-EnvelopeSandboxId $envObj + [string]$envObj.result.sandboxId } # ---------------- Backend-availability probe ---------------- @@ -407,17 +363,16 @@ if ($script:PreExistingDaemon) { $probe = Invoke-StateAware -ConfigFile 'wslc_state_aware_provision.json' $probeEnv = Parse-Envelope -Stdout $probe.Stdout -$probeError = Get-EnvelopeError $probeEnv -if ((Get-EnvelopeErrorCode $probeEnv) -eq 'backend_unavailable') { +if ($null -ne $probeEnv -and $probeEnv.error.code -eq 'backend_unavailable') { Write-Host "SKIPPED: wxc-exec reports backend_unavailable (built without --features wslc, or no WSLC runtime)" -ForegroundColor Yellow exit 0 } -$probeSandboxId = Get-EnvelopeSandboxId $probeEnv -if ($null -ne $probeSandboxId) { +if ($null -ne $probeEnv -and $null -ne $probeEnv.result -and $null -ne $probeEnv.result.sandboxId) { + $probeSandboxId = [string]$probeEnv.result.sandboxId Write-Host "Backend probe: provisioned $probeSandboxId, deprovisioning ..." -ForegroundColor DarkGray $null = Invoke-StateAware -ConfigFile 'wslc_state_aware_deprovision.json' -SandboxId $probeSandboxId -} elseif ($null -ne $probeError) { - Write-Host "WARN: probe provision errored (code=$(Get-EnvelopeErrorCode $probeEnv)): $(Get-JsonProperty $probeError 'message')" -ForegroundColor Yellow +} elseif ($null -ne $probeEnv -and $null -ne $probeEnv.error) { + Write-Host "WARN: probe provision errored (code=$($probeEnv.error.code)): $($probeEnv.error.message)" -ForegroundColor Yellow Write-Host " Continuing -- individual tests will report specific failures." -ForegroundColor Yellow } @@ -431,7 +386,7 @@ try { $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_provision.json' $envObj = Assert-ResultEnvelope $r "provision" if ($envObj) { - $script:sandboxId = Get-EnvelopeSandboxId $envObj + $script:sandboxId = [string]$envObj.result.sandboxId Assert-True ($script:sandboxId -match '^wslc:[0-9a-f]{32}$') ` "sandbox_id matches wslc:<32-hex> ($script:sandboxId)" } @@ -444,7 +399,7 @@ try { $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_start.json' -SandboxId $script:sandboxId $envObj = Assert-ResultEnvelope $r "start" if ($envObj) { - Assert-True ($null -eq (Get-EnvelopeResultProperty $envObj 'metadata')) "result.metadata absent (no start metadata in v1)" + Assert-True ($null -eq $envObj.result.metadata) "result.metadata absent (no start metadata in v1)" } } } @@ -458,7 +413,7 @@ try { Assert-True ($r.Stdout -match 'wslc-state-aware-exec-marker') ` "stdout contains the script's output (relayed live, not enveloped)" $maybeEnv = Parse-Envelope -Stdout $r.Stdout - Assert-True ($null -eq (Get-EnvelopeError $maybeEnv)) ` + Assert-True ($null -eq $maybeEnv -or $null -eq $maybeEnv.error) ` "stdout is not a state-aware error envelope on success" } } @@ -545,7 +500,7 @@ try { $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_exec_rejected_filesystem.json' -SandboxId $script:sandboxId Assert-True ($r.ExitCode -ne 0) "exit code is non-zero (policy rejected)" $envObj = Parse-Envelope -Stdout $r.Stdout - $code = Get-EnvelopeErrorCode $envObj + $code = if ($envObj) { $envObj.error.code } else { '' } Assert-True ($code -eq 'policy_validation') "error.code is 'policy_validation' (got '$code')" } | Out-Null } @@ -575,7 +530,7 @@ try { $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_stop.json' -SandboxId $script:sandboxId Assert-True ($r.ExitCode -ne 0) "exit code is non-zero (stop on stale sandbox failed as expected)" $envObj = Parse-Envelope -Stdout $r.Stdout - $code = Get-EnvelopeErrorCode $envObj + $code = if ($envObj) { $envObj.error.code } else { '' } Assert-True ($code -eq 'not_provisioned') "error.code is 'not_provisioned' (got '$code')" } | Out-Null } @@ -604,7 +559,7 @@ try { $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_provision_with_filesystem.json' $envObj = Assert-ResultEnvelope $r "filesystem provision" if ($envObj) { - $script:fsSandboxId = Get-EnvelopeSandboxId $envObj + $script:fsSandboxId = [string]$envObj.result.sandboxId Assert-True ($script:fsSandboxId -match '^wslc:[0-9a-f]{32}$') ` "sandbox_id matches wslc:<32-hex> ($script:fsSandboxId)" } @@ -690,7 +645,7 @@ try { $netProvisionedOk = Run-StateAwareTest "C: provision (bridged network)" { $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_provision_bridged.json' $envObj = Assert-ResultEnvelope $r "bridged provision" - if ($envObj) { $script:netSandboxId = Get-EnvelopeSandboxId $envObj } + if ($envObj) { $script:netSandboxId = [string]$envObj.result.sandboxId } } $netStartedOk = $false @@ -741,7 +696,7 @@ Run-StateAwareTest "D: provision (deniedPaths nested under mount rejected)" { $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_provision_rejected_denied.json' Assert-True ($r.ExitCode -ne 0) "exit code is non-zero (policy rejected)" $envObj = Parse-Envelope -Stdout $r.Stdout - $code = Get-EnvelopeErrorCode $envObj + $code = if ($envObj) { $envObj.error.code } else { '' } Assert-True ($code -eq 'policy_validation') "error.code is 'policy_validation' (got '$code')" } | Out-Null @@ -749,7 +704,7 @@ Run-StateAwareTest "D: provision (host filtering rejected)" { $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_provision_rejected_hosts.json' Assert-True ($r.ExitCode -ne 0) "exit code is non-zero (policy rejected)" $envObj = Parse-Envelope -Stdout $r.Stdout - $code = Get-EnvelopeErrorCode $envObj + $code = if ($envObj) { $envObj.error.code } else { '' } Assert-True ($code -eq 'policy_validation') "error.code is 'policy_validation' (got '$code')" } | Out-Null @@ -757,7 +712,7 @@ Run-StateAwareTest "D: provision (proxy at provision rejected)" { $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_provision_rejected_proxy.json' Assert-True ($r.ExitCode -ne 0) "exit code is non-zero (policy rejected)" $envObj = Parse-Envelope -Stdout $r.Stdout - $code = Get-EnvelopeErrorCode $envObj + $code = if ($envObj) { $envObj.error.code } else { '' } Assert-True ($code -eq 'policy_validation') "error.code is 'policy_validation' (got '$code')" } | Out-Null @@ -766,7 +721,7 @@ Run-StateAwareTest "D: start (filesystem policy rejected)" { $r = Invoke-StateAware -Request $req Assert-True ($r.ExitCode -ne 0) "exit code is non-zero (policy rejected)" $envObj = Parse-Envelope -Stdout $r.Stdout - $code = Get-EnvelopeErrorCode $envObj + $code = if ($envObj) { $envObj.error.code } else { '' } Assert-True ($code -eq 'policy_validation') "error.code is 'policy_validation' (got '$code')" } | Out-Null @@ -780,7 +735,7 @@ Run-StateAwareTest "D: start (network.proxy rejected post-provision)" { $r = Invoke-StateAware -Request $req Assert-True ($r.ExitCode -ne 0) "exit code is non-zero (policy rejected)" $envObj = Parse-Envelope -Stdout $r.Stdout - $code = Get-EnvelopeErrorCode $envObj + $code = if ($envObj) { $envObj.error.code } else { '' } Assert-True ($code -eq 'policy_validation') "error.code is 'policy_validation' (got '$code')" } | Out-Null @@ -789,7 +744,7 @@ Run-StateAwareTest "D: stop (network.proxy rejected post-provision)" { $r = Invoke-StateAware -Request $req Assert-True ($r.ExitCode -ne 0) "exit code is non-zero (policy rejected)" $envObj = Parse-Envelope -Stdout $r.Stdout - $code = Get-EnvelopeErrorCode $envObj + $code = if ($envObj) { $envObj.error.code } else { '' } Assert-True ($code -eq 'policy_validation') "error.code is 'policy_validation' (got '$code')" } | Out-Null @@ -808,7 +763,7 @@ try { $reProvisionedOk = Run-StateAwareTest "E: provision (restart cycle)" { $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_provision.json' $envObj = Assert-ResultEnvelope $r "restart provision" - if ($envObj) { $script:reSandboxId = Get-EnvelopeSandboxId $envObj } + if ($envObj) { $script:reSandboxId = [string]$envObj.result.sandboxId } } $reStartedOk = $false @@ -846,7 +801,7 @@ try { $script:reRestartSucceeded = $true Write-Host " INFO: WSLc supports restart-after-stop on the same sandbox" -ForegroundColor DarkGray } elseif ($arm -eq 'error') { - Write-Host " INFO: WSLc does NOT support restart-after-stop (error.code=$(Get-EnvelopeErrorCode $envObj)) -- documented limitation" -ForegroundColor DarkGray + Write-Host " INFO: WSLc does NOT support restart-after-stop (error.code=$($envObj.error.code)) -- documented limitation" -ForegroundColor DarkGray } } | Out-Null @@ -888,7 +843,7 @@ try { $edgeProvisionedOk = Run-StateAwareTest "F: provision (exec-edge sandbox)" { $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_provision.json' $envObj = Assert-ResultEnvelope $r "edge provision" - if ($envObj) { $script:edgeSandboxId = Get-EnvelopeSandboxId $envObj } + if ($envObj) { $script:edgeSandboxId = [string]$envObj.result.sandboxId } } # F1: exec BEFORE start -> not_started (the daemon knows the id but the @@ -899,7 +854,7 @@ try { $r = Invoke-StateAware -Request $req Assert-True ($r.ExitCode -ne 0) "exit code is non-zero (exec before start rejected)" $envObj = Parse-Envelope -Stdout $r.Stdout - $code = Get-EnvelopeErrorCode $envObj + $code = if ($envObj) { $envObj.error.code } else { '' } Assert-True ($code -eq 'not_started') "error.code is 'not_started' (got '$code')" } | Out-Null } @@ -921,7 +876,7 @@ try { $r = Invoke-StateAware -Request $slow Assert-True ($r.ExitCode -ne 0) "timed-out exec exits non-zero" $envObj = Parse-Envelope -Stdout $r.Stdout - $code = Get-EnvelopeErrorCode $envObj + $code = if ($envObj) { $envObj.error.code } else { '' } Assert-True ($code -eq 'backend_error') "timeout maps to 'backend_error' (got '$code')" $after = @{ phase = 'exec'; sandboxId = $script:edgeSandboxId; process = @{ commandLine = 'echo survived-timeout'; timeout = 30000 } } @@ -957,7 +912,7 @@ try { $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_deprovision.json' -SandboxId $script:edgeSandboxId Assert-True ($r.ExitCode -ne 0) "second deprovision exits non-zero" $envObj = Parse-Envelope -Stdout $r.Stdout - $code = Get-EnvelopeErrorCode $envObj + $code = if ($envObj) { $envObj.error.code } else { '' } Assert-True ($code -eq 'not_provisioned') "error.code is 'not_provisioned' (got '$code')" } | Out-Null } @@ -982,12 +937,12 @@ try { $mcAProvOk = Run-StateAwareTest "G: provision sandbox A" { $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_provision.json' $envObj = Assert-ResultEnvelope $r "multi-container A provision" - if ($envObj) { $script:mcSandboxA = Get-EnvelopeSandboxId $envObj } + if ($envObj) { $script:mcSandboxA = [string]$envObj.result.sandboxId } } $mcBProvOk = Run-StateAwareTest "G: provision sandbox B" { $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_provision.json' $envObj = Assert-ResultEnvelope $r "multi-container B provision" - if ($envObj) { $script:mcSandboxB = Get-EnvelopeSandboxId $envObj } + if ($envObj) { $script:mcSandboxB = [string]$envObj.result.sandboxId } } if ($mcAProvOk -and $mcBProvOk) { @@ -1102,7 +1057,7 @@ try { $r = Invoke-StateAware -ConfigFile 'wslc_state_aware_provision.json' $envObj = Assert-ResultEnvelope $r "post-teardown provision" if ($envObj) { - $script:recSandboxId = Get-EnvelopeSandboxId $envObj + $script:recSandboxId = [string]$envObj.result.sandboxId Assert-True ($script:recSandboxId -match '^wslc:[0-9a-f]{32}$') ` "sandbox_id matches wslc:<32-hex> ($script:recSandboxId)" } From 6c8c8339e5fdd1df4b0b45ec2a0bce9dd7e2f63d Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Tue, 18 Aug 2026 12:16:29 -0700 Subject: [PATCH 05/16] testing scaleset --- .github/workflows/Validation.Tests.Matrix.Job.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/Validation.Tests.Matrix.Job.yml b/.github/workflows/Validation.Tests.Matrix.Job.yml index 9a958d94a..bfd2ca10e 100644 --- a/.github/workflows/Validation.Tests.Matrix.Job.yml +++ b/.github/workflows/Validation.Tests.Matrix.Job.yml @@ -91,7 +91,7 @@ jobs: strategy: fail-fast: false matrix: ${{ fromJSON(needs.resolve.outputs.linux) }} - runs-on: [ self-hosted, "1ES.Pool=${{ matrix.pool }}", "JobId=mxc-e2e-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" ] + runs-on: ["${{ matrix.pool }}"] timeout-minutes: 60 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 From fbef27dbee19c85ac9be8d14b171f2b9bc22c298 Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Tue, 18 Aug 2026 12:51:11 -0700 Subject: [PATCH 06/16] scaleset isn't supported yet --- .github/workflows/Validation.Tests.Matrix.Job.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/Validation.Tests.Matrix.Job.yml b/.github/workflows/Validation.Tests.Matrix.Job.yml index bfd2ca10e..9a958d94a 100644 --- a/.github/workflows/Validation.Tests.Matrix.Job.yml +++ b/.github/workflows/Validation.Tests.Matrix.Job.yml @@ -91,7 +91,7 @@ jobs: strategy: fail-fast: false matrix: ${{ fromJSON(needs.resolve.outputs.linux) }} - runs-on: ["${{ matrix.pool }}"] + runs-on: [ self-hosted, "1ES.Pool=${{ matrix.pool }}", "JobId=mxc-e2e-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" ] timeout-minutes: 60 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 From cfaa550830f6f555fedee84df089eef15e60a119 Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Tue, 18 Aug 2026 12:57:09 -0700 Subject: [PATCH 07/16] create test file that wslc test script expects --- tests/scripts/run_wslc_all_tests.ps1 | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/scripts/run_wslc_all_tests.ps1 b/tests/scripts/run_wslc_all_tests.ps1 index 9c2a6cb49..993037bf1 100644 --- a/tests/scripts/run_wslc_all_tests.ps1 +++ b/tests/scripts/run_wslc_all_tests.ps1 @@ -215,6 +215,15 @@ $null = $results.Add((Run-WslcTest "wslc_stderr.json" -OutputContains "stdout me $null = $results.Add((Run-WslcTest "wslc_large_output.json")) Write-Host "`n--- Filesystem Tests ---" -ForegroundColor Cyan +# The filesystem configs mount C:\workspace and read test.txt from it. A +# developer machine usually has both already; a clean runner has neither, so +# create them here rather than leaving the mount empty. +$WorkspaceDir = "C:\workspace" +New-Item -ItemType Directory -Path $WorkspaceDir -Force | Out-Null +$WorkspaceFile = Join-Path $WorkspaceDir "test.txt" +if (-not (Test-Path $WorkspaceFile)) { + Set-Content $WorkspaceFile "workspace fixture" +} # wslc_filesystem.json also asserts cpuCount + memoryMb enforcement via nproc and /proc/meminfo. $null = $results.Add((Run-WslcTest "wslc_filesystem.json" ` -OutputMatches "(?s)PASS: filesystem mount visible.*PASS: cpuCount enforced.*PASS: memoryMb enforced")) From 1ce43fdae889a0fb0c6b102b00e2306c568e1629 Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Tue, 18 Aug 2026 13:46:20 -0700 Subject: [PATCH 08/16] stagger WSLC jobs --- .../workflows/Validation.Tests.Matrix.Job.yml | 14 ++++- docs/ci-validation-infrastructure.md | 43 ++++++++++++- scripts/ci/resolve-validation-test-matrix.mjs | 63 +++++++++++++++++++ scripts/ci/validation-test-matrix.json | 6 ++ 4 files changed, 124 insertions(+), 2 deletions(-) diff --git a/.github/workflows/Validation.Tests.Matrix.Job.yml b/.github/workflows/Validation.Tests.Matrix.Job.yml index 9a958d94a..fb702189a 100644 --- a/.github/workflows/Validation.Tests.Matrix.Job.yml +++ b/.github/workflows/Validation.Tests.Matrix.Job.yml @@ -38,7 +38,8 @@ jobs: fail-fast: false matrix: ${{ fromJSON(needs.resolve.outputs.windows) }} runs-on: [ self-hosted, "1ES.Pool=${{ matrix.pool }}", "JobId=mxc-e2e-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" ] - timeout-minutes: 60 + # Includes any startup stagger, so a delayed entry keeps the full test budget. + timeout-minutes: ${{ 60 + (matrix.startup_delay_minutes || 0) }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 @@ -48,6 +49,17 @@ jobs: name: ${{ matrix.artifact }} path: artifacts/bin + - name: Delayed start + if: matrix.startup_delay_minutes > 0 + shell: pwsh + run: | + # Spreads network-heavy setup across the pool's shared egress address + # instead of issuing every job's downloads at once. Configured by + # backendDelayedStart in scripts/ci/validation-test-matrix.json. + $minutes = ${{ matrix.startup_delay_minutes || 0 }} + Write-Host "Waiting $minutes minute(s) before downloading prerequisites." + Start-Sleep -Seconds ($minutes * 60) + - name: Prepare backend prerequisites timeout-minutes: 15 shell: pwsh diff --git a/docs/ci-validation-infrastructure.md b/docs/ci-validation-infrastructure.md index ec0e99486..c1baae81f 100644 --- a/docs/ci-validation-infrastructure.md +++ b/docs/ci-validation-infrastructure.md @@ -27,7 +27,7 @@ the individual local test scripts are documented in |------|------| | `.github/workflows/Validation.Tests.Scheduled.yml` | Scheduled entry point. Builds artifacts, then calls the matrix job. | | `.github/workflows/Validation.Tests.Matrix.Job.yml` | `workflow_call`-only. Resolves the plan and runs the per-family test jobs. | -| `scripts/ci/validation-test-matrix.json` | The matrix: OS versions, backends, triggers. | +| `scripts/ci/validation-test-matrix.json` | The matrix: OS versions, backends, triggers, job staggering. | | `scripts/ci/resolve-validation-test-matrix.mjs` | Matrix validator + plan expander. Emits the GitHub Actions matrices. | | `scripts/ci/prepare-windows-host.ps1` | Per-backend Windows host preparation / prerequisite assertions. | | `scripts/ci/prepare-linux-host.sh` | Per-backend Linux package install and service startup (distro-aware). | @@ -157,6 +157,40 @@ isolation-session, wslc; 25H2/24H2/23H2 × process-t3 + wslc) and 8 Linux (each of the four distros × bubblewrap + lxc). macOS resolves empty because Seatbelt has no wired suite. +### `backendDelayedStart` + +Optional. Staggers the start of jobs for a named backend instead of letting +them all begin at once: + +```json +"backendDelayedStart": [ + { "backend": "wslc", "minutes": 5 } +] +``` + +Every runner in a pool shares one egress address. A backend whose setup +downloads a large runtime or several container images therefore concentrates +that traffic into a burst when its jobs start together, which draws rate +limiting from public registries and stalled downloads. + +`minutes` is the gap between consecutive jobs of that backend, counted +independently per backend, following the resolved job order. With the entry +above, four WSLC jobs start at 0, 5, 10, and 15 minutes. + +The resolver emits the offset as `startup_delay_minutes` on each affected +matrix entry. The job sleeps that long before its first network step, and its +`timeout-minutes` grows by the same amount so a delayed entry keeps the full +test budget. Entries for other backends carry no such field and never wait. + +Omit the section (or leave it empty) to have every job start as soon as its +runner is ready. A backend id that no plan schedules is accepted and simply +never applies. + +Note this holds the runner while it sleeps: Actions cannot defer allocating a +matrix job, so the wait happens inside the job. Keep the value only as large as +the contention requires. It spreads simultaneous load — it does not help a +single download that stalls on its own. + ## Backend status Snapshot of what the matrix actually proves today. Update this table as backends @@ -282,6 +316,13 @@ Replace the explicit failure in the dispatcher with the suite invocation, add any host prerequisites, then add the OS/backend pair to a trigger. Always verify by testing it ahead of time. +### Stagger a backend's job starts + +Add or edit its `backendDelayedStart` entry in the catalog, then resolve +locally to confirm the offsets. Reach for this when a backend's setup is +network-heavy enough that concurrent jobs hit rate limits or stalled +downloads; remove the entry once that pressure is gone. + ### Change the schedule Everything schedule-related lives in `Validation.Tests.Scheduled.yml`: the two diff --git a/scripts/ci/resolve-validation-test-matrix.mjs b/scripts/ci/resolve-validation-test-matrix.mjs index 9c73382d2..79cae85b7 100644 --- a/scripts/ci/resolve-validation-test-matrix.mjs +++ b/scripts/ci/resolve-validation-test-matrix.mjs @@ -131,9 +131,41 @@ export function validateCatalog(catalog) { } } + validateBackendDelayedStart(catalog); + return { platforms, plans }; } +// backendDelayedStart is optional: an absent or empty section means every job +// starts as soon as its runner is ready. +function validateBackendDelayedStart(catalog) { + const delays = catalog.backendDelayedStart; + if (delays == null) { + return; + } + if (!Array.isArray(delays)) { + throw new Error('catalog backendDelayedStart must be an array'); + } + + const seen = new Set(); + for (const entry of delays) { + if (entry == null || typeof entry !== 'object' || Array.isArray(entry)) { + throw new Error('each backendDelayedStart entry must be an object'); + } + assertNonEmptyString(entry.backend, 'backendDelayedStart backend'); + if (seen.has(entry.backend)) { + throw new Error(`duplicate backendDelayedStart entry for ${entry.backend}`); + } + seen.add(entry.backend); + + if (!Number.isInteger(entry.minutes) || entry.minutes < 0) { + throw new Error( + `backendDelayedStart ${entry.backend} minutes must be a non-negative integer` + ); + } + } +} + export function expandPlan(catalog, plan) { const { platforms, plans } = validateCatalog(catalog); if (!plans.includes(plan)) { @@ -186,9 +218,40 @@ export function resolvePlan(catalog, plan) { suppressNonMacArm64(matrices); sortMatrices(matrices); + applyDelayedStart(matrices, catalog.backendDelayedStart); return matrices; } +// A backend named in backendDelayedStart has its jobs started at staggered +// offsets rather than all at once. This exists for backends whose setup +// downloads a large runtime or several container images: every runner in a +// pool shares one egress address, so simultaneous starts concentrate that +// traffic into a burst that draws rate limiting and stalled downloads. +// +// The entry's minutes value is the gap between consecutive jobs of that +// backend, counted independently per backend. Applied after sorting so the +// assignment follows the emitted order and stays reproducible. +function applyDelayedStart(matrices, delays) { + if (!Array.isArray(delays) || delays.length === 0) { + return; + } + + const stepMinutes = new Map(delays.map(entry => [entry.backend, entry.minutes])); + const scheduled = new Map(); + + for (const family of FAMILIES) { + for (const entry of matrices[family]) { + const step = stepMinutes.get(entry.backend); + if (step === undefined) { + continue; + } + const position = scheduled.get(entry.backend) ?? 0; + entry.startup_delay_minutes = position * step; + scheduled.set(entry.backend, position + 1); + } + } +} + // Windows and Linux ARM64 hosted VMs currently lack nested virtualization. // Keep their catalog entries intact for future enablement, but never emit them // until suitable test hosts are available. macOS remains ARM64-only. diff --git a/scripts/ci/validation-test-matrix.json b/scripts/ci/validation-test-matrix.json index 6b7fb8abc..cacaba214 100644 --- a/scripts/ci/validation-test-matrix.json +++ b/scripts/ci/validation-test-matrix.json @@ -325,6 +325,12 @@ } } ], + "backendDelayedStart": [ + { + "backend": "wslc", + "minutes": 2 + } + ], "triggers": { "pr": [], "nightly": [ From c6aad1655dc5839cb0f4a596aeb56ccaca0885c5 Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Tue, 18 Aug 2026 14:08:23 -0700 Subject: [PATCH 09/16] log upload should now include all windows logs --- .../workflows/Validation.Tests.Matrix.Job.yml | 22 ++-- scripts/ci/validation-test-matrix.json | 18 +-- tests/scripts/run_ci_backend_tests.ps1 | 107 +++++++++++++----- 3 files changed, 96 insertions(+), 51 deletions(-) diff --git a/.github/workflows/Validation.Tests.Matrix.Job.yml b/.github/workflows/Validation.Tests.Matrix.Job.yml index fb702189a..dc6726a57 100644 --- a/.github/workflows/Validation.Tests.Matrix.Job.yml +++ b/.github/workflows/Validation.Tests.Matrix.Job.yml @@ -83,16 +83,22 @@ jobs: Tee-Object -FilePath (Join-Path $env:RUNNER_TEMP 'mxc-ci.log') -Append if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - - name: Upload failure logs - if: failure() || cancelled() + - name: Upload logs + if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a with: name: logs-${{ inputs.plan }}-${{ matrix.os }}-${{ matrix.architecture }}-${{ matrix.backend }}-${{ github.run_attempt }} path: | ${{ runner.temp }}/mxc-ci.log - ${{ runner.temp }}/mxc-wpc-tests/logs - ${{ runner.temp }}/WinProcessContainer-Tests.results.* + ${{ runner.temp }}/mxc-wpc-tests + ${{ runner.temp }}/mxc-t3-workloads ${{ runner.temp }}/mxc_concurrent_oneshot + ${{ runner.temp }}/mxc_etw_test + ${{ runner.temp }}/wxc-wsb + ${{ runner.temp }}/wxc-sandbox-rendezvous + ${{ runner.temp }}/mxc-diagnostics-* + ${{ runner.temp }}/WinProcessContainer-Tests.results.* + ${{ runner.temp }}/WinProcessContainer-Tests.cargo.log if-no-files-found: ignore retention-days: 7 @@ -138,8 +144,8 @@ jobs: tee -a "$RUNNER_TEMP/mxc-ci.log" fi - - name: Upload failure logs - if: failure() || cancelled() + - name: Upload logs + if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a with: name: logs-${{ inputs.plan }}-${{ matrix.os }}-${{ matrix.architecture }}-${{ matrix.backend }}-${{ github.run_attempt }} @@ -175,8 +181,8 @@ jobs: '${{ matrix.backend }}' "$GITHUB_WORKSPACE/artifacts/bin" 2>&1 | tee "$RUNNER_TEMP/mxc-ci.log" - - name: Upload failure logs - if: failure() || cancelled() + - name: Upload logs + if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a with: name: logs-${{ inputs.plan }}-${{ matrix.os }}-${{ matrix.architecture }}-${{ matrix.backend }}-${{ github.run_attempt }} diff --git a/scripts/ci/validation-test-matrix.json b/scripts/ci/validation-test-matrix.json index cacaba214..c1bbd3816 100644 --- a/scripts/ci/validation-test-matrix.json +++ b/scripts/ci/validation-test-matrix.json @@ -397,25 +397,15 @@ { "os": "ubuntu-26.04", "backends": [ + "bubblewrap", "lxc" ] }, { - "os": "ubuntu-24.04", - "backends": [ - "lxc" - ] - }, - { - "os": "rhel-10", - "backends": [ - "lxc" - ] - }, - { - "os": "debian-13", + "os": "windows-prerelease-process-container", "backends": [ - "lxc" + "wslc", + "process-t1" ] }, { diff --git a/tests/scripts/run_ci_backend_tests.ps1 b/tests/scripts/run_ci_backend_tests.ps1 index f72a0b6a6..572b2618c 100644 --- a/tests/scripts/run_ci_backend_tests.ps1 +++ b/tests/scripts/run_ci_backend_tests.ps1 @@ -42,6 +42,47 @@ function Assert-File { } } +# The suites write their logs, configs, and transcripts under $env:TEMP, which +# is not the directory CI uploads from. Several also refuse to run with those +# paths pointed elsewhere, so copy the artifacts across afterwards instead of +# redirecting them. Best-effort by design: losing a log must not turn a passing +# run red, and a failing run keeps its original result. +function Copy-TempArtifacts { + if (-not $env:RUNNER_TEMP -or -not $env:TEMP) { + return + } + + $names = @( + 'mxc-wpc-tests', + 'mxc-t3-workloads', + 'mxc_concurrent_oneshot', + 'mxc_etw_test', + 'wxc-wsb', + 'wxc-sandbox-rendezvous', + 'WinProcessContainer-Tests.results.txt', + 'WinProcessContainer-Tests.results.json', + 'WinProcessContainer-Tests.cargo.log' + ) + + foreach ($name in $names) { + $source = Join-Path $env:TEMP $name + if (-not (Test-Path -LiteralPath $source)) { + continue + } + Copy-Item -Recurse -Force -LiteralPath $source ` + -Destination (Join-Path $env:RUNNER_TEMP $name) ` + -ErrorAction SilentlyContinue + } + + # MXC's own diagnostic dumps are timestamped per run, so match the family. + Get-ChildItem -Path $env:TEMP -Filter 'mxc-diagnostics-*' -ErrorAction SilentlyContinue | + ForEach-Object { + Copy-Item -Recurse -Force -LiteralPath $_.FullName ` + -Destination (Join-Path $env:RUNNER_TEMP $_.Name) ` + -ErrorAction SilentlyContinue + } +} + function Invoke-TestScript { param( [Parameter(Mandatory)][string]$Path, @@ -77,6 +118,8 @@ function Invoke-ProcessContainerTests { Copy-Item -LiteralPath $uiProbe -Destination (Join-Path $releaseDirectory 'wxc-ui-probe.exe') -Force $script = Join-Path $scriptRoot 'WinProcessContainer-Tests.ps1' + # -KeepArtifacts stops the harness deleting its scratch tree on a clean + # run, so a passing job still uploads its per-test logs and configs. # Skip build and Cargo phases because this job consumes a previously # built artifact; retain the host and containment behavior phases. $phases = @( @@ -96,45 +139,51 @@ function Invoke-ProcessContainerTests { -WxcRelease (Join-Path $releaseDirectory 'wxc-exec.exe') ` -UiProbeDebug (Join-Path $debugDirectory 'wxc-ui-probe.exe') ` -UiProbeRelease (Join-Path $releaseDirectory 'wxc-ui-probe.exe') ` + -KeepArtifacts ` -Phases $phases if ($LASTEXITCODE -ne 0) { throw "Process Container tests failed with exit code $LASTEXITCODE." } } -switch ($Backend) { - 'process-t1' { - Invoke-ProcessContainerTests - } - 'process-t3' { - Invoke-ProcessContainerTests - } - 'isolation-session' { - Invoke-TestScript -Path (Join-Path $scriptRoot 'run_isolation_session_tests.ps1') -Arguments @{ - WxcExePath = $wxc +try { + switch ($Backend) { + 'process-t1' { + Invoke-ProcessContainerTests } - } - 'windows-sandbox' { - Invoke-TestScript -Path (Join-Path $scriptRoot 'run_windows_sandbox_one_shot_tests.ps1') -Arguments @{ - BinDir = $binaryDirectoryPath + 'process-t3' { + Invoke-ProcessContainerTests } - } - 'wslc' { - # The current WSLC helper hardcodes the x64 target when locating assets. - if ($Architecture -ne 'x64') { - throw 'The existing WSLC test harness is not architecture-portable yet.' + 'isolation-session' { + Invoke-TestScript -Path (Join-Path $scriptRoot 'run_isolation_session_tests.ps1') -Arguments @{ + WxcExePath = $wxc + } } - Invoke-TestScript -Path (Join-Path $scriptRoot 'run_wslc_all_tests.ps1') -Arguments @{ - WxcExecPath = $wxc + 'windows-sandbox' { + Invoke-TestScript -Path (Join-Path $scriptRoot 'run_windows_sandbox_one_shot_tests.ps1') -Arguments @{ + BinDir = $binaryDirectoryPath + } } - } - 'microvm' { - Invoke-TestScript -Path (Join-Path $scriptRoot 'run_microvm_tests.ps1') -Arguments @{ - BinDir = $binaryDirectoryPath + 'wslc' { + # The current WSLC helper hardcodes the x64 target when locating assets. + if ($Architecture -ne 'x64') { + throw 'The existing WSLC test harness is not architecture-portable yet.' + } + Invoke-TestScript -Path (Join-Path $scriptRoot 'run_wslc_all_tests.ps1') -Arguments @{ + WxcExecPath = $wxc + } + } + 'microvm' { + Invoke-TestScript -Path (Join-Path $scriptRoot 'run_microvm_tests.ps1') -Arguments @{ + BinDir = $binaryDirectoryPath + } + } + 'hyperlight' { + # Keep unwired backends explicit so accidental activation fails loudly. + throw 'The Hyperlight CI backend is not wired to an existing test entry point yet.' } } - 'hyperlight' { - # Keep unwired backends explicit so accidental activation fails loudly. - throw 'The Hyperlight CI backend is not wired to an existing test entry point yet.' - } +} finally { + # Also runs when a suite throws, which is when these logs matter most. + Copy-TempArtifacts } From a2bf450ff6581a9a5d8fda84ad21c892544b2f73 Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Tue, 18 Aug 2026 14:16:05 -0700 Subject: [PATCH 10/16] removed math in yml --- .../workflows/Validation.Tests.Matrix.Job.yml | 35 ++++++++++++++++--- docs/ci-validation-infrastructure.md | 14 ++++---- scripts/ci/resolve-validation-test-matrix.mjs | 19 +++++++--- 3 files changed, 52 insertions(+), 16 deletions(-) diff --git a/.github/workflows/Validation.Tests.Matrix.Job.yml b/.github/workflows/Validation.Tests.Matrix.Job.yml index dc6726a57..bcd424866 100644 --- a/.github/workflows/Validation.Tests.Matrix.Job.yml +++ b/.github/workflows/Validation.Tests.Matrix.Job.yml @@ -38,8 +38,9 @@ jobs: fail-fast: false matrix: ${{ fromJSON(needs.resolve.outputs.windows) }} runs-on: [ self-hosted, "1ES.Pool=${{ matrix.pool }}", "JobId=mxc-e2e-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" ] - # Includes any startup stagger, so a delayed entry keeps the full test budget. - timeout-minutes: ${{ 60 + (matrix.startup_delay_minutes || 0) }} + # Resolved per entry: the test budget plus any startup stagger, so a + # delayed entry keeps its full budget. Workflow expressions cannot add. + timeout-minutes: 180 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 @@ -56,7 +57,7 @@ jobs: # Spreads network-heavy setup across the pool's shared egress address # instead of issuing every job's downloads at once. Configured by # backendDelayedStart in scripts/ci/validation-test-matrix.json. - $minutes = ${{ matrix.startup_delay_minutes || 0 }} + $minutes = ${{ matrix.startup_delay_minutes }} Write-Host "Waiting $minutes minute(s) before downloading prerequisites." Start-Sleep -Seconds ($minutes * 60) @@ -110,7 +111,7 @@ jobs: fail-fast: false matrix: ${{ fromJSON(needs.resolve.outputs.linux) }} runs-on: [ self-hosted, "1ES.Pool=${{ matrix.pool }}", "JobId=mxc-e2e-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" ] - timeout-minutes: 60 + timeout-minutes: 180 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 @@ -120,6 +121,18 @@ jobs: name: ${{ matrix.artifact }} path: artifacts/bin + - name: Delayed start + if: matrix.startup_delay_minutes > 0 + shell: bash + run: | + # Spreads network-heavy setup across the pool's shared egress address + # instead of issuing every job's downloads at once. Configured by + # backendDelayedStart in scripts/ci/validation-test-matrix.json. + set -euo pipefail + minutes=${{ matrix.startup_delay_minutes }} + echo "Waiting $minutes minute(s) before downloading prerequisites." + sleep $(( minutes * 60 )) + - name: Prepare backend prerequisites timeout-minutes: 15 shell: bash @@ -161,7 +174,7 @@ jobs: fail-fast: false matrix: ${{ fromJSON(needs.resolve.outputs.macos) }} runs-on: ${{ matrix.runner }} - timeout-minutes: 60 + timeout-minutes: 180 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 @@ -171,6 +184,18 @@ jobs: name: ${{ matrix.artifact }} path: artifacts/bin + - name: Delayed start + if: matrix.startup_delay_minutes > 0 + shell: bash + run: | + # Spreads network-heavy setup across the pool's shared egress address + # instead of issuing every job's downloads at once. Configured by + # backendDelayedStart in scripts/ci/validation-test-matrix.json. + set -euo pipefail + minutes=${{ matrix.startup_delay_minutes }} + echo "Waiting $minutes minute(s) before starting tests." + sleep $(( minutes * 60 )) + - name: Run backend tests timeout-minutes: 60 shell: bash diff --git a/docs/ci-validation-infrastructure.md b/docs/ci-validation-infrastructure.md index c1baae81f..366b939a9 100644 --- a/docs/ci-validation-infrastructure.md +++ b/docs/ci-validation-infrastructure.md @@ -75,9 +75,9 @@ Build artifacts are kept for 1 day — they exist only to feed these jobs. | `macos` | GitHub-hosted `${{ matrix.runner }}` | Download artifact → `chmod +x` → `run_ci_backend_tests.sh `. No host-prep step. | Per-job display name: `, , ` (macOS omits -the architecture). Job timeout 60 min; host prep 15 min; the test step 45 min -(60 on macOS), so a hung backend fails while the log is still useful. On failure -or cancellation the job uploads `mxc-ci.log` plus the Process Container log +the architecture). Job timeout 180 min; host prep 15 min; the test step 45 min +(60 on macOS), so a hung backend fails while the log is still useful. The job +uploads `mxc-ci.log` plus the Process Container log directories as `logs-----`, kept 7 days. ## The catalog @@ -177,10 +177,10 @@ limiting from public registries and stalled downloads. independently per backend, following the resolved job order. With the entry above, four WSLC jobs start at 0, 5, 10, and 15 minutes. -The resolver emits the offset as `startup_delay_minutes` on each affected -matrix entry. The job sleeps that long before its first network step, and its -`timeout-minutes` grows by the same amount so a delayed entry keeps the full -test budget. Entries for other backends carry no such field and never wait. +The resolver emits the offset as `startup_delay_minutes` on every matrix entry +(`0` where no stagger applies). The job sleeps that long before its first +network step; the job timeout is a fixed 180 minutes, wide enough to absorb any +configured wait. Omit the section (or leave it empty) to have every job start as soon as its runner is ready. A backend id that no plan schedules is accepted and simply diff --git a/scripts/ci/resolve-validation-test-matrix.mjs b/scripts/ci/resolve-validation-test-matrix.mjs index 79cae85b7..4383eb680 100644 --- a/scripts/ci/resolve-validation-test-matrix.mjs +++ b/scripts/ci/resolve-validation-test-matrix.mjs @@ -233,6 +233,7 @@ export function resolvePlan(catalog, plan) { // assignment follows the emitted order and stays reproducible. function applyDelayedStart(matrices, delays) { if (!Array.isArray(delays) || delays.length === 0) { + setDefaultDelays(matrices); return; } @@ -243,11 +244,21 @@ function applyDelayedStart(matrices, delays) { for (const entry of matrices[family]) { const step = stepMinutes.get(entry.backend); if (step === undefined) { - continue; + entry.startup_delay_minutes = 0; + } else { + const position = scheduled.get(entry.backend) ?? 0; + entry.startup_delay_minutes = position * step; + scheduled.set(entry.backend, position + 1); } - const position = scheduled.get(entry.backend) ?? 0; - entry.startup_delay_minutes = position * step; - scheduled.set(entry.backend, position + 1); + } + } +} + +// Every entry carries the field so the workflow's step condition is uniform. +function setDefaultDelays(matrices) { + for (const family of FAMILIES) { + for (const entry of matrices[family]) { + entry.startup_delay_minutes = 0; } } } From 87fc7bbb99dfe9d2468fccdd39fc1fc0e1616a57 Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Tue, 18 Aug 2026 15:10:34 -0700 Subject: [PATCH 11/16] Simpler way to collect logs --- .../workflows/Validation.Tests.Matrix.Job.yml | 23 ++-- docs/ci-validation-infrastructure.md | 78 ++++++++---- tests/scripts/run_ci_backend_tests.ps1 | 117 +++++++----------- 3 files changed, 111 insertions(+), 107 deletions(-) diff --git a/.github/workflows/Validation.Tests.Matrix.Job.yml b/.github/workflows/Validation.Tests.Matrix.Job.yml index bcd424866..79e4a2dd1 100644 --- a/.github/workflows/Validation.Tests.Matrix.Job.yml +++ b/.github/workflows/Validation.Tests.Matrix.Job.yml @@ -89,17 +89,20 @@ jobs: uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a with: name: logs-${{ inputs.plan }}-${{ matrix.os }}-${{ matrix.architecture }}-${{ matrix.backend }}-${{ github.run_attempt }} + # The dispatcher points TEMP here, so suite logs and scratch trees are + # collected without enumerating each one. Exclude bulk binaries that + # tooling also drops in TEMP; they dwarf the logs and aid no triage. path: | - ${{ runner.temp }}/mxc-ci.log - ${{ runner.temp }}/mxc-wpc-tests - ${{ runner.temp }}/mxc-t3-workloads - ${{ runner.temp }}/mxc_concurrent_oneshot - ${{ runner.temp }}/mxc_etw_test - ${{ runner.temp }}/wxc-wsb - ${{ runner.temp }}/wxc-sandbox-rendezvous - ${{ runner.temp }}/mxc-diagnostics-* - ${{ runner.temp }}/WinProcessContainer-Tests.results.* - ${{ runner.temp }}/WinProcessContainer-Tests.cargo.log + ${{ runner.temp }} + !${{ runner.temp }}/**/*.vhd + !${{ runner.temp }}/**/*.vhdx + !${{ runner.temp }}/**/*.tar + !${{ runner.temp }}/**/*.msi + !${{ runner.temp }}/**/*.msix + !${{ runner.temp }}/**/*.appx + !${{ runner.temp }}/**/*.zip + !${{ runner.temp }}/**/*.exe + !${{ runner.temp }}/**/*.dll if-no-files-found: ignore retention-days: 7 diff --git a/docs/ci-validation-infrastructure.md b/docs/ci-validation-infrastructure.md index 366b939a9..0d44ec532 100644 --- a/docs/ci-validation-infrastructure.md +++ b/docs/ci-validation-infrastructure.md @@ -31,7 +31,7 @@ the individual local test scripts are documented in | `scripts/ci/resolve-validation-test-matrix.mjs` | Matrix validator + plan expander. Emits the GitHub Actions matrices. | | `scripts/ci/prepare-windows-host.ps1` | Per-backend Windows host preparation / prerequisite assertions. | | `scripts/ci/prepare-linux-host.sh` | Per-backend Linux package install and service startup (distro-aware). | -| `tests/scripts/run_ci_backend_tests.ps1` | Windows dispatcher: backend id → existing backend suite. | +| `tests/scripts/run_ci_backend_tests.ps1` | Windows dispatcher: backend id → existing backend suite. Also points `TEMP` at `$RUNNER_TEMP` so logs get collected. | | `tests/scripts/run_ci_backend_tests.sh` | Linux/macOS dispatcher: backend id → existing backend suite. | ### Flow @@ -76,9 +76,8 @@ Build artifacts are kept for 1 day — they exist only to feed these jobs. Per-job display name: `, , ` (macOS omits the architecture). Job timeout 180 min; host prep 15 min; the test step 45 min -(60 on macOS), so a hung backend fails while the log is still useful. The job -uploads `mxc-ci.log` plus the Process Container log -directories as `logs-----`, kept 7 days. +(60 on macOS), so a hung backend fails while the log is still useful. Logs are +uploaded either way — see [Log collection](#log-collection). ## The catalog @@ -168,28 +167,27 @@ them all begin at once: ] ``` -Every runner in a pool shares one egress address. A backend whose setup -downloads a large runtime or several container images therefore concentrates -that traffic into a burst when its jobs start together, which draws rate -limiting from public registries and stalled downloads. +Every runner in a pool shares one egress address, so a backend whose setup +pulls down a large runtime or several container images concentrates all that +traffic into a burst the moment its jobs start together. Public registries +answer with rate limiting and stalled downloads. -`minutes` is the gap between consecutive jobs of that backend, counted -independently per backend, following the resolved job order. With the entry -above, four WSLC jobs start at 0, 5, 10, and 15 minutes. +`minutes` is the gap between consecutive jobs of that backend, counted per +backend and following the resolved job order. With the entry above, four WSLC +jobs start at 0, 5, 10, and 15 minutes. -The resolver emits the offset as `startup_delay_minutes` on every matrix entry -(`0` where no stagger applies). The job sleeps that long before its first -network step; the job timeout is a fixed 180 minutes, wide enough to absorb any -configured wait. +The resolver puts the offset on every matrix entry as +`startup_delay_minutes` — `0` where no stagger applies — and the job sleeps +that long before its first network step. Job timeout is a flat 180 minutes, +with plenty of room for any wait you'd reasonably configure. -Omit the section (or leave it empty) to have every job start as soon as its -runner is ready. A backend id that no plan schedules is accepted and simply -never applies. +Leave the section out (or empty) and every job starts as soon as its runner is +ready. A backend id that no plan schedules is accepted; it just never applies. -Note this holds the runner while it sleeps: Actions cannot defer allocating a -matrix job, so the wait happens inside the job. Keep the value only as large as -the contention requires. It spreads simultaneous load — it does not help a -single download that stalls on its own. +Do keep in mind that the runner is held while it sleeps — Actions can't defer +allocating a matrix job, so the wait has to happen inside it. Use no more than +the contention calls for. This spreads simultaneous load and nothing else; a +single download that stalls on its own is unaffected. ## Backend status @@ -238,13 +236,33 @@ message instead of surfacing later as an opaque backend error. - `bubblewrap` — installs `bwrap` (apt/dnf/yum/microdnf) and relaxes `kernel.apparmor_restrict_unprivileged_userns` (ephemeral CI hosts only). - `lxc` — installs the LXC stack, reloads the AppArmor profile, starts and waits - for `lxcbr0`, and prints network diagnostics. On RHEL-likes it first needs - EPEL, because Red Hat dropped LXC after RHEL 7 and ships no replacement. + for `lxcbr0`, enables bridge netfilter, and makes sure the bridge's NAT rule + is in place. On RHEL-likes it needs EPEL first, because Red Hat dropped LXC + after RHEL 7 and ships no replacement. - `microvm` — asserts the NanVix payload exists. - `hyperlight` — no-op. macOS has no preparation step. +## Log collection + +Every job uploads its logs whether it passed or failed, as +`logs-----`, kept 7 days. + +The catch is that `$env:TEMP` is not `$RUNNER_TEMP`. The Windows suites write their +scratch trees, transcripts, and results files under the user's temp directory +(`C:\Users\\AppData\Local\Temp`), but `upload-artifact` reads +`${{ runner.temp }}` (`C:\a\_work\_temp`). Anything left in the former is simply +never collected, which is why the artifact used to arrive nearly empty. + +So `run_ci_backend_tests.ps1` points `TEMP` and `TMP` at `$RUNNER_TEMP` before +it dispatches. Parameter defaults, `[System.IO.Path]::GetTempPath()`, and child +processes all read those variables, so everything temp-rooted lands in the +upload directory without CI having to know a single filename. + +Linux and macOS need none of this — those suites log to stdout, and the run +step tees that into `$RUNNER_TEMP/mxc-ci.log`. + ## Runbook Always finish with a local resolve, which runs the full catalog validation: @@ -319,9 +337,15 @@ by testing it ahead of time. ### Stagger a backend's job starts Add or edit its `backendDelayedStart` entry in the catalog, then resolve -locally to confirm the offsets. Reach for this when a backend's setup is -network-heavy enough that concurrent jobs hit rate limits or stalled -downloads; remove the entry once that pressure is gone. +locally to confirm the offsets. Worth reaching for when a backend's setup is +network-heavy enough that concurrent jobs run into rate limits or stalled +downloads — and worth removing again once that pressure is gone. + +### Collect a new log file + +Have the suite write it under `$env:TEMP`. The dispatcher redirects that to the +upload directory, so nothing in CI needs to change. See +[Log collection](#log-collection). ### Change the schedule diff --git a/tests/scripts/run_ci_backend_tests.ps1 b/tests/scripts/run_ci_backend_tests.ps1 index 572b2618c..3770f6390 100644 --- a/tests/scripts/run_ci_backend_tests.ps1 +++ b/tests/scripts/run_ci_backend_tests.ps1 @@ -42,45 +42,25 @@ function Assert-File { } } -# The suites write their logs, configs, and transcripts under $env:TEMP, which -# is not the directory CI uploads from. Several also refuse to run with those -# paths pointed elsewhere, so copy the artifacts across afterwards instead of -# redirecting them. Best-effort by design: losing a log must not turn a passing -# run red, and a failing run keeps its original result. -function Copy-TempArtifacts { - if (-not $env:RUNNER_TEMP -or -not $env:TEMP) { +# The suites, and MXC itself, write logs and scratch trees under $env:TEMP, +# which is not the directory CI uploads from. Rather than enumerate every +# artifact and copy it afterwards, point TEMP at the upload directory for the +# duration of the run: parameter defaults, .NET GetTempPath(), and child +# processes all follow it, so anything temp-rooted lands where CI collects it. +# +# The suites' scratch-root guards compare against $env:TEMP, so they stay +# satisfied — this redirects what TEMP means rather than pointing a path +# outside it. +function Redirect-TempToRunnerTemp { + if (-not $env:RUNNER_TEMP) { return } - - $names = @( - 'mxc-wpc-tests', - 'mxc-t3-workloads', - 'mxc_concurrent_oneshot', - 'mxc_etw_test', - 'wxc-wsb', - 'wxc-sandbox-rendezvous', - 'WinProcessContainer-Tests.results.txt', - 'WinProcessContainer-Tests.results.json', - 'WinProcessContainer-Tests.cargo.log' - ) - - foreach ($name in $names) { - $source = Join-Path $env:TEMP $name - if (-not (Test-Path -LiteralPath $source)) { - continue - } - Copy-Item -Recurse -Force -LiteralPath $source ` - -Destination (Join-Path $env:RUNNER_TEMP $name) ` - -ErrorAction SilentlyContinue + if (-not (Test-Path -LiteralPath $env:RUNNER_TEMP)) { + New-Item -ItemType Directory -Force -Path $env:RUNNER_TEMP | Out-Null } - - # MXC's own diagnostic dumps are timestamped per run, so match the family. - Get-ChildItem -Path $env:TEMP -Filter 'mxc-diagnostics-*' -ErrorAction SilentlyContinue | - ForEach-Object { - Copy-Item -Recurse -Force -LiteralPath $_.FullName ` - -Destination (Join-Path $env:RUNNER_TEMP $_.Name) ` - -ErrorAction SilentlyContinue - } + $env:TEMP = $env:RUNNER_TEMP + $env:TMP = $env:RUNNER_TEMP + Write-Host "Redirected TEMP to $env:RUNNER_TEMP so test logs are collected." } function Invoke-TestScript { @@ -146,44 +126,41 @@ function Invoke-ProcessContainerTests { } } -try { - switch ($Backend) { - 'process-t1' { - Invoke-ProcessContainerTests - } - 'process-t3' { - Invoke-ProcessContainerTests - } - 'isolation-session' { - Invoke-TestScript -Path (Join-Path $scriptRoot 'run_isolation_session_tests.ps1') -Arguments @{ - WxcExePath = $wxc - } +Redirect-TempToRunnerTemp + +switch ($Backend) { + 'process-t1' { + Invoke-ProcessContainerTests + } + 'process-t3' { + Invoke-ProcessContainerTests + } + 'isolation-session' { + Invoke-TestScript -Path (Join-Path $scriptRoot 'run_isolation_session_tests.ps1') -Arguments @{ + WxcExePath = $wxc } - 'windows-sandbox' { - Invoke-TestScript -Path (Join-Path $scriptRoot 'run_windows_sandbox_one_shot_tests.ps1') -Arguments @{ - BinDir = $binaryDirectoryPath - } + } + 'windows-sandbox' { + Invoke-TestScript -Path (Join-Path $scriptRoot 'run_windows_sandbox_one_shot_tests.ps1') -Arguments @{ + BinDir = $binaryDirectoryPath } - 'wslc' { - # The current WSLC helper hardcodes the x64 target when locating assets. - if ($Architecture -ne 'x64') { - throw 'The existing WSLC test harness is not architecture-portable yet.' - } - Invoke-TestScript -Path (Join-Path $scriptRoot 'run_wslc_all_tests.ps1') -Arguments @{ - WxcExecPath = $wxc - } + } + 'wslc' { + # The current WSLC helper hardcodes the x64 target when locating assets. + if ($Architecture -ne 'x64') { + throw 'The existing WSLC test harness is not architecture-portable yet.' } - 'microvm' { - Invoke-TestScript -Path (Join-Path $scriptRoot 'run_microvm_tests.ps1') -Arguments @{ - BinDir = $binaryDirectoryPath - } + Invoke-TestScript -Path (Join-Path $scriptRoot 'run_wslc_all_tests.ps1') -Arguments @{ + WxcExecPath = $wxc } - 'hyperlight' { - # Keep unwired backends explicit so accidental activation fails loudly. - throw 'The Hyperlight CI backend is not wired to an existing test entry point yet.' + } + 'microvm' { + Invoke-TestScript -Path (Join-Path $scriptRoot 'run_microvm_tests.ps1') -Arguments @{ + BinDir = $binaryDirectoryPath } } -} finally { - # Also runs when a suite throws, which is when these logs matter most. - Copy-TempArtifacts + 'hyperlight' { + # Keep unwired backends explicit so accidental activation fails loudly. + throw 'The Hyperlight CI backend is not wired to an existing test entry point yet.' + } } From 4902c428f982a375bb4463872edf3cccd648c65e Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Tue, 18 Aug 2026 15:41:58 -0700 Subject: [PATCH 12/16] reducing time between wslc jobs to 45 seconds to avoid idle agent from getting removed --- .../workflows/Validation.Tests.Matrix.Job.yml | 24 +++++++++---------- docs/ci-validation-infrastructure.md | 8 +++---- scripts/ci/resolve-validation-test-matrix.mjs | 16 ++++++------- scripts/ci/validation-test-matrix.json | 2 +- 4 files changed, 25 insertions(+), 25 deletions(-) diff --git a/.github/workflows/Validation.Tests.Matrix.Job.yml b/.github/workflows/Validation.Tests.Matrix.Job.yml index 79e4a2dd1..5fcaf425c 100644 --- a/.github/workflows/Validation.Tests.Matrix.Job.yml +++ b/.github/workflows/Validation.Tests.Matrix.Job.yml @@ -51,15 +51,15 @@ jobs: path: artifacts/bin - name: Delayed start - if: matrix.startup_delay_minutes > 0 + if: matrix.startup_delay_seconds > 0 shell: pwsh run: | # Spreads network-heavy setup across the pool's shared egress address # instead of issuing every job's downloads at once. Configured by # backendDelayedStart in scripts/ci/validation-test-matrix.json. - $minutes = ${{ matrix.startup_delay_minutes }} - Write-Host "Waiting $minutes minute(s) before downloading prerequisites." - Start-Sleep -Seconds ($minutes * 60) + $seconds = ${{ matrix.startup_delay_seconds }} + Write-Host "Waiting $seconds second(s) before downloading prerequisites." + Start-Sleep -Seconds $seconds - name: Prepare backend prerequisites timeout-minutes: 15 @@ -125,16 +125,16 @@ jobs: path: artifacts/bin - name: Delayed start - if: matrix.startup_delay_minutes > 0 + if: matrix.startup_delay_seconds > 0 shell: bash run: | # Spreads network-heavy setup across the pool's shared egress address # instead of issuing every job's downloads at once. Configured by # backendDelayedStart in scripts/ci/validation-test-matrix.json. set -euo pipefail - minutes=${{ matrix.startup_delay_minutes }} - echo "Waiting $minutes minute(s) before downloading prerequisites." - sleep $(( minutes * 60 )) + seconds=${{ matrix.startup_delay_seconds }} + echo "Waiting $seconds second(s) before downloading prerequisites." + sleep "$seconds" - name: Prepare backend prerequisites timeout-minutes: 15 @@ -188,16 +188,16 @@ jobs: path: artifacts/bin - name: Delayed start - if: matrix.startup_delay_minutes > 0 + if: matrix.startup_delay_seconds > 0 shell: bash run: | # Spreads network-heavy setup across the pool's shared egress address # instead of issuing every job's downloads at once. Configured by # backendDelayedStart in scripts/ci/validation-test-matrix.json. set -euo pipefail - minutes=${{ matrix.startup_delay_minutes }} - echo "Waiting $minutes minute(s) before starting tests." - sleep $(( minutes * 60 )) + seconds=${{ matrix.startup_delay_seconds }} + echo "Waiting $seconds second(s) before starting tests." + sleep "$seconds" - name: Run backend tests timeout-minutes: 60 diff --git a/docs/ci-validation-infrastructure.md b/docs/ci-validation-infrastructure.md index 0d44ec532..856cff73f 100644 --- a/docs/ci-validation-infrastructure.md +++ b/docs/ci-validation-infrastructure.md @@ -163,7 +163,7 @@ them all begin at once: ```json "backendDelayedStart": [ - { "backend": "wslc", "minutes": 5 } + { "backend": "wslc", "seconds": 300 } ] ``` @@ -172,12 +172,12 @@ pulls down a large runtime or several container images concentrates all that traffic into a burst the moment its jobs start together. Public registries answer with rate limiting and stalled downloads. -`minutes` is the gap between consecutive jobs of that backend, counted per +`seconds` is the gap between consecutive jobs of that backend, counted per backend and following the resolved job order. With the entry above, four WSLC -jobs start at 0, 5, 10, and 15 minutes. +jobs start at 0, 300, 600, and 900 seconds. The resolver puts the offset on every matrix entry as -`startup_delay_minutes` — `0` where no stagger applies — and the job sleeps +`startup_delay_seconds` — `0` where no stagger applies — and the job sleeps that long before its first network step. Job timeout is a flat 180 minutes, with plenty of room for any wait you'd reasonably configure. diff --git a/scripts/ci/resolve-validation-test-matrix.mjs b/scripts/ci/resolve-validation-test-matrix.mjs index 4383eb680..565fcec1d 100644 --- a/scripts/ci/resolve-validation-test-matrix.mjs +++ b/scripts/ci/resolve-validation-test-matrix.mjs @@ -158,9 +158,9 @@ function validateBackendDelayedStart(catalog) { } seen.add(entry.backend); - if (!Number.isInteger(entry.minutes) || entry.minutes < 0) { + if (!Number.isInteger(entry.seconds) || entry.seconds < 0) { throw new Error( - `backendDelayedStart ${entry.backend} minutes must be a non-negative integer` + `backendDelayedStart ${entry.backend} seconds must be a non-negative integer` ); } } @@ -228,7 +228,7 @@ export function resolvePlan(catalog, plan) { // pool shares one egress address, so simultaneous starts concentrate that // traffic into a burst that draws rate limiting and stalled downloads. // -// The entry's minutes value is the gap between consecutive jobs of that +// The entry's seconds value is the gap between consecutive jobs of that // backend, counted independently per backend. Applied after sorting so the // assignment follows the emitted order and stays reproducible. function applyDelayedStart(matrices, delays) { @@ -237,17 +237,17 @@ function applyDelayedStart(matrices, delays) { return; } - const stepMinutes = new Map(delays.map(entry => [entry.backend, entry.minutes])); + const stepSeconds = new Map(delays.map(entry => [entry.backend, entry.seconds])); const scheduled = new Map(); for (const family of FAMILIES) { for (const entry of matrices[family]) { - const step = stepMinutes.get(entry.backend); + const step = stepSeconds.get(entry.backend); if (step === undefined) { - entry.startup_delay_minutes = 0; + entry.startup_delay_seconds = 0; } else { const position = scheduled.get(entry.backend) ?? 0; - entry.startup_delay_minutes = position * step; + entry.startup_delay_seconds = position * step; scheduled.set(entry.backend, position + 1); } } @@ -258,7 +258,7 @@ function applyDelayedStart(matrices, delays) { function setDefaultDelays(matrices) { for (const family of FAMILIES) { for (const entry of matrices[family]) { - entry.startup_delay_minutes = 0; + entry.startup_delay_seconds = 0; } } } diff --git a/scripts/ci/validation-test-matrix.json b/scripts/ci/validation-test-matrix.json index c1bbd3816..1a013f30a 100644 --- a/scripts/ci/validation-test-matrix.json +++ b/scripts/ci/validation-test-matrix.json @@ -328,7 +328,7 @@ "backendDelayedStart": [ { "backend": "wslc", - "minutes": 2 + "seconds": 45 } ], "triggers": { From ecff4419b38de20d4add1ce0fdf614191d4ddcef Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Tue, 18 Aug 2026 16:04:39 -0700 Subject: [PATCH 13/16] Making PR ready --- .../Validation.Infrastructure.Tests.yml | 36 ------------------- scripts/ci/validation-test-matrix.json | 35 +----------------- 2 files changed, 1 insertion(+), 70 deletions(-) delete mode 100644 .github/workflows/Validation.Infrastructure.Tests.yml diff --git a/.github/workflows/Validation.Infrastructure.Tests.yml b/.github/workflows/Validation.Infrastructure.Tests.yml deleted file mode 100644 index 1785d2515..000000000 --- a/.github/workflows/Validation.Infrastructure.Tests.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: Validation Infrastructure Testing - -on: - push: - branches: - - user/emichlin/validation-infra-2 - -concurrency: - group: validation-infra-pr-tests-${{ github.ref }} - cancel-in-progress: true - -permissions: - actions: read - contents: read - -jobs: - dependency-feed-check: - uses: ./.github/workflows/Dependency.Feed.Check.Job.yml - - windows: - needs: dependency-feed-check - uses: ./.github/workflows/Build.Windows.Job.yml - - linux: - needs: dependency-feed-check - uses: ./.github/workflows/Build.Linux.Job.yml - - macos: - needs: dependency-feed-check - uses: ./.github/workflows/Build.MacOS.Job.yml - - test: - needs: [windows, linux, macos] - uses: ./.github/workflows/Validation.Tests.Matrix.Job.yml - with: - plan: enabled \ No newline at end of file diff --git a/scripts/ci/validation-test-matrix.json b/scripts/ci/validation-test-matrix.json index 1a013f30a..f0a2d46c3 100644 --- a/scripts/ci/validation-test-matrix.json +++ b/scripts/ci/validation-test-matrix.json @@ -393,39 +393,6 @@ } ], "weekly": [], - "enabled": [ - { - "os": "ubuntu-26.04", - "backends": [ - "bubblewrap", - "lxc" - ] - }, - { - "os": "windows-prerelease-process-container", - "backends": [ - "wslc", - "process-t1" - ] - }, - { - "os": "windows-25h2", - "backends": [ - "wslc" - ] - }, - { - "os": "windows-24h2", - "backends": [ - "wslc" - ] - }, - { - "os": "windows-23h2", - "backends": [ - "wslc" - ] - } - ] + "enabled": [] } } From d6ebe7c9cd914168e0add6b5996b982bdf54b624 Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Tue, 18 Aug 2026 16:12:58 -0700 Subject: [PATCH 14/16] Document backendDelayedStart and the CI TEMP redirect Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/copilot-instructions.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 575f4e7f6..bddc4d518 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -72,9 +72,10 @@ workflows) before calling the matrix job. - `scripts/ci/validation-test-matrix.json` is the catalog: `platforms` (each with per-architecture target/artifact/1ES pool and the backends that platform - supports) and `triggers` (which OS/backend pairs each plan runs). The - `triggers` keys *are* the plan list — the resolver reads them at run time, so - adding a plan needs no script change. + supports), `triggers` (which OS/backend pairs each plan runs), and the + optional `backendDelayedStart` (per-backend job-start stagger, in seconds). + The `triggers` keys *are* the plan list — the resolver reads them at run time, + so adding a plan needs no script change. - `scripts/ci/resolve-validation-test-matrix.mjs` validates that catalog and expands a plan (currently `pr`, `nightly`, `weekly`, `enabled`) into GitHub Actions matrices. It rejects an invalid catalog before any specialized test @@ -95,7 +96,9 @@ the matrix `backend` id to the repository's existing backend suite. Ids that share a suite get their own case (`process-t1` and `process-t3` both run `WinProcessContainer-Tests.ps1`, which derives the tier it expects from the host's own `--probe`). A backend with no wired suite fails loudly rather than -reporting a false success. +reporting a false success. The Windows dispatcher points `TEMP` at +`$RUNNER_TEMP` before running a suite, so anything a test writes to the temp +directory is picked up by the job's log upload without per-file CI wiring. ### Individual components From 2d6df7ea3b427201d5048f346e624dc9e68a2291 Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Tue, 18 Aug 2026 16:33:19 -0700 Subject: [PATCH 15/16] removing old comments --- .github/workflows/Validation.Tests.Matrix.Job.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/Validation.Tests.Matrix.Job.yml b/.github/workflows/Validation.Tests.Matrix.Job.yml index 5fcaf425c..817ebf880 100644 --- a/.github/workflows/Validation.Tests.Matrix.Job.yml +++ b/.github/workflows/Validation.Tests.Matrix.Job.yml @@ -38,8 +38,6 @@ jobs: fail-fast: false matrix: ${{ fromJSON(needs.resolve.outputs.windows) }} runs-on: [ self-hosted, "1ES.Pool=${{ matrix.pool }}", "JobId=mxc-e2e-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" ] - # Resolved per entry: the test budget plus any startup stagger, so a - # delayed entry keeps its full budget. Workflow expressions cannot add. timeout-minutes: 180 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 @@ -72,8 +70,6 @@ jobs: if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - name: Run backend tests - # A backend that hangs (rather than failing) would otherwise burn the - # full job timeout. Fail fast enough to keep the log useful. timeout-minutes: 45 shell: pwsh run: | From 233dc198c5db5ddc79a5ef49c62bd110a45ebe2e Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Tue, 18 Aug 2026 16:57:54 -0700 Subject: [PATCH 16/16] updating backend status in docs --- docs/ci-validation-infrastructure.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/ci-validation-infrastructure.md b/docs/ci-validation-infrastructure.md index 856cff73f..821c5b67b 100644 --- a/docs/ci-validation-infrastructure.md +++ b/docs/ci-validation-infrastructure.md @@ -199,8 +199,8 @@ get fixed or wired. | Process T1 | ✅ Good | Prerelease Windows only. Remaining failures are genuine MXC bugs or harness limitations. | | Process T3 | ✅ Good | Non-prerelease Windows builds only, until the testing suite is updated. | | Bubblewrap | ✅ Good | | -| LXC | ⚠️ Mostly good | Some networking tests fail on distros other than Ubuntu 24.04; host-vs-MXC cause not yet isolated. | -| WSLC | ⚠️ Mostly good | Can hit a download rate limit while updating WSL / pulling container images. Planned fix: split into several jobs spaced ~15 min apart. | +| LXC | ✅ Good | Some networking tests fail on distros other than Ubuntu 24.04; seems to be an issue with MXC. | +| WSLC | ✅ Good | Might have to retry hung jobs - this is an issue with overzealous agent reclaiming. | | IsolationSession | ⚠️ Blocked | `Feature_AgentSessionsBaseSupport` is not enabled on the pool image yet. | | Windows Sandbox | ⛔ Not scheduled | Dispatcher case is wired; no trigger entry yet. | | MicroVM | ⛔ Not working | Windows cold and warm starts hang; no Linux suite. The artifact payload is currently commented out in the build jobs. |