From 33e16b5d530c1578861066dcc8bfcd680a25447a Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Mon, 2 Nov 2026 12:03:31 +0000 Subject: [PATCH 01/37] Automate hosted LocalBox preparation and readiness checks Monitor nested deployment with Console token refresh, run default guest preparation and Pester tests, pass the event Entra group, and retain private diagnostics. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../labautomation/global-event-deploy.ps1 | 4 + .../labautomation/localbox-automation.ps1 | 285 +++++++++++++ .../resources/hosted-events/readme.md | 108 ++++- .../resources/invoke-localbox-automation.ps1 | 193 +++++++++ .../resources/localbox/readme.md | 2 +- .../resources/tests/global-localbox.tests.ps1 | 13 +- .../tests/localbox-automation.tests.ps1 | 382 ++++++++++++++++++ .../resources/tests/readme.md | 16 + 8 files changed, 986 insertions(+), 17 deletions(-) create mode 100644 03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation.ps1 create mode 100644 03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/invoke-localbox-automation.ps1 create mode 100644 03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-automation.tests.ps1 diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/global-event-deploy.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/global-event-deploy.ps1 index 8119976a5..fac2ba263 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/global-event-deploy.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/global-event-deploy.ps1 @@ -22,6 +22,7 @@ param( $ErrorActionPreference = 'Stop' . (Join-Path $PSScriptRoot 'localbox-credentials.ps1') +. (Join-Path $PSScriptRoot 'localbox-automation.ps1') . (Join-Path $PSScriptRoot 'hosted-tag-policy.ps1') . (Join-Path $PSScriptRoot 'hosted-defender.ps1') @@ -204,6 +205,9 @@ foreach ($participantObjectId in $participants) { } Wait-LocalBoxDeployment -ResourceGroupName $localBoxResourceGroupName -DeploymentName $localBoxDeployment.DeploymentName +$groupId = ($groupCredentials.HackboxCredential | Where-Object name -eq 'Lab Group ObjectId').value +$automation = Invoke-LocalBoxHostedAutomation -SubscriptionId $SubscriptionId -ResourceGroupName $localBoxResourceGroupName -AksAdminGroupObjectId $groupId +Write-Host "LocalBox preparation and health checks completed. Private results: $($automation.ResultsUri)" $groupCredentials @{ HackboxCredential = @{ diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation.ps1 new file mode 100644 index 000000000..d96435a46 --- /dev/null +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation.ps1 @@ -0,0 +1,285 @@ +function Invoke-LocalBoxAutomationAz { + param([Parameter(Mandatory)][string[]]$Arguments) + Update-MhhToken | Out-Null + $output = & az @Arguments --only-show-errors --output json 2>&1 + if ($LASTEXITCODE -ne 0) { throw "LocalBox automation Azure CLI failed: $($output -join "`n")" } + if ($output) { ($output -join "`n") | ConvertFrom-Json -AsHashtable -ErrorAction Stop } +} + +function Invoke-LocalBoxAutomationRequest { + param([string]$Path, [string]$Method = 'GET', [hashtable]$Body, [switch]$AllowNotFound) + Update-MhhToken | Out-Null + $arguments = @{ Path = $Path; Method = $Method; ErrorAction = 'Stop' } + if ($Body) { $arguments.Payload = $Body | ConvertTo-Json -Depth 30 -Compress } + $response = Invoke-AzRestMethod @arguments + if ($AllowNotFound -and $response.StatusCode -eq 404) { return } + if ($response.StatusCode -notin @(200, 201, 202, 204)) { + throw "LocalBox automation $Method failed (HTTP $($response.StatusCode)): $($response.Content)" + } + if ($response.Content) { $response.Content | ConvertFrom-Json -AsHashtable -ErrorAction Stop } +} + +function Wait-LocalBoxAutomation { + param([scriptblock]$Probe, [int]$TimeoutSeconds, [string]$Phase) + $deadline = [datetime]::UtcNow.AddSeconds($TimeoutSeconds) + do { + Update-MhhToken | Out-Null + $state = & $Probe + if ($state.Ready) { return $state } + if ([datetime]::UtcNow -ge $deadline) { throw "Timed out during $Phase. $($state.Detail)" } + Write-Host "LocalBox ${Phase}: $($state.Detail)" + Start-Sleep -Seconds 30 + } while ($true) +} + +function Wait-LocalBoxNestedDeployment { + param([string]$SubscriptionId, [string]$ResourceGroupName, [int]$TimeoutSeconds = 28800) + Wait-LocalBoxAutomation -Phase 'nested deployment' -TimeoutSeconds $TimeoutSeconds -Probe { + $deployments = @(Invoke-LocalBoxAutomationAz @('deployment', 'group', 'list', '--subscription', $SubscriptionId, '--resource-group', $ResourceGroupName)) + $clusterDeployment = @($deployments | Where-Object name -eq 'localcluster-deploy') + foreach ($deployment in @($deployments | Where-Object name -in @('localcluster-validate', 'localcluster-deploy'))) { + if ($deployment.properties.provisioningState -in @('Failed', 'Canceled')) { + throw "Nested deployment $($deployment.name) failed. Inspect its deployment operations and Client logs." + } + } + if ($clusterDeployment.Count -ne 1 -or $clusterDeployment[0].properties.provisioningState -ne 'Succeeded') { + return @{ Ready = $false; Detail = 'Waiting for localcluster-deploy to succeed.' } + } + $resources = @(Invoke-LocalBoxAutomationAz @('resource', 'list', '--subscription', $SubscriptionId, '--resource-group', $ResourceGroupName)) + foreach ($type in @('Microsoft.AzureStackHCI/clusters', 'Microsoft.ResourceConnector/appliances', 'Microsoft.ExtendedLocation/customLocations')) { + $matching = @($resources | Where-Object type -ieq $type) + if ($matching.Count -gt 1) { throw "Expected one $type in the shared LocalBox scope." } + if ($matching.Count -eq 0) { return @{ Ready = $false; Detail = "Waiting for $type." } } + $resource = Invoke-LocalBoxAutomationAz @('resource', 'show', '--ids', $matching[0].id) + if ($resource.properties.provisioningState -in @('Failed', 'Canceled')) { throw "$($resource.id) is $($resource.properties.provisioningState)." } + if ($resource.properties.provisioningState -ne 'Succeeded') { return @{ Ready = $false; Detail = "Waiting for $type provisioning." } } + if ($type -eq 'Microsoft.AzureStackHCI/clusters' -and $resource.properties.connectivityStatus -ne 'Connected') { + return @{ Ready = $false; Detail = 'Waiting for Azure Local connectivity.' } + } + if ($type -eq 'Microsoft.ResourceConnector/appliances' -and $resource.properties.status -ne 'Running') { + return @{ Ready = $false; Detail = 'Waiting for Arc Resource Bridge.' } + } + if ($type -eq 'Microsoft.ExtendedLocation/customLocations') { + $extensions = @($resource.properties.clusterExtensionIds | Where-Object { $_ -match '/hybridaksextension$' }) + if ($extensions.Count -ne 1) { return @{ Ready = $false; Detail = 'Waiting for hybridaksextension association.' } } + $extension = Invoke-LocalBoxAutomationAz @('resource', 'show', '--ids', $extensions[0], '--api-version', '2023-05-01') + if ($extension.properties.provisioningState -in @('Failed', 'Canceled')) { throw 'hybridaksextension provisioning failed.' } + if ($extension.properties.provisioningState -ne 'Succeeded') { return @{ Ready = $false; Detail = 'Waiting for hybridaksextension.' } } + } + } + @{ Ready = $true } + } | Out-Null +} + +function New-LocalBoxAutomationPackage { + param([string]$ResourceRoot, [string]$Directory) + $files = @('prepare-localbox.ps1', 'invoke-localbox-automation.ps1', 'test-sovereign-cloud.ps1', + 'tests/localbox.health.tests.ps1', 'tests/sovereign-lab.health.tests.ps1') + $contents = [ordered]@{} + $package = Join-Path $Directory 'package' + New-Item -ItemType Directory -Path "$package/tests" -Force | Out-Null + foreach ($file in $files) { + $source = Join-Path $ResourceRoot $file + $contents[$file] = (Get-FileHash -LiteralPath $source -Algorithm SHA256 -ErrorAction Stop).Hash + Copy-Item -LiteralPath $source -Destination (Join-Path $package $file) -ErrorAction Stop + } + $bytes = [Text.Encoding]::UTF8.GetBytes(($contents | ConvertTo-Json -Compress)) + $hash = [Convert]::ToHexString([Security.Cryptography.SHA256]::HashData($bytes)).ToLowerInvariant() + $archive = Join-Path $Directory 'package.zip' + Compress-Archive -Path "$package/*" -DestinationPath $archive -ErrorAction Stop + @{ Hash = $hash; Archive = [Convert]::ToBase64String([IO.File]::ReadAllBytes($archive)) } +} + +function New-LocalBoxGuestCommand { + param([hashtable]$Configuration, [string]$Archive) + $configurationBase64 = [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes(($Configuration | ConvertTo-Json -Compress))) + # Managed Run Command starts Windows PowerShell as SYSTEM. Launch PS7 explicitly. + @' +$ErrorActionPreference = 'Stop' +try { + $settings = [Text.Encoding]::UTF8.GetString([Convert]::FromBase64String('__CONFIG__')) | ConvertFrom-Json + $root = 'C:\LocalBox\MicroHackAutomation' + $directory = Join-Path $root $settings.RunId + foreach ($path in @($root, $directory)) { + if (Test-Path -LiteralPath $path) { + if ((Get-Item -LiteralPath $path -Force).Attributes -band [IO.FileAttributes]::ReparsePoint) { throw 'Automation directory must not be a link.' } + } else { New-Item -ItemType Directory -Path $path | Out-Null } + $identity = [Security.Principal.WindowsIdentity]::GetCurrent().User + $acl = New-Object Security.AccessControl.DirectorySecurity + $acl.SetOwner($identity) + $acl.SetAccessRuleProtection($true, $false) + $acl.AddAccessRule((New-Object Security.AccessControl.FileSystemAccessRule($identity, 'FullControl', 'ContainerInherit, ObjectInherit', 'None', 'Allow'))) + Set-Acl -LiteralPath $path -AclObject $acl + } + $archivePath = Join-Path $directory 'package.zip' + [IO.File]::WriteAllBytes($archivePath, [Convert]::FromBase64String('__ARCHIVE__')) + Expand-Archive -LiteralPath $archivePath -DestinationPath $directory -Force + Remove-Item -LiteralPath $archivePath -Force + $parametersPath = Join-Path $directory 'automation.json' + [IO.File]::WriteAllText($parametersPath, [Text.Encoding]::UTF8.GetString([Convert]::FromBase64String('__CONFIG__'))) + & 'C:\Program Files\PowerShell\7\pwsh.exe' -NoProfile -NonInteractive -File "$directory\invoke-localbox-automation.ps1" -ParametersPath $parametersPath + exit $LASTEXITCODE +} catch { + Write-Error $_ + exit 1 +} +'@.Replace('__CONFIG__', $configurationBase64).Replace('__ARCHIVE__', $Archive) +} + +function Assert-LocalBoxAutomationResult { + param([hashtable]$Result, [string]$RunId, [string]$PackageHash, [string]$GroupId, [string]$SubscriptionId, [string]$ResourceGroupName) + if ($Result.RunId -ne $RunId -or $Result.PackageHash -ne $PackageHash -or + $Result.AksAdminGroupObjectId -ne $GroupId -or $Result.SubscriptionId -ne $SubscriptionId -or + $Result.ResourceGroupName -ne $ResourceGroupName -or $Result.State -ne 'Succeeded') { + throw 'LocalBox automation result is missing, stale, mismatched or unsuccessful.' + } + foreach ($suite in @('Upstream', 'Sovereign')) { + $tests = $Result[$suite] + if (-not $tests -or $tests.Passed -le 0 -or $tests.Failed -ne 0 -or $tests.Skipped -ne 0 -or $tests.NotRun -ne 0) { + throw "LocalBox $suite tests did not establish readiness." + } + } + if ($Result.UpdateAssessment.Status -ne 'Succeeded' -or -not $Result.UpdateAssessment.Timestamp) { + throw 'Reference VM update assessment did not complete successfully.' + } +} + +function Assert-LocalBoxAutomationXml { + param([string]$Path, [hashtable]$Expected) + $settings = [Xml.XmlReaderSettings]::new() + $settings.DtdProcessing = [Xml.DtdProcessing]::Prohibit + $reader = [Xml.XmlReader]::Create($Path, $settings) + try { + $document = [Xml.XmlDocument]::new() + $document.XmlResolver = $null + $document.Load($reader) + $cases = @($document.SelectNodes('//test-case')) + if ($document.DocumentElement.get_LocalName() -ne 'test-results' -or $cases.Count -ne $Expected.Passed -or + @($cases | Where-Object { $_.GetAttribute('result') -ne 'Success' -or $_.GetAttribute('executed') -ne 'True' }).Count) { + throw "Pester XML does not match the successful test summary: $Path" + } + } + finally { $reader.Dispose() } +} + +function Invoke-LocalBoxHostedAutomation { + [CmdletBinding()] + param( + [Parameter(Mandatory)][string]$SubscriptionId, + [Parameter(Mandatory)][string]$ResourceGroupName, + [Parameter(Mandatory)][guid]$AksAdminGroupObjectId, + [string]$ResourceRoot = (Join-Path $PSScriptRoot '../resources'), + [int]$NestedTimeoutSeconds = 28800, + [int]$GuestTimeoutSeconds = 43200 + ) + $ErrorActionPreference = 'Stop' + if ($AksAdminGroupObjectId -eq [guid]::Empty) { throw 'A nonempty event Entra group ID is required.' } + $scope = "/subscriptions/$SubscriptionId/resourceGroups/$ResourceGroupName" + $vmId = "$scope/providers/Microsoft.Compute/virtualMachines/LocalBox-Client" + $commandPath = "$vmId/runCommands/sovereign-localbox-prepare" + $commandUri = "$commandPath`?api-version=2024-07-01" + $temporary = Join-Path ([IO.Path]::GetTempPath()) "localbox-automation-$([guid]::NewGuid())" + New-Item -ItemType Directory -Path $temporary | Out-Null + try { + $package = New-LocalBoxAutomationPackage -ResourceRoot $ResourceRoot -Directory $temporary + $existing = Invoke-LocalBoxAutomationRequest -Path "$commandUri&`$expand=instanceView" -AllowNotFound + $active = $existing -and ($existing.properties.provisioningState -in @('Creating', 'Updating') -or + $existing.properties.instanceView.executionState -in @('Pending', 'Running', 'Unknown') -or + -not $existing.properties.instanceView.executionState) + $runId = [guid]::NewGuid().ToString() + $startedAfter = [datetime]::UtcNow.ToString('o') + if ($active) { + $metadata = @{} + foreach ($item in $existing.properties.parameters) { $metadata[$item.name] = $item.value } + $existingRunId = [guid]::Empty + if ($metadata.PackageHash -ne $package.Hash -or $metadata.GroupId -ne $AksAdminGroupObjectId.ToString() -or + -not [guid]::TryParse([string]$metadata.RunId, [ref]$existingRunId) -or $existingRunId -eq [guid]::Empty -or + -not $metadata.StartedAfter) { + throw 'An active LocalBox automation command belongs to different content or group. Inspect it before retrying; it was not replaced.' + } + $runId = $metadata.RunId + $startedAfter = $metadata.StartedAfter + Write-Host "Resuming monitoring of LocalBox automation $runId." + } + else { + Wait-LocalBoxNestedDeployment -SubscriptionId $SubscriptionId -ResourceGroupName $ResourceGroupName -TimeoutSeconds $NestedTimeoutSeconds + } + $vm = Invoke-LocalBoxAutomationRequest -Path "$vmId`?api-version=2024-07-01" + $accounts = @(Invoke-LocalBoxAutomationAz @('storage', 'account', 'list', '--subscription', $SubscriptionId, '--resource-group', $ResourceGroupName) | + Where-Object { $_.name -match '^localbox[a-z0-9]+$' -and $_.name -notlike 'localboxdiag*' }) + if ($accounts.Count -ne 1) { throw 'Expected exactly one LocalBox staging storage account for private automation results.' } + $account = $accounts[0] + $container = 'microhack-automation' + $containerId = "$($account.id)/blobServices/default/containers/$container" + Invoke-LocalBoxAutomationRequest -Path "$containerId`?api-version=2023-05-01" -Method PUT -Body @{ properties = @{ publicAccess = 'None' } } | Out-Null + Update-MhhToken | Out-Null + $token = Get-AzAccessToken -ResourceUrl 'https://management.azure.com/' -ErrorAction Stop + $jwt = if ($token.Token -is [securestring]) { [System.Net.NetworkCredential]::new('', $token.Token).Password } else { [string]$token.Token } + $payload = $jwt.Split('.')[1].Replace('-', '+').Replace('_', '/') + $payload = $payload.PadRight($payload.Length + (4 - $payload.Length % 4) % 4, '=') + $caller = ([Text.Encoding]::UTF8.GetString([Convert]::FromBase64String($payload)) | ConvertFrom-Json).oid + $jwt = $null + foreach ($identity in @($caller, $vm.identity.principalId)) { + $id = [guid]::Empty + if (-not [guid]::TryParse([string]$identity, [ref]$id) -or $id -eq [guid]::Empty) { throw 'Unable to resolve automation identities for Blob RBAC.' } + Update-MhhToken | Out-Null + $grants = @(Get-AzRoleAssignment -ObjectId $identity -RoleDefinitionName 'Storage Blob Data Contributor' -Scope $containerId -ErrorAction Stop) + if (-not $grants.Count) { + New-AzRoleAssignment -ObjectId $identity -RoleDefinitionName 'Storage Blob Data Contributor' -Scope $containerId -ErrorAction Stop | Out-Null + } + } + $resultUri = "$($account.primaryEndpoints.blob)$container/$runId" + Write-Host "Private LocalBox automation results: $resultUri (Entra authentication required)." + if (-not $active) { + $configuration = @{ + RunId = $runId; PackageHash = $package.Hash; SubscriptionId = $SubscriptionId + ResourceGroupName = $ResourceGroupName; AksAdminGroupObjectId = $AksAdminGroupObjectId.ToString() + StorageAccountName = $account.name; Container = $container + } + $source = New-LocalBoxGuestCommand -Configuration $configuration -Archive $package.Archive + Invoke-LocalBoxAutomationRequest -Path $commandUri -Method PUT -Body @{ + location = $vm.location + properties = @{ + source = @{ script = $source }; asyncExecution = $true; timeoutInSeconds = $GuestTimeoutSeconds + parameters = @( + @{ name = 'RunId'; value = $runId }, @{ name = 'PackageHash'; value = $package.Hash } + @{ name = 'GroupId'; value = $AksAdminGroupObjectId.ToString() } + @{ name = 'StartedAfter'; value = $startedAfter } + ) + } + } | Out-Null + } + Wait-LocalBoxAutomation -Phase 'guest preparation and tests' -TimeoutSeconds ($GuestTimeoutSeconds + 300) -Probe { + $command = Invoke-LocalBoxAutomationRequest -Path "$commandUri&`$expand=instanceView" + $state = $command.properties.instanceView + if ($command.properties.provisioningState -in @('Failed', 'Canceled')) { + throw "LocalBox Run Command provisioning failed. Inspect $commandPath." + } + $reportedRun = @($command.properties.parameters | Where-Object name -eq 'RunId') + if ($reportedRun.Count -ne 1 -or $reportedRun[0].value -ne $runId -or -not $state.startTime -or + [datetimeoffset]$state.startTime -lt [datetimeoffset]$startedAfter) { + return @{ Ready = $false; Detail = "Waiting for execution $runId to start; ignoring previous instanceView." } + } + if ($state.executionState -in @('Failed', 'Canceled', 'TimedOut')) { + throw "LocalBox guest automation failed. Inspect $commandPath instanceView and $resultUri." + } + if ($state.executionState -eq 'Succeeded') { + if ($null -eq $state.exitCode -or $state.exitCode -ne 0) { throw "LocalBox guest exit code was not zero. Inspect $resultUri." } + return @{ Ready = $true } + } + @{ Ready = $false; Detail = "$($state.executionState); results: $resultUri" } + } | Out-Null + foreach ($name in @('status.json', 'health.json', 'health.xml', 'upstream.xml')) { + Invoke-LocalBoxAutomationAz @('storage', 'blob', 'download', '--account-name', $account.name, '--container-name', $container, + '--name', "$runId/$name", '--file', (Join-Path $temporary $name), '--auth-mode', 'login', '--overwrite', 'true') | Out-Null + } + $result = Get-Content -LiteralPath "$temporary/status.json" -Raw | ConvertFrom-Json -AsHashtable + Assert-LocalBoxAutomationResult -Result $result -RunId $runId -PackageHash $package.Hash -GroupId $AksAdminGroupObjectId.ToString() ` + -SubscriptionId $SubscriptionId -ResourceGroupName $ResourceGroupName + Assert-LocalBoxAutomationXml -Path "$temporary/health.xml" -Expected $result.Sovereign + Assert-LocalBoxAutomationXml -Path "$temporary/upstream.xml" -Expected $result.Upstream + Write-Warning 'Automated infrastructure, MDE installation and update assessment passed. Defender cloud reporting and real participant sign-in still require verification.' + @{ RunId = $runId; ResultsUri = $resultUri } + } + finally { Remove-Item -LiteralPath $temporary -Recurse -Force -ErrorAction Stop } +} diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md index 0d700e481..eb47e4f41 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md @@ -100,32 +100,114 @@ releasing a subscription. This hook is not part of manual/BYOS preparation. ## Complete LocalBox preparation -The global hook submits `localbox-*` in `rg-localbox-shared` in the published **Shared LocalBox Subscription ID**, then waits up to six hours for ARM completion, refreshing its Console authentication while polling. It does not publish LocalBox administrator credentials. A Client VM appearing after roughly 15-20 minutes confirms only that deployment has started. Full nested Azure Local provisioning can take 4-6 hours and must still be checked separately. +The global hook now remains active through nested deployment, default Sovereign +preparation, and infrastructure health checks. No organizer login, host password +reset, or manual preparation invocation is required for a greenfield hosted event. +It does not publish LocalBox administrator credentials. + +The [orchestrator](../../labautomation/localbox-automation.ps1) implements: + +1. Wait for `localbox-*` ARM completion, then `localcluster-deploy`, connected + Azure Local, a running Arc Resource Bridge, custom location and + `hybridaksextension`. A successful Client bootstrap extension is not sufficient. +2. Use the validated `Get-MhhDefaultLabGroup` result (also published as **Lab + Group ObjectId**) as `AksAdminGroupObjectId`. No guest Graph lookup is needed. +3. Package the preparation script, guest runner and health tests from the + **imported content directory** into the Managed Run Command payload. A content + hash identifies the package; there is no second download from upstream `main`. +4. Run `sovereign-localbox-prepare` asynchronously on LocalBox-Client as **SYSTEM**, + explicitly using PowerShell 7. The [guest runner](../invoke-localbox-automation.ps1) + prepares storage, logical networks, AKS, proxy RBAC, MetalLB, the Windows + image and the default reference VM. +5. Wait for the reference VM to be Running, Arc Connected and MDE installation + Succeeded. Request an Update Manager assessment and require a successful, + timestamped result. This assesses updates; it does **not** install patches. +6. Run installed upstream Jumpstart infrastructure Pester tests, then the + Sovereign **LocalBox Full** suite. The Client retrieves a private AKS admin + kubeconfig for infrastructure probes using its managed identity. It is not + shared with participants and is deleted after the run. +7. Require a zero guest exit code, matching run/content/group/scope metadata, + nonempty passing suites without skipped/not-run tests, and matching NUnit XML + before publishing global discovery credentials and returning to Console. + +`Update-MhhToken` runs in the **Console process** on each poll and before Azure +operations, including result retrieval. It refreshes Console clients, not guest +credentials or job time limits. The guest uses its own managed identity and +locally resolved nested Windows credentials. Generated reference-VM credentials +remain DPAPI-protected under SYSTEM; do not copy them into Console output or +attempt to decrypt them from a different Windows account. + +### Timeouts and scheduling + +The initial ARM wait allows six hours, nested readiness eight hours, and the +Managed Run Command twelve hours (with five additional minutes for host polling). +These are independent ceilings, **not** expected durations. Confirm with the +Console operator that its global-job limit accommodates the intended run; token +refresh cannot prevent a platform-enforced job timeout. Subscription and +participant jobs now start **after** this global readiness gate, rather than +overlapping nested deployment. The subsequent preparation script took **4 hours 24 minutes 26 seconds** in the TEST4 run reported on 25 October 2026, including reference-VM creation. This is additional post-provisioning time, not the total event deployment duration or a -guaranteed runtime. MDE was still Pending when preparation finished. See the +guaranteed runtime. MDE was still Pending when preparation finished; the new +automation waits separately for installation and assesses updates. See the [timing evidence and planning guidance](../localbox/readme.md#preparation-time); allow time for both preparation and later assessments. -1. Inspect the Azure deployment, Client bootstrap logs and Azure Local/Arc status. Resolve failures before proceeding. -2. If the Client password is unknown, an authorized event lead/coach uses the Azure VM's **Help > Reset password** to set a known password for the `arcdemo` host account, then connects to `LocalBox-Client` through Bastion. Do not expose this password through participant credentials or job logs. A host password reset does not change the nested-node Windows credentials. See [LocalBox credentials](#localbox-credentials). -3. Copy **Lab Group ObjectId** from the Console's **Credentials** tab for the preparation prompt or `-AksAdminGroupObjectId`. Confirm that intended Azure lab identities/coaches are members; see the group dependency below. -4. Follow [LocalBox preparation](../localbox/readme.md), **once per event on the global LocalBox Client**, in elevated PowerShell 7. It creates AKS and then installs/verifies the shared MetalLB extension and reserved ARP pool. Azure authentication uses the Client VM's managed identity; preparation constructs the nested Windows credential locally from the installed configuration without printing it. An explicit `-NodeCredential` overrides that value after a nested-account rotation. - Preparation includes `sov-ref-vm` by default; run it **the day before the event** for inspection while participant VMs onboard, or opt out with `-IncludeReferenceVm:$false`. Follow the [reference readiness checks](../localbox/readme.md#reference-readiness-and-participant-use); Console plan configuration and successful provisioning do not establish completed VM assessments. Participants must not modify this shared VM; no RBAC restrictions are added. -5. Run [Pester health checks](../tests/readme.md) for LocalBox and every selected participant lab. A control-plane-only pass does not establish full readiness. -6. Console shared setup enables and verifies **Defender for Servers Plan 2** and **Endpoint protection** on every event subscription. Complete the [Defender readiness checklist](../localbox/manual-preparation.md#defender-for-servers-readiness-organizer) and [participant VM readiness exercise](../localbox/manual-preparation.md#step-6-test-the-environment), including guest management, actual Defender onboarding and Update Manager. Verified subscription settings do not prove that a VM has completed onboarding. -7. Verify an intended participant can use Entra-authenticated `az connectedk8s proxy` and a private application port-forward from their Codespace. Participants inspect MetalLB, not install it, and never need LocalBox-Client access. This validates the lab access path, not routing to the MetalLB service VIP. +### Diagnostics and reruns + +Results use a private `microhack-automation` container in the existing LocalBox +staging storage account. The Console identity and Client identity receive +**Storage Blob Data Contributor at that container only**. No SAS tokens, account +keys or participant data-plane grants are created. Storage firewall rules are +not changed. The Console logs the authenticated result URL and run ID. + +Each run has a separate prefix containing `status.json`, `automation.log`, +`upstream.xml`, `health.xml` and `health.json` when those phases produced them. +Status is updated at phase boundaries. Failures retain available diagnostics and +resources; a failed upload also fails the command. Only the listed diagnostics +are uploaded, never the AKS kubeconfig or reference-VM credential file. + +An organizer with Blob data access can download a run using: + +```bash +az storage blob download-batch --account-name \ + --source microhack-automation --pattern '/*' \ + --destination ./localbox-results --auth-mode login +``` + +Inspect **LocalBox-Client > Run commands > sovereign-localbox-prepare** and its +instance view if no diagnostic blobs were uploaded. Bootstrap failures before +the guest runner starts are reported there. A matching active command is +monitored without resubmission. A different active content hash/group is rejected, +not overwritten. After a completed/failed command, retrying starts a new tracked +run and reuses preparation's resource checks; it does not delete the lab or reset +existing VM passwords. Investigate failures before retrying. + +### Remaining event validation + +The automated gate verifies **infrastructure**, MDE extension installation and +an Update Manager assessment. It explicitly reports Defender cloud device +reporting and participant sign-in as `NotVerified`; these are not inferred from +an installed extension, a resource-count check or admin kubeconfig access. + +- After participant deployments, run [health checks](../tests/readme.md) for each + selected lab; they cannot run inside the earlier global stage. +- Complete the [Defender readiness checklist](../localbox/manual-preparation.md#defender-for-servers-readiness-organizer), + including actual device reporting and recommendation/assessment visibility. +- Verify intended lab users/coaches belong to the Console group and test a real + participant's Entra-authenticated proxy and private application port-forward. +- Preserve the [manual preparation guide](../localbox/readme.md) for BYOS and + deliberate recovery only. Do not run it concurrently with hosted automation. Global setup registers `Microsoft.KubernetesRuntime`, waits for registration, resolves its tenant service-principal object ID using the Console's existing Graph-capable identity, and records it in the shared RG tag `microhack-k8s-runtime-object-id`. Missing registration or identity resolution fails setup explicitly. Organizer preparation reads that nonsecret tag; it does -not require Graph permissions on the Client managed identity. This does not move -AKS or MetalLB installation into the Console hook: they remain part of the -organizer's post-provisioning preparation. +not require Graph permissions on the Client managed identity. The global hook +invokes AKS and MetalLB preparation inside the Client after nested readiness. ### LocalBox registration region and location policies diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/invoke-localbox-automation.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/invoke-localbox-automation.ps1 new file mode 100644 index 000000000..698c86f56 --- /dev/null +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/invoke-localbox-automation.ps1 @@ -0,0 +1,193 @@ +#Requires -Version 7.0 +[CmdletBinding()] +param([string]$ParametersPath, [switch]$HealthOnly, + [string]$ManifestPath = 'C:\LocalBox\sovereign-localbox.json') + +function Initialize-LocalBoxAutomationEnvironment { + foreach ($name in @('subscriptionId', 'resourceGroup', 'tenantId', 'autoDeployClusterResource', 'LocalBoxConfigFile', 'LocalBoxDir')) { + $value = [Environment]::GetEnvironmentVariable($name, 'Machine') + if (-not $value) { throw "LocalBox machine environment is missing $name." } + [Environment]::SetEnvironmentVariable($name, $value, 'Process') + } + $env:PATH = [Environment]::GetEnvironmentVariable('PATH', 'Machine') + ';' + $env:PATH + foreach ($command in @('az', 'kubectl', 'Get-VM')) { Get-Command $command -ErrorAction Stop | Out-Null } +} + +function Assert-LocalBoxAutomationTests { + param($Result) + if ($Result.TotalCount -le 0 -or $Result.FailedCount -ne 0 -or $Result.SkippedCount -ne 0 -or + $Result.NotRunCount -ne 0 -or $Result.PassedCount -ne $Result.TotalCount) { + throw 'Required Pester tests failed, were skipped, or did not run.' + } + @{ Passed = $Result.PassedCount; Failed = $Result.FailedCount; Skipped = $Result.SkippedCount; NotRun = $Result.NotRunCount } +} + +function Send-LocalBoxAutomationFiles { + param([hashtable]$Configuration, [string]$Directory) + foreach ($name in @('status.json', 'health.json', 'health.xml', 'upstream.xml', 'automation.log')) { + $path = Join-Path $Directory $name + if (Test-Path -LiteralPath $path) { + Invoke-LocalBoxAz @('storage', 'blob', 'upload', '--account-name', $Configuration.StorageAccountName, + '--container-name', $Configuration.Container, '--name', "$($Configuration.RunId)/$name", + '--file', $path, '--auth-mode', 'login', '--overwrite', 'true') -NoOutput + } + } +} + +function Write-LocalBoxAutomationStatus { + param([hashtable]$Configuration, [string]$Directory, [hashtable]$Status) + Write-Host "LocalBox automation phase: $($Status.Phase)" + $Status | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath "$Directory/status.json" + Invoke-LocalBoxAz @('storage', 'blob', 'upload', '--account-name', $Configuration.StorageAccountName, + '--container-name', $Configuration.Container, '--name', "$($Configuration.RunId)/status.json", + '--file', "$Directory/status.json", '--auth-mode', 'login', '--overwrite', 'true') -NoOutput +} + +function Invoke-LocalBoxAutomationHealth { + param([hashtable]$Configuration, [string]$Directory, [string]$ResourceRoot, [string]$ManifestPath) + $nodeConfig = Import-PowerShellDataFile -LiteralPath $env:LocalBoxConfigFile + $credential = Resolve-LocalBoxNodeCredential -Configuration $nodeConfig + $global:LASTEXITCODE = 0 + & "$ResourceRoot/test-sovereign-cloud.ps1" -Scope LocalBox -Mode Full -DownloadTests:$false ` + -LocalBoxManifestPath $ManifestPath -LocalBoxKubeconfig "$Directory/aks.kubeconfig" ` + -NodeCredential $credential -TimeoutMinutes 5 -OutputDirectory $Directory + if ($LASTEXITCODE -ne 0) { throw 'Sovereign Cloud health runner exited unsuccessfully.' } +} + +function Invoke-LocalBoxAutomationHealthProcess { + param([string]$ResourceRoot, [string]$InputPath, [string]$ManifestPath) + & "$PSHOME/pwsh.exe" -NoProfile -NonInteractive -File "$ResourceRoot/invoke-localbox-automation.ps1" -ParametersPath $InputPath -ManifestPath $ManifestPath -HealthOnly + if ($LASTEXITCODE -ne 0) { throw 'Sovereign Cloud Full health tests failed. Inspect health.xml and health.json.' } +} + +function Invoke-LocalBoxGuestAutomation { + param([hashtable]$Configuration, [string]$Directory, [string]$ResourceRoot, [string]$InputPath, + [string]$ManifestPath = 'C:\LocalBox\sovereign-localbox.json') + $ErrorActionPreference = 'Stop' + $status = @{ + RunId = $Configuration.RunId; PackageHash = $Configuration.PackageHash + SubscriptionId = $Configuration.SubscriptionId; ResourceGroupName = $Configuration.ResourceGroupName + AksAdminGroupObjectId = $Configuration.AksAdminGroupObjectId + State = 'Running'; StartedAt = [datetime]::UtcNow.ToString('o'); Phase = 'Prerequisites' + DefenderCloudReporting = 'NotVerified'; ParticipantSignIn = 'NotVerified' + } + $lock = [Threading.Mutex]::new($false, 'Global\MicroHackLocalBoxAutomation') + $locked = $false + $transcribing = $false + $previousConfig = $env:AZURE_CONFIG_DIR + $previousExtensions = $env:AZURE_EXTENSION_DIR + try { + $locked = $lock.WaitOne(0) + if (-not $locked) { throw 'Another hosted LocalBox automation is running.' } + Start-Transcript -LiteralPath "$Directory/automation.log" -Force | Out-Null + $transcribing = $true + if ($Configuration.SubscriptionId -ine $env:subscriptionId -or $Configuration.ResourceGroupName -ine $env:resourceGroup) { + throw 'Automation scope does not match the LocalBox machine environment.' + } + $group = [guid]::Empty + if (-not [guid]::TryParse($Configuration.AksAdminGroupObjectId, [ref]$group) -or $group -eq [guid]::Empty) { + throw 'A valid Console event group ID is required; interactive prompting is disabled.' + } + $env:AZURE_CONFIG_DIR = Join-Path $Directory '.azure' + $env:AZURE_EXTENSION_DIR = Join-Path (Split-Path $Directory) 'cliextensions' + Invoke-LocalBoxAz @('login', '--identity') | Out-Null + Invoke-LocalBoxAz @('account', 'set', '--subscription', $Configuration.SubscriptionId) | Out-Null + # Blob role assignments may still be propagating when the guest starts. + Wait-SovereignCheck -TimeoutSeconds 600 -Check { + Invoke-LocalBoxAz @('storage', 'blob', 'list', '--account-name', $Configuration.StorageAccountName, + '--container-name', $Configuration.Container, '--auth-mode', 'login', '--num-results', '1') | Out-Null + } + $status.Phase = 'Preparation' + Write-LocalBoxAutomationStatus $Configuration $Directory $status + & "$ResourceRoot/prepare-localbox.ps1" -SubscriptionId $Configuration.SubscriptionId -ResourceGroupName $Configuration.ResourceGroupName ` + -AksAdminGroupObjectId $group.ToString() -ManifestPath $ManifestPath -Confirm:$false + if (-not $?) { throw 'LocalBox preparation failed.' } + $manifest = Get-Content -LiteralPath $ManifestPath -Raw | ConvertFrom-Json -AsHashtable + if ($manifest.AksPreparationSkipped -or -not $manifest.ReferenceVm -or $manifest.AksAdminGroupObjectId -ne $group.ToString()) { + throw 'Preparation manifest does not contain the required AKS, reference VM and event group.' + } + $status.Phase = 'ReferenceVmOnboarding' + Write-LocalBoxAutomationStatus $Configuration $Directory $status + Wait-SovereignCheck -TimeoutSeconds 7200 -Check { + $readiness = Get-LocalBoxReferenceVmReadiness $manifest.ReferenceVm.MachineId + if ($readiness.MdeProvisioningState -eq 'Failed') { throw 'MDE provisioning failed.' } + if ($readiness.PowerState -ne 'Running' -or $readiness.ArcStatus -ne 'Connected' -or $readiness.MdeProvisioningState -ne 'Succeeded') { + throw "Reference VM not ready: power=$($readiness.PowerState), Arc=$($readiness.ArcStatus), MDE=$($readiness.MdeProvisioningState)." + } + } + $status.Phase = 'UpdateAssessment' + Write-LocalBoxAutomationStatus $Configuration $Directory $status + $installed = @(Invoke-LocalBoxAz @('extension', 'list')) + if ('connectedmachine' -notin $installed.name) { Invoke-LocalBoxAz @('extension', 'add', '--name', 'connectedmachine') -TimeoutSeconds 900 | Out-Null } + $assessment = Invoke-LocalBoxAz @('connectedmachine', 'assess-patches', '--ids', $manifest.ReferenceVm.MachineId) -TimeoutSeconds 3600 + if ($assessment.status -ne 'Succeeded' -or -not $assessment.lastModifiedDateTime) { + throw 'Reference VM Update Manager assessment did not return Succeeded with a timestamp.' + } + $status.UpdateAssessment = @{ Status = $assessment.status; Timestamp = $assessment.lastModifiedDateTime; ActivityId = $assessment.assessmentActivityId } + $status.Phase = 'Pester' + Write-LocalBoxAutomationStatus $Configuration $Directory $status + $pester = Get-Module -ListAvailable Pester | Where-Object { $_.Version -ge [version]'5.7.1' -and $_.Version -lt [version]'6.0' } + if (-not $pester) { Install-Module Pester -RequiredVersion 5.7.1 -Scope CurrentUser -Force -ErrorAction Stop } + Import-Module Pester -MinimumVersion 5.7.1 -MaximumVersion 5.999.999 -ErrorAction Stop + $upstream = New-PesterConfiguration + $upstream.Run.Path = @('C:\LocalBox\Tests\common.tests.ps1', 'C:\LocalBox\Tests\azlocal.tests.ps1') + foreach ($path in $upstream.Run.Path.Value) { if (-not (Test-Path -LiteralPath $path)) { throw "Missing upstream test file: $path" } } + $upstream.Run.PassThru = $true + $upstream.TestResult.Enabled = $true + $upstream.TestResult.OutputFormat = 'NUnitXml' + $upstream.TestResult.OutputPath = "$Directory/upstream.xml" + $status.Upstream = Assert-LocalBoxAutomationTests (Invoke-Pester -Configuration $upstream) + Invoke-LocalBoxAz @('aksarc', 'get-credentials', '--subscription', $Configuration.SubscriptionId, + '--resource-group', $Configuration.ResourceGroupName, '--name', 'localbox-aks', '--admin', + '--file', "$Directory/aks.kubeconfig", '--overwrite-existing') -NoOutput + Invoke-LocalBoxAutomationHealthProcess -ResourceRoot $ResourceRoot -InputPath $InputPath -ManifestPath $ManifestPath + $health = Get-Content -LiteralPath "$Directory/health.json" -Raw | ConvertFrom-Json -AsHashtable + if (-not $health.FullReadiness) { throw 'Sovereign Cloud tests did not establish full infrastructure health.' } + $status.Sovereign = @{ Passed = $health.Passed; Failed = $health.Failed; Skipped = $health.Skipped; NotRun = $health.NotRun } + $status.State = 'Succeeded' + $status.Phase = 'Complete' + } + catch { + $status.State = 'Failed' + $status.Error = $_.Exception.Message + throw + } + finally { + try { + if ($transcribing) { Stop-Transcript | Out-Null } + if ($locked) { + $status.CompletedAt = [datetime]::UtcNow.ToString('o') + $status | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath "$Directory/status.json" + Send-LocalBoxAutomationFiles $Configuration $Directory + } + } + finally { + $env:AZURE_CONFIG_DIR = $previousConfig + $env:AZURE_EXTENSION_DIR = $previousExtensions + if ($locked) { + foreach ($path in @("$Directory/aks.kubeconfig", "$Directory/.azure")) { + if (Test-Path -LiteralPath $path) { Remove-Item -LiteralPath $path -Recurse -Force -ErrorAction Stop } + } + $lock.ReleaseMutex() + } + $lock.Dispose() + } + } +} + +if ($MyInvocation.InvocationName -ne '.') { + $ErrorActionPreference = 'Stop' + if (-not $IsWindows) { throw 'Hosted guest automation requires Windows LocalBox-Client.' } + Initialize-LocalBoxAutomationEnvironment + $requestedManifestPath = $ManifestPath + . "$PSScriptRoot/prepare-localbox.ps1" + . "$PSScriptRoot/test-sovereign-cloud.ps1" + $automation = Get-Content -LiteralPath $ParametersPath -Raw | ConvertFrom-Json -AsHashtable + $directory = Split-Path -Parent $ParametersPath + if ($HealthOnly) { + Invoke-LocalBoxAutomationHealth -Configuration $automation -Directory $directory -ResourceRoot $PSScriptRoot -ManifestPath $requestedManifestPath + } + else { + Invoke-LocalBoxGuestAutomation -Configuration $automation -Directory $directory -ResourceRoot $PSScriptRoot -InputPath $ParametersPath -ManifestPath $requestedManifestPath + } +} diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/localbox/readme.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/localbox/readme.md index ce29cc29a..4f67e0634 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/localbox/readme.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/localbox/readme.md @@ -1,6 +1,6 @@ # LocalBox post-provisioning preparation -Run [prepare-localbox.ps1](../prepare-localbox.ps1) **inside LocalBox-Client**, after Jumpstart has finished deploying the nested Azure Local cluster. This is common to hosted and manual delivery. It does not deploy a new LocalBox or replace the existing Azure AKS/K3s labs. +Run [prepare-localbox.ps1](../prepare-localbox.ps1) **inside LocalBox-Client**, after Jumpstart has finished deploying the nested Azure Local cluster. Hosted Console events now [invoke it automatically](../hosted-events/readme.md#complete-localbox-preparation), with AKS and the reference VM enabled, then run infrastructure health checks. The instructions below are for manual/BYOS delivery and deliberate recovery; do not run preparation concurrently with the hosted job. The script does not deploy a new LocalBox or replace the existing Azure AKS/K3s labs. ## Preparation time diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/global-localbox.tests.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/global-localbox.tests.ps1 index a70f64c34..9cb6f7f43 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/global-localbox.tests.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/global-localbox.tests.ps1 @@ -5,7 +5,7 @@ BeforeAll { foreach ($name in @('global-event-deploy.ps1', 'shared-deploy-lab.ps1', 'lab-defaults.json')) { Copy-Item "$root/labautomation/$name" $fixture } - foreach ($name in @('quota-helpers.ps1', 'localbox-credentials.ps1', 'hosted-tag-policy.ps1', 'hosted-defender.ps1')) { + foreach ($name in @('quota-helpers.ps1', 'localbox-credentials.ps1', 'localbox-automation.ps1', 'hosted-tag-policy.ps1', 'hosted-defender.ps1')) { Set-Content "$fixture/$name" '' } Set-Content "$fixture/resource-providers.ps1" 'Register-TestProviders' @@ -27,6 +27,7 @@ Submit-TestLocalBox @PSBoundParameters function Get-AzRoleAssignment { param($ObjectId, $RoleDefinitionName, $RoleDefinitionId, $Scope) } function New-AzRoleAssignment { param($ObjectId, $RoleDefinitionName, $Scope) } function Wait-LocalBoxDeployment { param($ResourceGroupName, $DeploymentName) } + function Invoke-LocalBoxHostedAutomation { param($SubscriptionId, $ResourceGroupName, $AksAdminGroupObjectId) } function Submit-TestLocalBox { param($SubscriptionId, $ResourceGroupName, $Location, $AzureLocalResourceProviderObjectId, $AzureLocalInstanceLocation, [switch]$UseConsoleCredentials, [switch]$NoWait) } @@ -85,6 +86,7 @@ Describe 'Event-wide LocalBox orchestration' { Mock Get-AzRoleAssignment {} Mock New-AzRoleAssignment { $state.Assignments.Add(@{ User = $ObjectId; Role = $RoleDefinitionName; Scope = $Scope }) } Mock Wait-LocalBoxDeployment { $state.Events.Add('wait') } + Mock Invoke-LocalBoxHostedAutomation { $state.Events.Add('automation'); @{ ResultsUri = 'https://private/results' } } Mock Submit-TestLocalBox { $state.Events.Add("deploy:$SubscriptionId") @{ DeploymentName = 'localbox-new'; ProvisioningState = 'Submitted' } @@ -106,7 +108,11 @@ Describe 'Event-wide LocalBox orchestration' { $SubscriptionId -eq $state.Subscription -and $ResourceGroupName -eq 'rg-localbox-shared' -and $AzureLocalInstanceLocation -eq 'australiaeast' -and $UseConsoleCredentials -and $NoWait } - $state.Events -join ',' | Should -Be "providers,tags:$($state.Subscription),defender:$($state.Subscription),deploy:$($state.Subscription),wait" + $state.Events -join ',' | Should -Be "providers,tags:$($state.Subscription),defender:$($state.Subscription),deploy:$($state.Subscription),wait,automation" + Should -Invoke Invoke-LocalBoxHostedAutomation -Times 1 -Exactly -ParameterFilter { + $SubscriptionId -eq $state.Subscription -and $ResourceGroupName -eq 'rg-localbox-shared' -and + $AksAdminGroupObjectId -eq '30000000-0000-0000-0000-000000000001' + } $state.Assignments.Count | Should -Be 6 foreach ($user in $users) { @($state.Assignments | Where-Object { $_.User -eq $user -and $_.Scope -eq $scope -and $_.Role -in @('Reader', 'Azure Stack HCI VM Contributor') }).Count | Should -Be 2 @@ -145,7 +151,7 @@ Describe 'Event-wide LocalBox orchestration' { Should -Invoke Submit-TestLocalBox -Times 1 -ParameterFilter { $Location -eq 'swedencentral' -and $SubscriptionId -eq $state.Subscription } } It 'stops without publishing discovery credentials when fails' -TestCases @( - @{ Stage = 'tags' }; @{ Stage = 'defender' }; @{ Stage = 'roles' }; @{ Stage = 'wait' }; @{ Stage = 'deploy' }; @{ Stage = 'graph' } + @{ Stage = 'tags' }; @{ Stage = 'defender' }; @{ Stage = 'roles' }; @{ Stage = 'wait' }; @{ Stage = 'deploy' }; @{ Stage = 'graph' }; @{ Stage = 'automation' } ) { param($Stage) switch ($Stage) { @@ -153,6 +159,7 @@ Describe 'Event-wide LocalBox orchestration' { 'defender' { Mock Initialize-MhhHostedDefender { throw 'Test failure' } } 'roles' { Mock Get-AzRoleAssignment { throw 'Test failure' } } 'wait' { Mock Wait-LocalBoxDeployment { throw 'Test failure' } } + 'automation' { Mock Invoke-LocalBoxHostedAutomation { throw 'Test failure' } } 'deploy' { Mock Submit-TestLocalBox { throw 'Test failure' } } 'graph' { Mock az { $global:LASTEXITCODE = 1 } } } diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-automation.tests.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-automation.tests.ps1 new file mode 100644 index 000000000..21ec8a3a8 --- /dev/null +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-automation.tests.ps1 @@ -0,0 +1,382 @@ +BeforeAll { + $resourceRoot = (Resolve-Path "$PSScriptRoot/..").Path + . "$resourceRoot/../labautomation/localbox-automation.ps1" + . "$resourceRoot/invoke-localbox-automation.ps1" + function Update-MhhToken {} + function az {} + function Invoke-AzRestMethod { param($Path, $Method, $Payload) } + function Get-AzAccessToken { param($ResourceUrl) } + function Get-AzRoleAssignment { param($ObjectId, $RoleDefinitionName, $Scope) } + function New-AzRoleAssignment { param($ObjectId, $RoleDefinitionName, $Scope) } + $subscription = '10000000-0000-0000-0000-000000000001' + $group = '20000000-0000-0000-0000-000000000001' + $identity = '30000000-0000-0000-0000-000000000001' + $scope = "/subscriptions/$subscription/resourceGroups/rg-localbox-shared" + $vmId = "$scope/providers/Microsoft.Compute/virtualMachines/LocalBox-Client" + $storageId = "$scope/providers/Microsoft.Storage/storageAccounts/localbox123" + $suite = @{ Passed = 1; Failed = 0; Skipped = 0; NotRun = 0 } + $xml = '' +} + +Describe 'Token-aware LocalBox polling' { + BeforeEach { Mock Update-MhhToken {}; Mock Start-Sleep {} } + It 'refreshes credentials on every poll, including completion' { + $counter = @{ Value = 0 } + Wait-LocalBoxAutomation -Phase test -TimeoutSeconds 60 -Probe { + $counter.Value++ + @{ Ready = $counter.Value -eq 3; Detail = 'waiting' } + } | Out-Null + Should -Invoke Update-MhhToken -Times 3 -Exactly + Should -Invoke Start-Sleep -Times 2 -Exactly + } + It 'fails instead of returning success on timeout' { + { Wait-LocalBoxAutomation -Phase test -TimeoutSeconds 0 -Probe { @{ Ready = $false } } } | Should -Throw '*Timed out*' + } + It 'does not continue with failed token refresh' { + Mock Update-MhhToken { throw 'Token refresh failed' } + { Wait-LocalBoxAutomation -Phase test -TimeoutSeconds 60 -Probe { throw 'Should not run' } } | Should -Throw '*Token refresh failed*' + Should -Invoke Start-Sleep -Times 0 + } + It 'propagates CLI errors and malformed JSON' { + Mock az { $global:LASTEXITCODE = 1; 'AuthorizationFailed' } + { Invoke-LocalBoxAutomationAz @('resource', 'list') } | Should -Throw '*AuthorizationFailed*' + Mock az { $global:LASTEXITCODE = 0; 'not JSON' } + { Invoke-LocalBoxAutomationAz @('resource', 'list') } | Should -Throw + } + It 'treats only an allowed 404 as absence' { + Mock Invoke-AzRestMethod { @{ StatusCode = 404; Content = '' } } + Invoke-LocalBoxAutomationRequest -Path '/test' -AllowNotFound | Should -BeNullOrEmpty + { Invoke-LocalBoxAutomationRequest -Path '/test' } | Should -Throw '*404*' + Mock Invoke-AzRestMethod { @{ StatusCode = 403; Content = 'Denied' } } + { Invoke-LocalBoxAutomationRequest -Path '/test' -AllowNotFound } | Should -Throw '*403*' + } +} + +Describe 'Nested deployment gate' { + BeforeEach { + Mock Update-MhhToken {} + Mock Start-Sleep {} + Mock Invoke-LocalBoxAutomationAz { + switch ($Arguments[0]) { + 'deployment' { @{ name = 'localcluster-deploy'; properties = @{ provisioningState = 'Succeeded' } } } + 'resource' { + if ($Arguments[1] -eq 'list') { + foreach ($type in @('Microsoft.AzureStackHCI/clusters', 'Microsoft.ResourceConnector/appliances', 'Microsoft.ExtendedLocation/customLocations')) { + @{ type = $type; id = "/$type/test" } + } + } + else { + @{ properties = @{ provisioningState = 'Succeeded'; connectivityStatus = 'Connected'; status = 'Running' + clusterExtensionIds = @('/test/hybridaksextension') } } + } + } + } + } + } + It 'requires cluster deployment, connectivity, bridge, custom location and extension' { + Wait-LocalBoxNestedDeployment -SubscriptionId $subscription -ResourceGroupName rg-localbox-shared -TimeoutSeconds 1 + Should -Invoke Invoke-LocalBoxAutomationAz -Times 6 -Exactly + } + It 'rejects a failed nested deployment rather than continuing preparation' { + Mock Invoke-LocalBoxAutomationAz { @{ name = 'localcluster-deploy'; properties = @{ provisioningState = 'Failed' } } } -ParameterFilter { $Arguments[0] -eq 'deployment' } + { Wait-LocalBoxNestedDeployment -SubscriptionId $subscription -ResourceGroupName rg-localbox-shared -TimeoutSeconds 0 } | Should -Throw '*Nested deployment*failed*' + } + It 'does not equate ARM success with cluster connectivity' { + Mock Invoke-LocalBoxAutomationAz { @{ properties = @{ provisioningState = 'Succeeded'; connectivityStatus = 'Disconnected' } } } -ParameterFilter { + $Arguments[1] -eq 'show' -and $Arguments[3] -match 'AzureStackHCI' + } + { Wait-LocalBoxNestedDeployment -SubscriptionId $subscription -ResourceGroupName rg-localbox-shared -TimeoutSeconds 0 } | Should -Throw '*connectivity*' + } + It 'stops on failed hybrid extension' { + Mock Invoke-LocalBoxAutomationAz { @{ properties = @{ provisioningState = 'Failed' } } } -ParameterFilter { + $Arguments[1] -eq 'show' -and $Arguments[3] -match 'hybridaksextension' + } + { Wait-LocalBoxNestedDeployment -SubscriptionId $subscription -ResourceGroupName rg-localbox-shared -TimeoutSeconds 0 } | Should -Throw '*hybridaksextension*failed*' + } +} + +Describe 'Imported-content package and PS7 bootstrap' { + It 'packages local content without fetching a moving GitHub branch' { + $one = New-Item -ItemType Directory -Path "$TestDrive/one" + $two = New-Item -ItemType Directory -Path "$TestDrive/two" + $first = New-LocalBoxAutomationPackage $resourceRoot $one.FullName + $second = New-LocalBoxAutomationPackage $resourceRoot $two.FullName + $first.Hash | Should -Be $second.Hash + $first.Hash | Should -Match '^[a-f0-9]{64}$' + Expand-Archive "$TestDrive/one/package.zip" "$TestDrive/extracted" + Test-Path "$TestDrive/extracted/tests/localbox.health.tests.ps1" | Should -BeTrue + (Get-FileHash "$TestDrive/extracted/prepare-localbox.ps1").Hash | Should -Be (Get-FileHash "$resourceRoot/prepare-localbox.ps1").Hash + $command = New-LocalBoxGuestCommand -Configuration @{ RunId = [guid]::NewGuid().ToString(); AksAdminGroupObjectId = $group } -Archive $first.Archive + $errors = $null + [void][Management.Automation.Language.Parser]::ParseInput($command, [ref]$null, [ref]$errors) + $errors.Count | Should -Be 0 + $command | Should -Match 'PowerShell\\7\\pwsh.exe' + $command | Should -Match 'exit \$LASTEXITCODE' + $command | Should -Not -Match 'raw.githubusercontent|__CONFIG__|__ARCHIVE__' + [Text.Encoding]::UTF8.GetByteCount($command) | Should -BeLessThan 262144 + } + It 'changes the package identity when a source file changes' { + Copy-Item -Path "$resourceRoot/prepare-localbox.ps1", "$resourceRoot/invoke-localbox-automation.ps1", "$resourceRoot/test-sovereign-cloud.ps1" -Destination $TestDrive + New-Item -ItemType Directory "$TestDrive/tests" | Out-Null + Copy-Item "$resourceRoot/tests/*.health.tests.ps1" "$TestDrive/tests" + New-Item -ItemType Directory "$TestDrive/before", "$TestDrive/after" | Out-Null + $before = New-LocalBoxAutomationPackage $TestDrive "$TestDrive/before" + Add-Content "$TestDrive/prepare-localbox.ps1" '# changed content' + $after = New-LocalBoxAutomationPackage $TestDrive "$TestDrive/after" + $before.Hash | Should -Not -Be $after.Hash + } +} + +Describe 'Guest readiness result contract' { + BeforeEach { + $result = @{ + RunId = 'run'; PackageHash = 'hash'; SubscriptionId = $subscription; ResourceGroupName = 'rg-localbox-shared' + AksAdminGroupObjectId = $group; State = 'Succeeded'; Upstream = $suite.Clone(); Sovereign = $suite.Clone() + UpdateAssessment = @{ Status = 'Succeeded'; Timestamp = '2026-11-02T00:00:00Z' } + } + } + It 'accepts the exact run and both successful test suites' { + Assert-LocalBoxAutomationResult $result run hash $group $subscription rg-localbox-shared + } + It 'rejects incorrect ' -TestCases @( + @{ Field = 'RunId' }; @{ Field = 'PackageHash' }; @{ Field = 'SubscriptionId' }; @{ Field = 'ResourceGroupName' } + @{ Field = 'AksAdminGroupObjectId' }; @{ Field = 'State' } + ) { + param($Field) + $result[$Field] = 'wrong' + { Assert-LocalBoxAutomationResult $result run hash $group $subscription rg-localbox-shared } | Should -Throw '*stale*' + } + It 'rejects failed, skipped, absent or empty suites' -TestCases @( + @{ Field = 'Passed'; Value = 0 }; @{ Field = 'Failed'; Value = 1 }; @{ Field = 'Skipped'; Value = 1 } + @{ Field = 'NotRun'; Value = 1 }; @{ Field = 'Failed'; Value = $null } + ) { + param($Field, $Value) + $result.Sovereign[$Field] = $Value + { Assert-LocalBoxAutomationResult $result run hash $group $subscription rg-localbox-shared } | Should -Throw '*tests*' + } + It 'requires a successful update assessment with a timestamp' { + $result.UpdateAssessment.Timestamp = $null + { Assert-LocalBoxAutomationResult $result run hash $group $subscription rg-localbox-shared } | Should -Throw '*assessment*' + } + It 'validates XML test cases rather than trusting a success-shaped summary' { + Set-Content "$TestDrive/good.xml" $xml + Assert-LocalBoxAutomationXml "$TestDrive/good.xml" $suite + Set-Content "$TestDrive/bad.xml" ($xml.Replace('Success', 'Failure')) + { Assert-LocalBoxAutomationXml "$TestDrive/bad.xml" $suite } | Should -Throw '*XML*' + Set-Content "$TestDrive/empty.xml" '' + { Assert-LocalBoxAutomationXml "$TestDrive/empty.xml" $suite } | Should -Throw '*XML*' + } + It 'rejects skipped upstream tests and returns real Pester counts' { + $pester = @{ TotalCount = 1; PassedCount = 1; FailedCount = 0; SkippedCount = 0; NotRunCount = 0 } + (Assert-LocalBoxAutomationTests $pester).Passed | Should -Be 1 + $pester.SkippedCount = 1 + { Assert-LocalBoxAutomationTests $pester } | Should -Throw '*skipped*' + } +} + +Describe 'Hosted LocalBox automation lifecycle' { + BeforeEach { + $tracking = @{ Body = $null; Polls = 0; Existing = $null; ExitCode = 0; ExecutionState = 'Succeeded' } + Mock Update-MhhToken {} + Mock Start-Sleep {} + Mock Wait-LocalBoxNestedDeployment {} + Mock New-LocalBoxAutomationPackage { @{ Hash = 'package'; Archive = 'UEs=' } } + Mock New-LocalBoxGuestCommand { 'guest code' } + Mock Get-AzAccessToken { + @{ Token = ('head.' + [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes("{`"oid`":`"$identity`"}")).TrimEnd('=') + '.signature') } + } + Mock Get-AzRoleAssignment { @{ RoleDefinitionName = 'Storage Blob Data Contributor' } } + Mock New-AzRoleAssignment {} + Mock Invoke-LocalBoxAutomationRequest { + if ($Path -like '*runCommands*') { + if ($Method -eq 'PUT') { $tracking.Body = $Body; return @{} } + if (-not $tracking.Body) { return $tracking.Existing } + $tracking.Polls++ + return @{ properties = @{ + provisioningState = 'Succeeded'; parameters = $tracking.Body.properties.parameters + instanceView = @{ executionState = $tracking.ExecutionState; exitCode = $tracking.ExitCode; startTime = [datetime]::UtcNow.ToString('o') } + } } + } + if ($Path -like '*containers*') { return @{} } + @{ location = 'spaincentral'; identity = @{ principalId = $identity } } + } + Mock Invoke-LocalBoxAutomationAz { + if ($Arguments[0..2] -join ' ' -eq 'storage account list') { + return @{ name = 'localbox123'; id = $storageId; primaryEndpoints = @{ blob = 'https://localbox123.blob.core.windows.net/' } } + } + if ($Arguments[0..2] -join ' ' -eq 'storage blob download') { + $file = $Arguments[[array]::IndexOf($Arguments, '--file') + 1] + if ($file.EndsWith('.xml')) { Set-Content $file $xml } + else { + $id = ($tracking.Body.properties.parameters | Where-Object name -eq RunId).value + @{ + RunId = $id; PackageHash = 'package'; AksAdminGroupObjectId = $group; SubscriptionId = $subscription + ResourceGroupName = 'rg-localbox-shared'; State = 'Succeeded'; Upstream = $suite; Sovereign = $suite + UpdateAssessment = @{ Status = 'Succeeded'; Timestamp = [datetime]::UtcNow.ToString('o') } + } | ConvertTo-Json -Depth 8 | Set-Content $file + } + return @{} + } + throw "Unexpected CLI request: $Arguments" + } + } + It 'submits an asynchronous bounded guest command and verifies both XML suites' { + $result = Invoke-LocalBoxHostedAutomation $subscription rg-localbox-shared $group + $tracking.Body.properties.asyncExecution | Should -BeTrue + $tracking.Body.properties.timeoutInSeconds | Should -Be 43200 + ($tracking.Body.properties.parameters | Where-Object name -eq GroupId).value | Should -Be $group + $result.ResultsUri | Should -Match 'https://localbox123.blob.core.windows.net/microhack-automation/' + Should -Invoke Wait-LocalBoxNestedDeployment -Times 1 + Should -Invoke New-AzRoleAssignment -Times 0 + Should -Invoke Invoke-LocalBoxAutomationAz -Times 4 -ParameterFilter { $Arguments[2] -eq 'download' } + } + It 'fails on nonzero, absent or failed guest execution results' -TestCases @( + @{ Code = 1; State = 'Succeeded' }; @{ Code = $null; State = 'Succeeded' }; @{ Code = 0; State = 'Failed' } + ) { + param($Code, $State) + $tracking.ExitCode = $Code + $tracking.ExecutionState = $State + { Invoke-LocalBoxHostedAutomation $subscription rg-localbox-shared $group } | Should -Throw '*LocalBox guest*' + Should -Invoke Invoke-LocalBoxAutomationAz -Times 0 -ParameterFilter { $Arguments[2] -eq 'download' } + } + It 'does not overwrite an active command from another revision or group' { + $tracking.Existing = @{ properties = @{ instanceView = @{ executionState = 'Running' }; parameters = @(@{ name = 'PackageHash'; value = 'old' }) } } + { Invoke-LocalBoxHostedAutomation $subscription rg-localbox-shared $group } | Should -Throw '*different content or group*' + Should -Invoke Invoke-LocalBoxAutomationRequest -Times 0 -ParameterFilter { $Method -eq 'PUT' } + } + It 'resumes an existing matching execution without restarting guest work' { + $id = [guid]::NewGuid().ToString() + $tracking.Existing = @{ properties = @{ provisioningState = 'Succeeded'; instanceView = @{ executionState = 'Running' } + parameters = @( + @{ name = 'RunId'; value = $id }; @{ name = 'PackageHash'; value = 'package' } + @{ name = 'GroupId'; value = $group }; @{ name = 'StartedAfter'; value = '2026-01-01T00:00:00Z' } + ) + } } + Mock Invoke-LocalBoxAutomationRequest { + if (-not $tracking.Body) { + $tracking.Body = $tracking.Existing + return $tracking.Existing + } + @{ properties = @{ + provisioningState = 'Succeeded'; parameters = $tracking.Body.properties.parameters + instanceView = @{ executionState = 'Succeeded'; exitCode = 0; startTime = [datetime]::UtcNow.ToString('o') } + } } + } -ParameterFilter { $Path -like '*runCommands*' -and $Method -ne 'PUT' } + (Invoke-LocalBoxHostedAutomation $subscription rg-localbox-shared $group).RunId | Should -Be $id + Should -Invoke Wait-LocalBoxNestedDeployment -Times 0 + Should -Invoke Invoke-LocalBoxAutomationRequest -Times 0 -ParameterFilter { $Method -eq 'PUT' -and $Path -like '*runCommands*' } + } + It 'ignores a previous execution instance view while a new command starts' { + Mock Invoke-LocalBoxAutomationRequest { + $tracking.Polls++ + $start = if ($tracking.Polls -eq 1) { '2020-01-01T00:00:00Z' } else { [datetime]::UtcNow.ToString('o') } + @{ properties = @{ + provisioningState = 'Succeeded'; parameters = $tracking.Body.properties.parameters + instanceView = @{ executionState = 'Succeeded'; exitCode = 0; startTime = $start } + } } + } -ParameterFilter { $Path -like '*runCommands*' -and $Method -ne 'PUT' -and $null -ne $tracking.Body } + Invoke-LocalBoxHostedAutomation $subscription rg-localbox-shared $group | Out-Null + $tracking.Polls | Should -Be 2 + Should -Invoke Start-Sleep -Times 1 + } +} + +Describe 'Guest default preparation and health orchestration' { + BeforeAll { + function Invoke-LocalBoxAz { param($Arguments, $TimeoutSeconds, [switch]$NoOutput) } + function Wait-SovereignCheck { param($TimeoutSeconds, $Check) } + function Get-LocalBoxReferenceVmReadiness { param($MachineId) } + function Invoke-TestPreparation { param($SubscriptionId, $ResourceGroupName, $AksAdminGroupObjectId) } + $guestFixture = Join-Path $TestDrive 'guest' + New-Item -ItemType Directory $guestFixture | Out-Null + @' +[CmdletBinding(SupportsShouldProcess)] +param($SubscriptionId, $ResourceGroupName, $AksAdminGroupObjectId, $ManifestPath) +Invoke-TestPreparation -SubscriptionId $SubscriptionId -ResourceGroupName $ResourceGroupName -AksAdminGroupObjectId $AksAdminGroupObjectId +'@ | Set-Content "$guestFixture/prepare-localbox.ps1" + } + BeforeEach { + $guestDirectory = Join-Path $TestDrive ([guid]::NewGuid().ToString()) + New-Item -ItemType Directory $guestDirectory | Out-Null + $guestConfig = @{ + RunId = [guid]::NewGuid().ToString(); PackageHash = 'content' + SubscriptionId = $subscription; ResourceGroupName = 'rg-localbox-shared'; AksAdminGroupObjectId = $group + StorageAccountName = 'localbox123'; Container = 'microhack-automation' + } + $oldSubscription = $env:subscriptionId + $oldResourceGroup = $env:resourceGroup + $oldProfile = $env:AZURE_CONFIG_DIR + $env:subscriptionId = $subscription + $env:resourceGroup = 'rg-localbox-shared' + $env:AZURE_CONFIG_DIR = 'caller-profile' + Mock Start-Transcript {} + Mock Stop-Transcript {} + Mock Invoke-TestPreparation {} + Mock Send-LocalBoxAutomationFiles {} + Mock Write-LocalBoxAutomationStatus {} + Mock Wait-SovereignCheck { & $Check } + Mock Get-LocalBoxReferenceVmReadiness { @{ PowerState = 'Running'; ArcStatus = 'Connected'; MdeProvisioningState = 'Succeeded' } } + Mock Invoke-LocalBoxAz { + if ($Arguments[0..1] -join ' ' -eq 'extension list') { return @{ name = 'connectedmachine' } } + if ($Arguments[0..1] -join ' ' -eq 'connectedmachine assess-patches') { + return @{ status = 'Succeeded'; lastModifiedDateTime = '2026-11-02T01:00:00Z'; assessmentActivityId = 'assessment' } + } + } + $manifestFile = Join-Path $guestDirectory 'manifest.json' + @{ AksPreparationSkipped = $false; AksAdminGroupObjectId = $group; ReferenceVm = @{ MachineId = 'reference' } } | + ConvertTo-Json -Depth 6 | Set-Content $manifestFile + Mock Test-Path { $true } -ParameterFilter { $LiteralPath -like 'C:\LocalBox\Tests\*' } + Mock Get-Module { @{ Version = [version]'5.7.1' } } -ParameterFilter { $Name -eq 'Pester' } + Mock Import-Module {} -ParameterFilter { $Name -eq 'Pester' } + Mock Invoke-Pester { @{ TotalCount = 1; PassedCount = 1; FailedCount = 0; SkippedCount = 0; NotRunCount = 0 } } + Mock Invoke-LocalBoxAutomationHealthProcess { + @{ FullReadiness = $true; Passed = 1; Failed = 0; Skipped = 0; NotRun = 0 } | ConvertTo-Json | Set-Content "$guestDirectory/health.json" + } + } + AfterEach { + $env:subscriptionId = $oldSubscription + $env:resourceGroup = $oldResourceGroup + $env:AZURE_CONFIG_DIR = $oldProfile + } + It 'prepares defaults with the exact event group and gates on assessment and both test suites' { + Invoke-LocalBoxGuestAutomation $guestConfig $guestDirectory $guestFixture 'input.json' -ManifestPath $manifestFile + Should -Invoke Invoke-TestPreparation -Times 1 -Exactly -ParameterFilter { + $AksAdminGroupObjectId -eq $group -and $SubscriptionId -eq $subscription + } + Should -Invoke Invoke-LocalBoxAz -Times 1 -ParameterFilter { $Arguments[0..1] -join ' ' -eq 'connectedmachine assess-patches' } + Should -Invoke Invoke-LocalBoxAutomationHealthProcess -Times 1 + Should -Invoke Send-LocalBoxAutomationFiles -Times 1 + $status = Get-Content "$guestDirectory/status.json" -Raw | ConvertFrom-Json + $status.State | Should -Be 'Succeeded' + $status.UpdateAssessment.Status | Should -Be 'Succeeded' + $status.DefenderCloudReporting | Should -Be 'NotVerified' + $env:AZURE_CONFIG_DIR | Should -Be 'caller-profile' + } + It 'records and uploads preparation failure without running tests or deleting the lab' { + Mock Invoke-TestPreparation { throw 'Storage validation failed' } + { Invoke-LocalBoxGuestAutomation $guestConfig $guestDirectory $guestFixture 'input.json' -ManifestPath $manifestFile } | Should -Throw '*Storage validation failed*' + $status = Get-Content "$guestDirectory/status.json" -Raw | ConvertFrom-Json + $status.State | Should -Be 'Failed' + $status.Phase | Should -Be 'Preparation' + Should -Invoke Send-LocalBoxAutomationFiles -Times 1 + Should -Invoke Invoke-LocalBoxAutomationHealthProcess -Times 0 + $env:AZURE_CONFIG_DIR | Should -Be 'caller-profile' + } + It 'fails on unsuccessful update assessment instead of claiming readiness' { + Mock Invoke-LocalBoxAz { @{ status = 'Failed' } } -ParameterFilter { $Arguments[0..1] -join ' ' -eq 'connectedmachine assess-patches' } + { Invoke-LocalBoxGuestAutomation $guestConfig $guestDirectory $guestFixture 'input.json' -ManifestPath $manifestFile } | Should -Throw '*assessment*' + Should -Invoke Invoke-LocalBoxAutomationHealthProcess -Times 0 + } + It 'propagates Full health process failure and cleans up the private kubeconfig' { + Set-Content "$guestDirectory/aks.kubeconfig" 'private test fixture' + Mock Invoke-LocalBoxAutomationHealthProcess { throw 'Full health failed' } + { Invoke-LocalBoxGuestAutomation $guestConfig $guestDirectory $guestFixture 'input.json' -ManifestPath $manifestFile } | Should -Throw '*Full health failed*' + Test-Path "$guestDirectory/aks.kubeconfig" | Should -BeFalse + (Get-Content "$guestDirectory/status.json" -Raw | ConvertFrom-Json).State | Should -Be 'Failed' + } + It 'fails explicitly if diagnostic upload is unavailable' { + Mock Send-LocalBoxAutomationFiles { throw 'Blob upload failed' } + { Invoke-LocalBoxGuestAutomation $guestConfig $guestDirectory $guestFixture 'input.json' -ManifestPath $manifestFile } | Should -Throw '*Blob upload failed*' + $env:AZURE_CONFIG_DIR | Should -Be 'caller-profile' + } +} diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md index 869d68ef4..472df68ee 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md @@ -45,6 +45,22 @@ required to validate regional service availability and effective inherited polic ## Global event LocalBox checks +Run `Invoke-Pester ./localbox-automation.tests.ps1,./global-localbox.tests.ps1 -Output Detailed` +for the hosted readiness gate. Offline tests cover token refresh on every poll, +nested readiness beyond ARM success, imported-content packaging, event-group +propagation, asynchronous execution, active-run resume/conflicts, stale instance +views, nonzero guest exit codes, run-correlated JSON/XML, default guest preparation, +update-assessment failure, credential-file cleanup and diagnostic-upload failures. +They do not exercise Windows SYSTEM, real managed Run Command, Blob RBAC propagation, +AKS admin credentials, or a multi-hour Console token rotation; those are live gates. + +Hosted automation runs upstream infrastructure tests and the existing LocalBox +Full suite **inside the Client**, without downloading a second copy of Sovereign +tests from `main`. The automation wrapper (not the read-only test runner) obtains +and later removes a private AKS admin kubeconfig and requests one reference-VM +update assessment. This is distinct from actual participant Entra access or +Defender cloud reporting. See [automation diagnostics and limits](../hosted-events/readme.md#complete-localbox-preparation). + Run `Invoke-Pester ./global-localbox.tests.ps1 -Output Detailed` for offline coverage of the Console's exact global-hook signature, cost/schema opt-in, one LocalBox deployment followed by two subscription preparations, all-event RBAC, From 6c2e4ab4b182649cc951f49babc7fff6daa7225e Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Mon, 2 Nov 2026 22:37:18 +0000 Subject: [PATCH 02/37] Fix LocalBox guest packaging for Console content mounts Resolve missing guest sources from a checksum-verified immutable revision and validate the package before deployment changes. Cover labautomation-only mounts, corrupt sources, and preflight failure propagation. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../labautomation/global-event-deploy.ps1 | 4 +- .../localbox-automation-source.json | 11 +++ .../labautomation/localbox-automation.ps1 | 66 +++++++++++++---- .../resources/hosted-events/readme.md | 32 +++++++-- .../resources/tests/global-localbox.tests.ps1 | 18 +++-- .../tests/localbox-automation.tests.ps1 | 72 +++++++++++++++++++ .../resources/tests/readme.md | 6 ++ 7 files changed, 183 insertions(+), 26 deletions(-) create mode 100644 03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation-source.json diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/global-event-deploy.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/global-event-deploy.ps1 index fac2ba263..985ae755c 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/global-event-deploy.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/global-event-deploy.ps1 @@ -38,6 +38,7 @@ foreach ($participant in $participants) { } } $groupCredentials = @(Get-LocalBoxConsoleGroupCredential) +$localBoxAutomationPackage = New-LocalBoxAutomationPackage $azureLocalResourceProviderAppId = '1412d89f-b8a8-4111-b4fd-e82905cbd85d' Update-MhhToken | Out-Null @@ -206,7 +207,8 @@ foreach ($participantObjectId in $participants) { Wait-LocalBoxDeployment -ResourceGroupName $localBoxResourceGroupName -DeploymentName $localBoxDeployment.DeploymentName $groupId = ($groupCredentials.HackboxCredential | Where-Object name -eq 'Lab Group ObjectId').value -$automation = Invoke-LocalBoxHostedAutomation -SubscriptionId $SubscriptionId -ResourceGroupName $localBoxResourceGroupName -AksAdminGroupObjectId $groupId +$automation = Invoke-LocalBoxHostedAutomation -SubscriptionId $SubscriptionId -ResourceGroupName $localBoxResourceGroupName ` + -AksAdminGroupObjectId $groupId -Package $localBoxAutomationPackage Write-Host "LocalBox preparation and health checks completed. Private results: $($automation.ResultsUri)" $groupCredentials diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation-source.json b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation-source.json new file mode 100644 index 000000000..b58ed3bb3 --- /dev/null +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation-source.json @@ -0,0 +1,11 @@ +{ + "repository": "janegilring/MicroHack", + "commit": "33e16b5d530c1578861066dcc8bfcd680a25447a", + "files": { + "prepare-localbox.ps1": "516c4f832b0b93b987ed595ea082aac3b27958c8ca12d8a1e73c0f353d36c75e", + "invoke-localbox-automation.ps1": "3d1af01aef3117b9325a9d116f1e51614b51c6c375939eaffc9dd3c3f02e4040", + "test-sovereign-cloud.ps1": "e23583ac0a3747261d57469a7ae3e31438c0cb70beb9bf1c751a6244e4ea5117", + "tests/localbox.health.tests.ps1": "92f922dff75b1bcecc7c3e9a523c613930bd1a538ef8bb0fd50dd2ee1267b4d9", + "tests/sovereign-lab.health.tests.ps1": "2916dd41ea0e53b4599bfb72a8f21ac6d152a476aff72593bd33e075fdde7ada" + } +} diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation.ps1 index d96435a46..f8b242e04 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation.ps1 @@ -72,22 +72,59 @@ function Wait-LocalBoxNestedDeployment { } function New-LocalBoxAutomationPackage { - param([string]$ResourceRoot, [string]$Directory) + param( + [string]$ResourceRoot = (Join-Path $PSScriptRoot '../resources'), + [string]$Directory, + [string]$SourceManifestPath = (Join-Path $PSScriptRoot 'localbox-automation-source.json') + ) + $ErrorActionPreference = 'Stop' $files = @('prepare-localbox.ps1', 'invoke-localbox-automation.ps1', 'test-sovereign-cloud.ps1', 'tests/localbox.health.tests.ps1', 'tests/sovereign-lab.health.tests.ps1') - $contents = [ordered]@{} - $package = Join-Path $Directory 'package' - New-Item -ItemType Directory -Path "$package/tests" -Force | Out-Null - foreach ($file in $files) { - $source = Join-Path $ResourceRoot $file - $contents[$file] = (Get-FileHash -LiteralPath $source -Algorithm SHA256 -ErrorAction Stop).Hash - Copy-Item -LiteralPath $source -Destination (Join-Path $package $file) -ErrorAction Stop + $ownsDirectory = -not $Directory + if ($ownsDirectory) { $Directory = Join-Path ([IO.Path]::GetTempPath()) "localbox-package-$([guid]::NewGuid())" } + try { + $contents = [ordered]@{} + $package = Join-Path $Directory 'package' + New-Item -ItemType Directory -Path "$package/tests" -Force | Out-Null + $missing = @($files | Where-Object { -not (Test-Path -LiteralPath (Join-Path $ResourceRoot $_) -PathType Leaf) }) + $sourceManifest = $null + if ($missing.Count) { + $sourceManifest = Get-Content -LiteralPath $SourceManifestPath -Raw | ConvertFrom-Json -AsHashtable + if ($sourceManifest.repository -notmatch '^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$' -or + $sourceManifest.commit -notmatch '^[a-fA-F0-9]{40}$' -or $sourceManifest.files.Count -ne $files.Count) { + throw 'LocalBox source manifest must identify one repository, immutable commit and the exact guest file set.' + } + foreach ($file in $files) { + if ($sourceManifest.files[$file] -notmatch '^[a-fA-F0-9]{64}$') { throw "Missing or invalid SHA256 for guest source $file." } + } + Write-Host "LocalBox guest sources are absent from the content mount ($($missing -join ', ')). Downloading the complete pinned set from $($sourceManifest.repository) at $($sourceManifest.commit)." + } + else { Write-Host 'Packaging LocalBox guest scripts from the local resources directory.' } + foreach ($file in $files) { + $destination = Join-Path $package $file + if ($sourceManifest) { + $base = "https://raw.githubusercontent.com/$($sourceManifest.repository)/$($sourceManifest.commit)/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources" + Invoke-WebRequest -Uri "$base/$file" -OutFile $destination -TimeoutSec 120 -ErrorAction Stop | Out-Null + } + else { + Copy-Item -LiteralPath (Join-Path $ResourceRoot $file) -Destination $destination -ErrorAction Stop + } + $contents[$file] = (Get-FileHash -LiteralPath $destination -Algorithm SHA256 -ErrorAction Stop).Hash + if ($sourceManifest -and $contents[$file] -ine $sourceManifest.files[$file]) { + throw "SHA256 mismatch for LocalBox guest source $file. No guest code was submitted." + } + } + $bytes = [Text.Encoding]::UTF8.GetBytes(($contents | ConvertTo-Json -Compress)) + $hash = [Convert]::ToHexString([Security.Cryptography.SHA256]::HashData($bytes)).ToLowerInvariant() + $archive = Join-Path $Directory 'package.zip' + Compress-Archive -Path "$package/*" -DestinationPath $archive -ErrorAction Stop + @{ Hash = $hash; Archive = [Convert]::ToBase64String([IO.File]::ReadAllBytes($archive)) } + } + finally { + if ($ownsDirectory -and (Test-Path -LiteralPath $Directory)) { + Remove-Item -LiteralPath $Directory -Recurse -Force -ErrorAction Stop + } } - $bytes = [Text.Encoding]::UTF8.GetBytes(($contents | ConvertTo-Json -Compress)) - $hash = [Convert]::ToHexString([Security.Cryptography.SHA256]::HashData($bytes)).ToLowerInvariant() - $archive = Join-Path $Directory 'package.zip' - Compress-Archive -Path "$package/*" -DestinationPath $archive -ErrorAction Stop - @{ Hash = $hash; Archive = [Convert]::ToBase64String([IO.File]::ReadAllBytes($archive)) } } function New-LocalBoxGuestCommand { @@ -169,6 +206,7 @@ function Invoke-LocalBoxHostedAutomation { [Parameter(Mandatory)][string]$ResourceGroupName, [Parameter(Mandatory)][guid]$AksAdminGroupObjectId, [string]$ResourceRoot = (Join-Path $PSScriptRoot '../resources'), + [hashtable]$Package, [int]$NestedTimeoutSeconds = 28800, [int]$GuestTimeoutSeconds = 43200 ) @@ -181,7 +219,7 @@ function Invoke-LocalBoxHostedAutomation { $temporary = Join-Path ([IO.Path]::GetTempPath()) "localbox-automation-$([guid]::NewGuid())" New-Item -ItemType Directory -Path $temporary | Out-Null try { - $package = New-LocalBoxAutomationPackage -ResourceRoot $ResourceRoot -Directory $temporary + if (-not $Package) { $Package = New-LocalBoxAutomationPackage -ResourceRoot $ResourceRoot -Directory $temporary } $existing = Invoke-LocalBoxAutomationRequest -Path "$commandUri&`$expand=instanceView" -AllowNotFound $active = $existing -and ($existing.properties.provisioningState -in @('Creating', 'Updating') -or $existing.properties.instanceView.executionState -in @('Pending', 'Running', 'Unknown') -or diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md index eb47e4f41..4fb1198d0 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md @@ -107,14 +107,19 @@ It does not publish LocalBox administrator credentials. The [orchestrator](../../labautomation/localbox-automation.ps1) implements: -1. Wait for `localbox-*` ARM completion, then `localcluster-deploy`, connected +1. Use the validated `Get-MhhDefaultLabGroup` result (also published as **Lab + Group ObjectId**) as `AksAdminGroupObjectId`. No guest Graph lookup is needed. +2. Package the preparation script, guest runner and health tests before making + deployment changes. A full checkout uses local resources. Console can mount + only `labautomation`, without its sibling `resources` directory; in that case + download the complete guest file set from the immutable commit in + [localbox-automation-source.json](../../labautomation/localbox-automation-source.json) + and verify every SHA256. Never mix partial local files with downloaded files + or fall back to a moving `main` branch. A content hash identifies the package, + which is reused in the Managed Run Command payload after ARM completion. +3. Wait for `localbox-*` ARM completion, then `localcluster-deploy`, connected Azure Local, a running Arc Resource Bridge, custom location and `hybridaksextension`. A successful Client bootstrap extension is not sufficient. -2. Use the validated `Get-MhhDefaultLabGroup` result (also published as **Lab - Group ObjectId**) as `AksAdminGroupObjectId`. No guest Graph lookup is needed. -3. Package the preparation script, guest runner and health tests from the - **imported content directory** into the Managed Run Command payload. A content - hash identifies the package; there is no second download from upstream `main`. 4. Run `sovereign-localbox-prepare` asynchronously on LocalBox-Client as **SYSTEM**, explicitly using PowerShell 7. The [guest runner](../invoke-localbox-automation.ps1) prepares storage, logical networks, AKS, proxy RBAC, MetalLB, the Windows @@ -157,6 +162,14 @@ allow time for both preparation and later assessments. ### Diagnostics and reruns +If an older run failed with `Could not find a part of the path +'/content/.../resources/prepare-localbox.ps1'`, it reached ARM completion but +could not package guest automation. Re-import the corrected branch (including +the source manifest) and retry; the global hook can reuse the existing LocalBox +deployment. Do not delete the environment to repair a missing content file. +Missing manifests, unavailable downloads and checksum mismatches now fail +preflight, before provisioning or changing subscription settings. + Results use a private `microhack-automation` container in the existing LocalBox staging storage account. The Console identity and Client identity receive **Storage Blob Data Contributor at that container only**. No SAS tokens, account @@ -185,6 +198,13 @@ not overwritten. After a completed/failed command, retrying starts a new tracked run and reuses preparation's resource checks; it does not delete the lab or reset existing VM passwords. Investigate failures before retrying. +When changing any packaged guest script or health test, publish those sources +first, then update the source manifest's repository, full commit SHA and file +digests together. The offline tests require the pin's digests to match the +checkout. Re-import the final content revision, not the intermediate source +commit. The feature branch currently pins the tested guest sources in the fork; +promotion upstream must retain an accessible immutable source revision. + ### Remaining event validation The automated gate verifies **infrastructure**, MDE extension installation and diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/global-localbox.tests.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/global-localbox.tests.ps1 index 9cb6f7f43..9f6e50126 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/global-localbox.tests.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/global-localbox.tests.ps1 @@ -27,7 +27,8 @@ Submit-TestLocalBox @PSBoundParameters function Get-AzRoleAssignment { param($ObjectId, $RoleDefinitionName, $RoleDefinitionId, $Scope) } function New-AzRoleAssignment { param($ObjectId, $RoleDefinitionName, $Scope) } function Wait-LocalBoxDeployment { param($ResourceGroupName, $DeploymentName) } - function Invoke-LocalBoxHostedAutomation { param($SubscriptionId, $ResourceGroupName, $AksAdminGroupObjectId) } + function Invoke-LocalBoxHostedAutomation { param($SubscriptionId, $ResourceGroupName, $AksAdminGroupObjectId, $Package) } + function New-LocalBoxAutomationPackage {} function Submit-TestLocalBox { param($SubscriptionId, $ResourceGroupName, $Location, $AzureLocalResourceProviderObjectId, $AzureLocalInstanceLocation, [switch]$UseConsoleCredentials, [switch]$NoWait) } @@ -75,6 +76,7 @@ Describe 'Event-wide LocalBox orchestration' { Mock Update-MhhToken {} Mock Get-AzContext { @{ Subscription = @{ Id = $state.Subscription } } } Mock Get-LocalBoxConsoleGroupCredential { @{ HackboxCredential = @{ name = 'Lab Group ObjectId'; value = '30000000-0000-0000-0000-000000000001' } } } + Mock New-LocalBoxAutomationPackage { $state.Events.Add('package'); @{ Hash = 'validated'; Archive = 'package' } } Mock Register-TestProviders { $state.Events.Add('providers') } Mock Initialize-MhhHostedTagPolicy { $state.Events.Add("tags:$SubscriptionId") } Mock Initialize-MhhHostedDefender { $state.Events.Add("defender:$SubscriptionId") } @@ -108,10 +110,10 @@ Describe 'Event-wide LocalBox orchestration' { $SubscriptionId -eq $state.Subscription -and $ResourceGroupName -eq 'rg-localbox-shared' -and $AzureLocalInstanceLocation -eq 'australiaeast' -and $UseConsoleCredentials -and $NoWait } - $state.Events -join ',' | Should -Be "providers,tags:$($state.Subscription),defender:$($state.Subscription),deploy:$($state.Subscription),wait,automation" + $state.Events -join ',' | Should -Be "package,providers,tags:$($state.Subscription),defender:$($state.Subscription),deploy:$($state.Subscription),wait,automation" Should -Invoke Invoke-LocalBoxHostedAutomation -Times 1 -Exactly -ParameterFilter { $SubscriptionId -eq $state.Subscription -and $ResourceGroupName -eq 'rg-localbox-shared' -and - $AksAdminGroupObjectId -eq '30000000-0000-0000-0000-000000000001' + $AksAdminGroupObjectId -eq '30000000-0000-0000-0000-000000000001' -and $Package.Hash -eq 'validated' } $state.Assignments.Count | Should -Be 6 foreach ($user in $users) { @@ -151,10 +153,11 @@ Describe 'Event-wide LocalBox orchestration' { Should -Invoke Submit-TestLocalBox -Times 1 -ParameterFilter { $Location -eq 'swedencentral' -and $SubscriptionId -eq $state.Subscription } } It 'stops without publishing discovery credentials when fails' -TestCases @( - @{ Stage = 'tags' }; @{ Stage = 'defender' }; @{ Stage = 'roles' }; @{ Stage = 'wait' }; @{ Stage = 'deploy' }; @{ Stage = 'graph' }; @{ Stage = 'automation' } + @{ Stage = 'tags' }; @{ Stage = 'defender' }; @{ Stage = 'roles' }; @{ Stage = 'wait' }; @{ Stage = 'deploy' }; @{ Stage = 'graph' }; @{ Stage = 'automation' }; @{ Stage = 'package' } ) { param($Stage) switch ($Stage) { + 'package' { Mock New-LocalBoxAutomationPackage { throw 'Test failure' } } 'tags' { Mock Initialize-MhhHostedTagPolicy { throw 'Test failure' } } 'defender' { Mock Initialize-MhhHostedDefender { throw 'Test failure' } } 'roles' { Mock Get-AzRoleAssignment { throw 'Test failure' } } @@ -167,7 +170,12 @@ Describe 'Event-wide LocalBox orchestration' { { & "$fixture/global-event-deploy.ps1" -SubscriptionId $state.Subscription -PreferredLocation spaincentral -AllowedEntraUserIds $users | ForEach-Object { $captured.Add($_) } } | Should -Throw $captured.Count | Should -Be 0 - if ($Stage -in @('tags', 'defender', 'graph')) { Should -Invoke Submit-TestLocalBox -Times 0 } + if ($Stage -in @('tags', 'defender', 'graph', 'package')) { Should -Invoke Submit-TestLocalBox -Times 0 } + if ($Stage -eq 'package') { + Should -Invoke Register-TestProviders -Times 0 + Should -Invoke Initialize-MhhHostedTagPolicy -Times 0 + Should -Invoke Initialize-MhhHostedDefender -Times 0 + } } It 'rejects the wrong subscription context before mutating resources' -TestCases @( @{ Hook = 'global-event-deploy.ps1' }; @{ Hook = 'shared-deploy-lab.ps1' } diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-automation.tests.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-automation.tests.ps1 index 21ec8a3a8..379254f3c 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-automation.tests.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-automation.tests.ps1 @@ -96,6 +96,7 @@ Describe 'Nested deployment gate' { } Describe 'Imported-content package and PS7 bootstrap' { + BeforeEach { Mock Invoke-WebRequest { throw 'Unexpected download for a complete local checkout.' } } It 'packages local content without fetching a moving GitHub branch' { $one = New-Item -ItemType Directory -Path "$TestDrive/one" $two = New-Item -ItemType Directory -Path "$TestDrive/two" @@ -127,6 +128,72 @@ Describe 'Imported-content package and PS7 bootstrap' { } } +Describe 'Console mount without guest resources' { + BeforeEach { + $originalGuestRoot = $resourceRoot + $mount = Join-Path $TestDrive ([guid]::NewGuid().ToString()) + New-Item -ItemType Directory "$mount/labautomation" | Out-Null + foreach ($file in @('localbox-automation.ps1', 'localbox-automation-source.json')) { + Copy-Item "$resourceRoot/../labautomation/$file" "$mount/labautomation" + } + $sourceManifestPath = "$mount/labautomation/localbox-automation-source.json" + $pin = Get-Content $sourceManifestPath -Raw | ConvertFrom-Json -AsHashtable + Mock Invoke-WebRequest { + $prefix = "https://raw.githubusercontent.com/$($pin.repository)/$($pin.commit)/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/" + $url = $Uri.AbsoluteUri + $url.StartsWith($prefix) | Should -BeTrue + Copy-Item -LiteralPath (Join-Path $originalGuestRoot $url.Substring($prefix.Length)) -Destination $OutFile + } + } + It 'keeps the pin synchronized with the guest sources in this checkout' { + $pin.commit | Should -Match '^[a-f0-9]{40}$' + $pin.files.Count | Should -Be 5 + foreach ($file in $pin.files.Keys) { + (Get-FileHash -LiteralPath (Join-Path $resourceRoot $file) -Algorithm SHA256).Hash | + Should -Be $pin.files[$file] + } + } + It 'packages the actual labautomation-only layout using default paths' { + . "$mount/labautomation/localbox-automation.ps1" + $package = New-LocalBoxAutomationPackage + $local = New-LocalBoxAutomationPackage -ResourceRoot $resourceRoot + $package.Hash | Should -Be $local.Hash + Test-Path "$mount/resources" | Should -BeFalse + Should -Invoke Invoke-WebRequest -Times 5 -Exactly + } + It 'uses the whole pinned set rather than mixing partial local files with downloads' { + New-Item -ItemType Directory "$mount/resources" | Out-Null + Set-Content "$mount/resources/prepare-localbox.ps1" 'incomplete checkout - must not be used' + $package = New-LocalBoxAutomationPackage -ResourceRoot "$mount/resources" -SourceManifestPath $sourceManifestPath + $local = New-LocalBoxAutomationPackage -ResourceRoot $resourceRoot + $package.Hash | Should -Be $local.Hash + Should -Invoke Invoke-WebRequest -Times 5 -Exactly + } + It 'rejects mutable refs or incomplete checksums before downloading' -TestCases @( + @{ Invalid = 'branch' }; @{ Invalid = 'checksum' }; @{ Invalid = 'extra-file' } + ) { + param($Invalid) + switch ($Invalid) { + 'branch' { $pin.commit = 'main' } + 'checksum' { $pin.files['prepare-localbox.ps1'] = 'bad' } + 'extra-file' { $pin.files['unwanted.ps1'] = '0' * 64 } + } + $pin | ConvertTo-Json -Depth 5 | Set-Content $sourceManifestPath + { New-LocalBoxAutomationPackage -ResourceRoot "$mount/resources" -SourceManifestPath $sourceManifestPath } | Should -Throw + Should -Invoke Invoke-WebRequest -Times 0 + } + It 'rejects downloaded content that does not match the pinned SHA256' { + Mock Invoke-WebRequest { Set-Content -LiteralPath $OutFile 'incorrect content' } + { New-LocalBoxAutomationPackage -ResourceRoot "$mount/resources" -SourceManifestPath $sourceManifestPath } | + Should -Throw '*SHA256 mismatch*' + } + It 'propagates download failures without a success-shaped package' { + Mock Invoke-WebRequest { throw 'Source download failed: HTTP 404' } + { New-LocalBoxAutomationPackage -ResourceRoot "$mount/resources" -SourceManifestPath $sourceManifestPath } | + Should -Throw '*HTTP 404*' + } +} + Describe 'Guest readiness result contract' { BeforeEach { $result = @{ @@ -230,6 +297,11 @@ Describe 'Hosted LocalBox automation lifecycle' { Should -Invoke New-AzRoleAssignment -Times 0 Should -Invoke Invoke-LocalBoxAutomationAz -Times 4 -ParameterFilter { $Arguments[2] -eq 'download' } } + It 'uses the already validated preflight package without downloading it again' { + Invoke-LocalBoxHostedAutomation $subscription rg-localbox-shared $group -Package @{ Hash = 'package'; Archive = 'UEs=' } | Out-Null + Should -Invoke New-LocalBoxAutomationPackage -Times 0 + Should -Invoke New-LocalBoxGuestCommand -Times 1 -ParameterFilter { $Archive -eq 'UEs=' } + } It 'fails on nonzero, absent or failed guest execution results' -TestCases @( @{ Code = 1; State = 'Succeeded' }; @{ Code = $null; State = 'Succeeded' }; @{ Code = 0; State = 'Failed' } ) { diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md index 472df68ee..9402860b5 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md @@ -54,6 +54,12 @@ update-assessment failure, credential-file cleanup and diagnostic-upload failure They do not exercise Windows SYSTEM, real managed Run Command, Blob RBAC propagation, AKS admin credentials, or a multi-hour Console token rotation; those are live gates. +Packaging tests also emulate the Console's `labautomation`-only mount, with no +sibling `resources` directory. They verify immutable source pins and SHA256 +checks, reject missing/mutable/corrupt sources, prevent mixing partial local +sources with downloads, and check that packaging failure stops the global hook +before Azure changes. Source-pin digests must match the checkout's guest files. + Hosted automation runs upstream infrastructure tests and the existing LocalBox Full suite **inside the Client**, without downloading a second copy of Sovereign tests from `main`. The automation wrapper (not the read-only test runner) obtains From 810f70ccca10b5b0d69f4709dc5481cec7c04410 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Mon, 2 Nov 2026 22:52:47 +0000 Subject: [PATCH 03/37] Provision kubectl for unattended LocalBox health checks Capture guest prerequisites inside diagnostics and install a private cluster-compatible client for SYSTEM without changing machine tools. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../resources/invoke-localbox-automation.ps1 | 62 +++++++++++++++++-- 1 file changed, 58 insertions(+), 4 deletions(-) diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/invoke-localbox-automation.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/invoke-localbox-automation.ps1 index 698c86f56..1c0ee4ae8 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/invoke-localbox-automation.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/invoke-localbox-automation.ps1 @@ -9,8 +9,46 @@ function Initialize-LocalBoxAutomationEnvironment { if (-not $value) { throw "LocalBox machine environment is missing $name." } [Environment]::SetEnvironmentVariable($name, $value, 'Process') } - $env:PATH = [Environment]::GetEnvironmentVariable('PATH', 'Machine') + ';' + $env:PATH - foreach ($command in @('az', 'kubectl', 'Get-VM')) { Get-Command $command -ErrorAction Stop | Out-Null } + $env:PATH = $env:PATH + ';' + [Environment]::GetEnvironmentVariable('PATH', 'Machine') + foreach ($command in @('az', 'Get-VM')) { Get-Command $command -ErrorAction Stop | Out-Null } +} + +function Get-LocalBoxKubectlClientVersion { + param([string]$Path) + $output = & $Path version --client --output json 2>&1 + if ($LASTEXITCODE -ne 0) { throw "kubectl client version check failed: $($output -join "`n")" } + $client = ($output -join "`n") | ConvertFrom-Json -AsHashtable -ErrorAction Stop + if ($client.clientVersion.gitVersion -notmatch '^v?(?\d+\.\d+\.\d+)(?:[-+][0-9A-Za-z.-]+)?$') { + throw 'kubectl did not report a valid client version.' + } + [version]$Matches.Version +} + +function Initialize-LocalBoxAutomationKubectl { + param([string]$InstanceId, [string]$Directory) + $instance = Get-LocalBoxResource $InstanceId + if ($instance.properties.kubernetesVersion -notmatch '^v?(?\d+\.\d+\.\d+)(?:[-+][0-9A-Za-z.-]+)?$') { + throw 'AKS did not report a valid Kubernetes version for kubectl installation.' + } + $version = [version]$Matches.Version + $existing = Get-Command kubectl -CommandType Application -ErrorAction SilentlyContinue + if ($existing) { + $client = Get-LocalBoxKubectlClientVersion -Path $existing.Path + if ($client.Major -eq $version.Major -and $client.Minor -eq $version.Minor) { return $existing.Path } + } + $tools = Join-Path (Split-Path -Parent $Directory) "tools/kubectl-$version" + New-Item -ItemType Directory -Path $tools -Force -ErrorAction Stop | Out-Null + $kubectl = Join-Path $tools 'kubectl.exe' + if (-not (Test-Path -LiteralPath $kubectl -PathType Leaf)) { + Write-Host "Installing kubectl $version for unattended SYSTEM health checks in $tools." + Invoke-LocalBoxAz @('aks', 'install-cli', '--client-version', $version.ToString(), + '--install-location', $kubectl, '--kubelogin-install-location', (Join-Path $tools 'kubelogin.exe')) -TimeoutSeconds 900 -NoOutput + } + if ((Get-LocalBoxKubectlClientVersion -Path $kubectl) -ne $version) { + throw "The private kubectl installation does not match requested version $version." + } + $env:PATH = $tools + [IO.Path]::PathSeparator + $env:PATH + return $kubectl } function Assert-LocalBoxAutomationTests { @@ -46,6 +84,7 @@ function Write-LocalBoxAutomationStatus { function Invoke-LocalBoxAutomationHealth { param([hashtable]$Configuration, [string]$Directory, [string]$ResourceRoot, [string]$ManifestPath) $nodeConfig = Import-PowerShellDataFile -LiteralPath $env:LocalBoxConfigFile + Get-Command kubectl -CommandType Application -ErrorAction Stop | Out-Null $credential = Resolve-LocalBoxNodeCredential -Configuration $nodeConfig $global:LASTEXITCODE = 0 & "$ResourceRoot/test-sovereign-cloud.ps1" -Scope LocalBox -Mode Full -DownloadTests:$false ` @@ -74,13 +113,16 @@ function Invoke-LocalBoxGuestAutomation { $lock = [Threading.Mutex]::new($false, 'Global\MicroHackLocalBoxAutomation') $locked = $false $transcribing = $false + $azureReady = $false $previousConfig = $env:AZURE_CONFIG_DIR $previousExtensions = $env:AZURE_EXTENSION_DIR + $previousPath = $env:PATH try { $locked = $lock.WaitOne(0) if (-not $locked) { throw 'Another hosted LocalBox automation is running.' } Start-Transcript -LiteralPath "$Directory/automation.log" -Force | Out-Null $transcribing = $true + Initialize-LocalBoxAutomationEnvironment if ($Configuration.SubscriptionId -ine $env:subscriptionId -or $Configuration.ResourceGroupName -ine $env:resourceGroup) { throw 'Automation scope does not match the LocalBox machine environment.' } @@ -92,6 +134,7 @@ function Invoke-LocalBoxGuestAutomation { $env:AZURE_EXTENSION_DIR = Join-Path (Split-Path $Directory) 'cliextensions' Invoke-LocalBoxAz @('login', '--identity') | Out-Null Invoke-LocalBoxAz @('account', 'set', '--subscription', $Configuration.SubscriptionId) | Out-Null + $azureReady = $true # Blob role assignments may still be propagating when the guest starts. Wait-SovereignCheck -TimeoutSeconds 600 -Check { Invoke-LocalBoxAz @('storage', 'blob', 'list', '--account-name', $Configuration.StorageAccountName, @@ -124,6 +167,9 @@ function Invoke-LocalBoxGuestAutomation { throw 'Reference VM Update Manager assessment did not return Succeeded with a timestamp.' } $status.UpdateAssessment = @{ Status = $assessment.status; Timestamp = $assessment.lastModifiedDateTime; ActivityId = $assessment.assessmentActivityId } + $status.Phase = 'KubernetesClient' + Write-LocalBoxAutomationStatus $Configuration $Directory $status + $status.KubectlPath = Initialize-LocalBoxAutomationKubectl -InstanceId $manifest.AksInstanceId -Directory $Directory $status.Phase = 'Pester' Write-LocalBoxAutomationStatus $Configuration $Directory $status $pester = Get-Module -ListAvailable Pester | Where-Object { $_.Version -ge [version]'5.7.1' -and $_.Version -lt [version]'6.0' } @@ -158,12 +204,20 @@ function Invoke-LocalBoxGuestAutomation { if ($locked) { $status.CompletedAt = [datetime]::UtcNow.ToString('o') $status | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath "$Directory/status.json" - Send-LocalBoxAutomationFiles $Configuration $Directory + if ($azureReady) { + try { Send-LocalBoxAutomationFiles $Configuration $Directory } + catch { + if ($status.Error) { throw "$($status.Error) Diagnostic upload also failed: $($_.Exception.Message)" } + throw + } + } + else { Write-Warning "Guest Azure authentication was not established. Diagnostics remain in $Directory and Run Command instanceView." } } } finally { $env:AZURE_CONFIG_DIR = $previousConfig $env:AZURE_EXTENSION_DIR = $previousExtensions + $env:PATH = $previousPath if ($locked) { foreach ($path in @("$Directory/aks.kubeconfig", "$Directory/.azure")) { if (Test-Path -LiteralPath $path) { Remove-Item -LiteralPath $path -Recurse -Force -ErrorAction Stop } @@ -178,13 +232,13 @@ function Invoke-LocalBoxGuestAutomation { if ($MyInvocation.InvocationName -ne '.') { $ErrorActionPreference = 'Stop' if (-not $IsWindows) { throw 'Hosted guest automation requires Windows LocalBox-Client.' } - Initialize-LocalBoxAutomationEnvironment $requestedManifestPath = $ManifestPath . "$PSScriptRoot/prepare-localbox.ps1" . "$PSScriptRoot/test-sovereign-cloud.ps1" $automation = Get-Content -LiteralPath $ParametersPath -Raw | ConvertFrom-Json -AsHashtable $directory = Split-Path -Parent $ParametersPath if ($HealthOnly) { + Initialize-LocalBoxAutomationEnvironment Invoke-LocalBoxAutomationHealth -Configuration $automation -Directory $directory -ResourceRoot $PSScriptRoot -ManifestPath $requestedManifestPath } else { From 6da1e0333201a2ef0d5fee00ba83ec8333ec290c Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Mon, 2 Nov 2026 22:53:20 +0000 Subject: [PATCH 04/37] Pin and test the SYSTEM-compatible LocalBox guest runner Cover private kubectl installation, version validation, retries and early diagnostics; update the immutable Console source pin and hosted-event documentation. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../localbox-automation-source.json | 4 +- .../resources/hosted-events/readme.md | 12 ++ .../tests/localbox-automation.tests.ps1 | 128 +++++++++++++++++- 3 files changed, 141 insertions(+), 3 deletions(-) diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation-source.json b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation-source.json index b58ed3bb3..b100a55d6 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation-source.json +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation-source.json @@ -1,9 +1,9 @@ { "repository": "janegilring/MicroHack", - "commit": "33e16b5d530c1578861066dcc8bfcd680a25447a", + "commit": "810f70ccca10b5b0d69f4709dc5481cec7c04410", "files": { "prepare-localbox.ps1": "516c4f832b0b93b987ed595ea082aac3b27958c8ca12d8a1e73c0f353d36c75e", - "invoke-localbox-automation.ps1": "3d1af01aef3117b9325a9d116f1e51614b51c6c375939eaffc9dd3c3f02e4040", + "invoke-localbox-automation.ps1": "3ba30929c9b9904254f0f30ffa4bcecc2e494ccbeb4309e376b9e9ff7930a29d", "test-sovereign-cloud.ps1": "e23583ac0a3747261d57469a7ae3e31438c0cb70beb9bf1c751a6244e4ea5117", "tests/localbox.health.tests.ps1": "92f922dff75b1bcecc7c3e9a523c613930bd1a538ef8bb0fd50dd2ee1267b4d9", "tests/sovereign-lab.health.tests.ps1": "2916dd41ea0e53b4599bfb72a8f21ac6d152a476aff72593bd33e075fdde7ada" diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md index 4fb1198d0..9e9abdf56 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md @@ -124,6 +124,14 @@ The [orchestrator](../../labautomation/localbox-automation.ps1) implements: explicitly using PowerShell 7. The [guest runner](../invoke-localbox-automation.ps1) prepares storage, logical networks, AKS, proxy RBAC, MetalLB, the Windows image and the default reference VM. + + SYSTEM does not inherit the interactive administrator's tools or profile. + Before Kubernetes health checks, the runner reuses an available `kubectl` with + the cluster's major/minor version, or uses `az aks install-cli` to install the + cluster's exact version in its private automation tools directory. The CLI + also installs its bundled `kubelogin` dependency there. Only the runner and its + child health process receive the private tools PATH; existing installations + and the machine PATH are not changed. Retries validate and reuse this client. 5. Wait for the reference VM to be Running, Arc Connected and MDE installation Succeeded. Request an Update Manager assessment and require a successful, timestamped result. This assesses updates; it does **not** install patches. @@ -181,6 +189,10 @@ Each run has a separate prefix containing `status.json`, `automation.log`, Status is updated at phase boundaries. Failures retain available diagnostics and resources; a failed upload also fails the command. Only the listed diagnostics are uploaded, never the AKS kubeconfig or reference-VM credential file. +Prerequisite checks run inside transcript/status capture. If failure occurs +before guest Azure authentication, diagnostics remain in the private run +directory on LocalBox-Client and the error is reported through Run Command +instanceView; no Blob upload is attempted without an authenticated guest. An organizer with Blob data access can download a run using: diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-automation.tests.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-automation.tests.ps1 index 379254f3c..cd19e5d82 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-automation.tests.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-automation.tests.ps1 @@ -353,6 +353,99 @@ Describe 'Hosted LocalBox automation lifecycle' { } } +Describe 'Unattended kubectl provisioning' { + BeforeAll { + function Invoke-LocalBoxAz { param($Arguments, $TimeoutSeconds, [switch]$NoOutput) } + function Get-LocalBoxResource { param($Id) } + } + BeforeEach { + $clientDirectory = Join-Path $TestDrive ([guid]::NewGuid().ToString()) + $clientRun = Join-Path $clientDirectory 'run' + $clientPath = Join-Path $clientDirectory 'tools/kubectl-1.31.5/kubectl.exe' + $originalPath = $env:PATH + Mock Get-LocalBoxResource { @{ properties = @{ kubernetesVersion = '1.31.5' } } } + Mock Get-Command { $null } -ParameterFilter { $Name -eq 'kubectl' } + Mock Get-LocalBoxKubectlClientVersion { [version]'1.31.5' } + Mock Invoke-LocalBoxAz {} + } + AfterEach { $env:PATH = $originalPath } + It 'installs a matching client privately when SYSTEM cannot resolve kubectl' { + Initialize-LocalBoxAutomationKubectl '/aks/instance' $clientRun | Should -Be $clientPath + Should -Invoke Get-LocalBoxResource -Times 1 -Exactly -ParameterFilter { $Id -eq '/aks/instance' } + Should -Invoke Invoke-LocalBoxAz -Times 1 -Exactly -ParameterFilter { + $Arguments[0..1] -join ' ' -eq 'aks install-cli' -and + $Arguments[$Arguments.IndexOf('--client-version') + 1] -eq '1.31.5' -and + $Arguments[$Arguments.IndexOf('--install-location') + 1] -eq $clientPath -and + $Arguments[$Arguments.IndexOf('--kubelogin-install-location') + 1] -eq (Join-Path (Split-Path $clientPath) 'kubelogin.exe') -and + $TimeoutSeconds -eq 900 -and $NoOutput + } + Should -Invoke Get-LocalBoxKubectlClientVersion -Times 1 -Exactly -ParameterFilter { $Path -eq $clientPath } + $env:PATH.Split([IO.Path]::PathSeparator)[0] | Should -Be (Split-Path $clientPath) + } + It 'reuses an existing same-minor client without upgrading or installing anything' { + Mock Get-Command { @{ Path = '/existing/kubectl.exe' } } -ParameterFilter { $Name -eq 'kubectl' } + Mock Get-LocalBoxKubectlClientVersion { [version]'1.31.4' } + Initialize-LocalBoxAutomationKubectl '/aks/instance' $clientRun | Should -Be '/existing/kubectl.exe' + Should -Invoke Invoke-LocalBoxAz -Times 0 + $env:PATH | Should -Be $originalPath + } + It 'does not overwrite an existing client from a different minor release' { + Mock Get-Command { @{ Path = '/existing/kubectl.exe' } } -ParameterFilter { $Name -eq 'kubectl' } + Mock Get-LocalBoxKubectlClientVersion { [version]'1.35.0' } -ParameterFilter { $Path -eq '/existing/kubectl.exe' } + Initialize-LocalBoxAutomationKubectl '/aks/instance' $clientRun | Should -Be $clientPath + Should -Invoke Invoke-LocalBoxAz -Times 1 -Exactly + } + It 'revalidates and reuses a private client on retry' { + New-Item -ItemType Directory -Path (Split-Path $clientPath) -Force | Out-Null + Set-Content $clientPath 'fixture' + Initialize-LocalBoxAutomationKubectl '/aks/instance' $clientRun | Should -Be $clientPath + Should -Invoke Invoke-LocalBoxAz -Times 0 + Should -Invoke Get-LocalBoxKubectlClientVersion -Times 1 -Exactly -ParameterFilter { $Path -eq $clientPath } + } + It 'never falls back to latest when the cluster version is missing or invalid' -TestCases @( + @{ Version = $null }; @{ Version = 'latest' }; @{ Version = '../other' } + ) { + param($Version) + Mock Get-LocalBoxResource { @{ properties = @{ kubernetesVersion = $Version } } } + { Initialize-LocalBoxAutomationKubectl '/aks/instance' $clientRun } | Should -Throw '*valid Kubernetes version*' + Should -Invoke Invoke-LocalBoxAz -Times 0 + } + It 'propagates installation errors without adding a broken directory to PATH' { + Mock Invoke-LocalBoxAz { throw 'Download failed' } + { Initialize-LocalBoxAutomationKubectl '/aks/instance' $clientRun } | Should -Throw '*Download failed*' + $env:PATH | Should -Be $originalPath + } + It 'rejects an incorrect installed version before running health tests' { + Mock Get-LocalBoxKubectlClientVersion { [version]'1.35.0' } + { Initialize-LocalBoxAutomationKubectl '/aks/instance' $clientRun } | Should -Throw '*does not match requested version*' + $env:PATH | Should -Be $originalPath + } +} + +Describe 'kubectl client version validation' { + BeforeAll { + function Test-KubectlProcess { + $global:LASTEXITCODE = $script:kubectlExit + $script:kubectlOutput + } + } + BeforeEach { $script:kubectlExit = 0; $script:kubectlOutput = '{"clientVersion":{"gitVersion":"v1.31.5"}}' } + It 'reads the actual client version' { + Get-LocalBoxKubectlClientVersion Test-KubectlProcess | Should -Be ([version]'1.31.5') + } + It 'rejects a failing executable' { + $script:kubectlExit = 1 + { Get-LocalBoxKubectlClientVersion Test-KubectlProcess } | Should -Throw '*client version check failed*' + } + It 'rejects malformed or missing version output' -TestCases @( + @{ Output = 'not JSON' }; @{ Output = '{}' }; @{ Output = '{"clientVersion":{"gitVersion":"latest"}}' } + ) { + param($Output) + $script:kubectlOutput = $Output + { Get-LocalBoxKubectlClientVersion Test-KubectlProcess } | Should -Throw + } +} + Describe 'Guest default preparation and health orchestration' { BeforeAll { function Invoke-LocalBoxAz { param($Arguments, $TimeoutSeconds, [switch]$NoOutput) } @@ -383,6 +476,8 @@ Invoke-TestPreparation -SubscriptionId $SubscriptionId -ResourceGroupName $Resou $env:AZURE_CONFIG_DIR = 'caller-profile' Mock Start-Transcript {} Mock Stop-Transcript {} + Mock Initialize-LocalBoxAutomationEnvironment {} + Mock Initialize-LocalBoxAutomationKubectl { 'private/kubectl.exe' } Mock Invoke-TestPreparation {} Mock Send-LocalBoxAutomationFiles {} Mock Write-LocalBoxAutomationStatus {} @@ -395,7 +490,7 @@ Invoke-TestPreparation -SubscriptionId $SubscriptionId -ResourceGroupName $Resou } } $manifestFile = Join-Path $guestDirectory 'manifest.json' - @{ AksPreparationSkipped = $false; AksAdminGroupObjectId = $group; ReferenceVm = @{ MachineId = 'reference' } } | + @{ AksPreparationSkipped = $false; AksAdminGroupObjectId = $group; AksInstanceId = '/aks/instance'; ReferenceVm = @{ MachineId = 'reference' } } | ConvertTo-Json -Depth 6 | Set-Content $manifestFile Mock Test-Path { $true } -ParameterFilter { $LiteralPath -like 'C:\LocalBox\Tests\*' } Mock Get-Module { @{ Version = [version]'5.7.1' } } -ParameterFilter { $Name -eq 'Pester' } @@ -417,6 +512,9 @@ Invoke-TestPreparation -SubscriptionId $SubscriptionId -ResourceGroupName $Resou } Should -Invoke Invoke-LocalBoxAz -Times 1 -ParameterFilter { $Arguments[0..1] -join ' ' -eq 'connectedmachine assess-patches' } Should -Invoke Invoke-LocalBoxAutomationHealthProcess -Times 1 + Should -Invoke Initialize-LocalBoxAutomationKubectl -Times 1 -Exactly -ParameterFilter { + $InstanceId -eq '/aks/instance' -and $Directory -eq $guestDirectory + } Should -Invoke Send-LocalBoxAutomationFiles -Times 1 $status = Get-Content "$guestDirectory/status.json" -Raw | ConvertFrom-Json $status.State | Should -Be 'Succeeded' @@ -424,6 +522,34 @@ Invoke-TestPreparation -SubscriptionId $SubscriptionId -ResourceGroupName $Resou $status.DefenderCloudReporting | Should -Be 'NotVerified' $env:AZURE_CONFIG_DIR | Should -Be 'caller-profile' } + It 'captures prerequisite failures locally even before Azure authentication' { + Mock Initialize-LocalBoxAutomationEnvironment { throw 'Get-VM unavailable' } + { Invoke-LocalBoxGuestAutomation $guestConfig $guestDirectory $guestFixture 'input.json' -ManifestPath $manifestFile } | Should -Throw '*Get-VM unavailable*' + $status = Get-Content "$guestDirectory/status.json" -Raw | ConvertFrom-Json + $status.State | Should -Be 'Failed' + $status.Phase | Should -Be 'Prerequisites' + $status.Error | Should -Be 'Get-VM unavailable' + Should -Invoke Start-Transcript -Times 1 + Should -Invoke Stop-Transcript -Times 1 + Should -Invoke Send-LocalBoxAutomationFiles -Times 0 + Should -Invoke Invoke-TestPreparation -Times 0 + } + It 'reports kubectl provisioning failures and restores the caller PATH' { + $callerPath = $env:PATH + Mock Initialize-LocalBoxAutomationKubectl { $env:PATH = 'broken'; throw 'kubectl install failed' } + { Invoke-LocalBoxGuestAutomation $guestConfig $guestDirectory $guestFixture 'input.json' -ManifestPath $manifestFile } | Should -Throw '*kubectl install failed*' + $env:PATH | Should -Be $callerPath + $status = Get-Content "$guestDirectory/status.json" -Raw | ConvertFrom-Json + $status.State | Should -Be 'Failed' + $status.Phase | Should -Be 'KubernetesClient' + Should -Invoke Send-LocalBoxAutomationFiles -Times 1 + Should -Invoke Invoke-LocalBoxAutomationHealthProcess -Times 0 + } + It 'preserves the original error if diagnostic upload also fails' { + Mock Invoke-TestPreparation { throw 'Preparation failed' } + Mock Send-LocalBoxAutomationFiles { throw 'Blob upload failed' } + { Invoke-LocalBoxGuestAutomation $guestConfig $guestDirectory $guestFixture 'input.json' -ManifestPath $manifestFile } | Should -Throw '*Preparation failed*Blob upload failed*' + } It 'records and uploads preparation failure without running tests or deleting the lab' { Mock Invoke-TestPreparation { throw 'Storage validation failed' } { Invoke-LocalBoxGuestAutomation $guestConfig $guestDirectory $guestFixture 'input.json' -ManifestPath $manifestFile } | Should -Throw '*Storage validation failed*' From 948c71fb412b29b64070457ddf5489bcc16cfb06 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 3 Nov 2026 18:56:34 +0000 Subject: [PATCH 05/37] Route LocalBox AKS health checks through the internal gateway Use a validated temporary ActiveStore API host route around the health child process, preserve existing routes and verify cleanup on success and failure. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../resources/invoke-localbox-automation.ps1 | 72 ++++++++++++++++++- 1 file changed, 71 insertions(+), 1 deletion(-) diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/invoke-localbox-automation.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/invoke-localbox-automation.ps1 index 1c0ee4ae8..2859168e6 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/invoke-localbox-automation.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/invoke-localbox-automation.ps1 @@ -99,6 +99,75 @@ function Invoke-LocalBoxAutomationHealthProcess { if ($LASTEXITCODE -ne 0) { throw 'Sovereign Cloud Full health tests failed. Inspect health.xml and health.json.' } } +function Invoke-LocalBoxAutomationRoutedHealth { + param([string]$ResourceRoot, [string]$InputPath, [string]$ManifestPath, [string]$Kubeconfig) + $ErrorActionPreference = 'Stop' + $config = Import-PowerShellDataFile -LiteralPath $env:LocalBoxConfigFile + foreach ($key in @('AKSControlPlaneIP', 'PhysicalHostInternalIP', 'SDNLABRoute')) { + $address = $null + if (-not [Net.IPAddress]::TryParse([string]$config[$key], [ref]$address) -or + $address.AddressFamily -ne [Net.Sockets.AddressFamily]::InterNetwork -or $address.ToString() -ne $config[$key]) { + throw "LocalBox configuration requires a canonical IPv4 address for $key." + } + } + $server = & kubectl --kubeconfig $Kubeconfig config view --minify -o 'jsonpath={.clusters[0].cluster.server}' + if ($LASTEXITCODE -ne 0) { throw 'Cannot read the AKS API endpoint from the private kubeconfig.' } + $endpoint = $null + if (-not [uri]::TryCreate([string]$server, [UriKind]::Absolute, [ref]$endpoint) -or + $endpoint.Scheme -ne 'https' -or $endpoint.Host -ne $config.AKSControlPlaneIP) { + throw 'The kubeconfig API endpoint does not match the configured LocalBox AKS control-plane IP.' + } + if ([string]::IsNullOrWhiteSpace($config.InternalSwitch)) { throw 'LocalBox configuration is missing InternalSwitch.' } + $interfaces = @(Get-NetIPAddress -AddressFamily IPv4 -InterfaceAlias "vEthernet ($($config.InternalSwitch))" -ErrorAction Stop | + Where-Object IPAddress -eq $config.PhysicalHostInternalIP) + if ($interfaces.Count -ne 1 -or $interfaces[0].PrefixLength -lt 1 -or $interfaces[0].PrefixLength -gt 30) { + throw 'Cannot uniquely resolve the configured LocalBox internal interface and subnet.' + } + $interface = $interfaces[0] + $size = [math]::Pow(2, 32 - $interface.PrefixLength) + $hostNumber = ConvertTo-LocalBoxIPv4Number $config.PhysicalHostInternalIP + $gatewayNumber = ConvertTo-LocalBoxIPv4Number $config.SDNLABRoute + if ([math]::Floor($hostNumber / $size) -ne [math]::Floor($gatewayNumber / $size) -or + $gatewayNumber -eq $hostNumber -or $gatewayNumber % $size -in @(0, ($size - 1))) { + throw 'The configured internal gateway is not a usable peer in the Client internal subnet.' + } + $prefix = "$($endpoint.Host)/32" + $existing = @(Get-NetRoute -AddressFamily IPv4 -PolicyStore ActiveStore -ErrorAction Stop | + Where-Object DestinationPrefix -eq $prefix) + if ($existing.Count -gt 1 -or ($existing.Count -eq 1 -and + ($existing[0].InterfaceIndex -ne $interface.InterfaceIndex -or $existing[0].NextHop -ne $config.SDNLABRoute))) { + throw "An ambiguous or conflicting route already exists for $prefix; it will not be replaced." + } + $ownedRoute = $null + $healthError = $null + try { + if (-not $existing.Count) { + Write-Host "Temporarily routing AKS API $prefix through $($config.SDNLABRoute) on $($interface.InterfaceAlias)." + $ownedRoute = New-NetRoute -DestinationPrefix $prefix -InterfaceIndex $interface.InterfaceIndex ` + -NextHop $config.SDNLABRoute -PolicyStore ActiveStore -ErrorAction Stop + } + else { Write-Host "Reusing the existing AKS API route for $prefix without modifying it." } + Invoke-LocalBoxAutomationHealthProcess -ResourceRoot $ResourceRoot -InputPath $InputPath -ManifestPath $ManifestPath + } + catch { $healthError = $_.Exception.Message; throw } + finally { + if ($ownedRoute) { + try { + $ownedRoute | Remove-NetRoute -Confirm:$false -ErrorAction Stop + $remaining = @(Get-NetRoute -AddressFamily IPv4 -PolicyStore ActiveStore -ErrorAction Stop | Where-Object { + $_.DestinationPrefix -eq $prefix -and $_.InterfaceIndex -eq $interface.InterfaceIndex -and $_.NextHop -eq $config.SDNLABRoute + }) + if ($remaining.Count) { throw "Temporary AKS API route $prefix remains after cleanup." } + Write-Host "Removed temporary AKS API route $prefix." + } + catch { + if ($healthError) { throw "$healthError Route cleanup also failed: $($_.Exception.Message)" } + throw + } + } + } +} + function Invoke-LocalBoxGuestAutomation { param([hashtable]$Configuration, [string]$Directory, [string]$ResourceRoot, [string]$InputPath, [string]$ManifestPath = 'C:\LocalBox\sovereign-localbox.json') @@ -186,7 +255,8 @@ function Invoke-LocalBoxGuestAutomation { Invoke-LocalBoxAz @('aksarc', 'get-credentials', '--subscription', $Configuration.SubscriptionId, '--resource-group', $Configuration.ResourceGroupName, '--name', 'localbox-aks', '--admin', '--file', "$Directory/aks.kubeconfig", '--overwrite-existing') -NoOutput - Invoke-LocalBoxAutomationHealthProcess -ResourceRoot $ResourceRoot -InputPath $InputPath -ManifestPath $ManifestPath + Invoke-LocalBoxAutomationRoutedHealth -ResourceRoot $ResourceRoot -InputPath $InputPath -ManifestPath $ManifestPath ` + -Kubeconfig "$Directory/aks.kubeconfig" $health = Get-Content -LiteralPath "$Directory/health.json" -Raw | ConvertFrom-Json -AsHashtable if (-not $health.FullReadiness) { throw 'Sovereign Cloud tests did not establish full infrastructure health.' } $status.Sovereign = @{ Passed = $health.Passed; Failed = $health.Failed; Skipped = $health.Skipped; NotRun = $health.NotRun } From f9e966ef1c11ef3359c5f29d2949deedf0ae3a16 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 3 Nov 2026 18:56:50 +0000 Subject: [PATCH 06/37] Pin and test temporary LocalBox AKS health routing Document route ownership and lifecycle and cover matching/conflicting routes, endpoint validation, interface selection and success/failure cleanup. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../localbox-automation-source.json | 4 +- .../resources/hosted-events/readme.md | 11 ++ .../tests/localbox-automation.tests.ps1 | 151 ++++++++++++++++++ 3 files changed, 164 insertions(+), 2 deletions(-) diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation-source.json b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation-source.json index b100a55d6..babe74e21 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation-source.json +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation-source.json @@ -1,9 +1,9 @@ { "repository": "janegilring/MicroHack", - "commit": "810f70ccca10b5b0d69f4709dc5481cec7c04410", + "commit": "948c71fb412b29b64070457ddf5489bcc16cfb06", "files": { "prepare-localbox.ps1": "516c4f832b0b93b987ed595ea082aac3b27958c8ca12d8a1e73c0f353d36c75e", - "invoke-localbox-automation.ps1": "3ba30929c9b9904254f0f30ffa4bcecc2e494ccbeb4309e376b9e9ff7930a29d", + "invoke-localbox-automation.ps1": "90efd7ba0d1973fa411ba9c037af23aeaa223bf9992fa65ff109810f73020edd", "test-sovereign-cloud.ps1": "e23583ac0a3747261d57469a7ae3e31438c0cb70beb9bf1c751a6244e4ea5117", "tests/localbox.health.tests.ps1": "92f922dff75b1bcecc7c3e9a523c613930bd1a538ef8bb0fd50dd2ee1267b4d9", "tests/sovereign-lab.health.tests.ps1": "2916dd41ea0e53b4599bfb72a8f21ac6d152a476aff72593bd33e075fdde7ada" diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md index 9e9abdf56..484045238 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md @@ -139,6 +139,17 @@ The [orchestrator](../../labautomation/localbox-automation.ps1) implements: Sovereign **LocalBox Full** suite. The Client retrieves a private AKS admin kubeconfig for infrastructure probes using its managed identity. It is not shared with participants and is deleted after the run. + During the Sovereign checks, the parent runner temporarily routes only the + private AKS API IPv4 address (`/32`) through the configured internal gateway + (`SDNLABRoute`) and internal switch interface. The endpoint must match + `AKSControlPlaneIP`; no default route, subnet route, firewall or persistent + routing setting is changed. An existing matching route is reused and left + intact; conflicting routes fail explicitly. A newly created route uses + Windows `ActiveStore` and is removed in `finally` after the health child + succeeds or fails. Removal is verified before readiness can pass. + If the parent process is forcibly terminated, its `finally` cannot run; + inspect the API host route before retrying rather than deleting existing + routes indiscriminately. ActiveStore routes do not survive a reboot. 7. Require a zero guest exit code, matching run/content/group/scope metadata, nonempty passing suites without skipped/not-run tests, and matching NUnit XML before publishing global discovery credentials and returning to Console. diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-automation.tests.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-automation.tests.ps1 index cd19e5d82..e624479d9 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-automation.tests.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-automation.tests.ps1 @@ -446,6 +446,151 @@ Describe 'kubectl client version validation' { } } +Describe 'Temporary AKS API routing for hosted health checks' { + BeforeAll { + . "$resourceRoot/prepare-localbox.ps1" + function kubectl {} + function Get-NetIPAddress { param($AddressFamily, $InterfaceAlias) } + function Get-NetRoute { param($AddressFamily, $PolicyStore) } + function New-NetRoute { param($DestinationPrefix, $InterfaceIndex, $NextHop, $PolicyStore) } + function Remove-NetRoute { + [CmdletBinding(SupportsShouldProcess)] + param([Parameter(ValueFromPipeline)]$InputObject) + process {} + } + } + BeforeEach { + $previousLocalConfig = $env:LocalBoxConfigFile + $env:LocalBoxConfigFile = "$TestDrive/route-config.psd1" + Set-Content $env:LocalBoxConfigFile '@{}' + $routeConfig = @{ + AKSControlPlaneIP = '10.10.0.5'; PhysicalHostInternalIP = '192.168.1.20' + SDNLABRoute = '192.168.1.1'; InternalSwitch = 'InternalSwitch' + } + $routeState = @{ Active = @(); HealthCalls = 0 } + $expectedRoute = [pscustomobject]@{ DestinationPrefix = '10.10.0.5/32'; InterfaceIndex = 42; NextHop = '192.168.1.1' } + Mock Import-PowerShellDataFile { $routeConfig } + Mock kubectl { $global:LASTEXITCODE = 0; 'https://10.10.0.5:6443' } + Mock Get-NetIPAddress { + [pscustomobject]@{ IPAddress = '192.168.1.20'; PrefixLength = 24; InterfaceIndex = 42; InterfaceAlias = 'vEthernet (InternalSwitch)' } + } + Mock Get-NetRoute { $routeState.Active } + Mock New-NetRoute { $routeState.Active = @($expectedRoute); $expectedRoute } + Mock Remove-NetRoute { $routeState.Active = @() } + Mock Invoke-LocalBoxAutomationHealthProcess { + $routeState.Active.Count | Should -Be 1 + $routeState.HealthCalls++ + } + } + AfterEach { $env:LocalBoxConfigFile = $previousLocalConfig } + It 'adds only a nonpersistent API host route while health checks run and verifies cleanup' { + Invoke-LocalBoxAutomationRoutedHealth 'source' 'input.json' 'manifest.json' 'private.kubeconfig' + Should -Invoke kubectl -Times 1 + Should -Invoke New-NetRoute -Times 1 -Exactly -ParameterFilter { + $DestinationPrefix -eq '10.10.0.5/32' -and $InterfaceIndex -eq 42 -and + $NextHop -eq '192.168.1.1' -and $PolicyStore -eq 'ActiveStore' + } + Should -Invoke Invoke-LocalBoxAutomationHealthProcess -Times 1 -Exactly -ParameterFilter { + $ResourceRoot -eq 'source' -and $InputPath -eq 'input.json' -and $ManifestPath -eq 'manifest.json' + } + Should -Invoke Remove-NetRoute -Times 1 -Exactly -ParameterFilter { $InputObject -eq $expectedRoute } + Should -Invoke Get-NetRoute -Times 2 -Exactly -ParameterFilter { $PolicyStore -eq 'ActiveStore' } + $routeState.Active.Count | Should -Be 0 + } + It 'cleans up an owned route when the health child fails' { + Mock Invoke-LocalBoxAutomationHealthProcess { throw 'Health child failed' } + { Invoke-LocalBoxAutomationRoutedHealth 'source' 'input' 'manifest' 'kubeconfig' } | Should -Throw '*Health child failed*' + Should -Invoke Remove-NetRoute -Times 1 + $routeState.Active.Count | Should -Be 0 + } + It 'preserves an existing matching route' { + $routeState.Active = @($expectedRoute) + Invoke-LocalBoxAutomationRoutedHealth 'source' 'input' 'manifest' 'kubeconfig' + Should -Invoke New-NetRoute -Times 0 + Should -Invoke Remove-NetRoute -Times 0 + $routeState.Active.Count | Should -Be 1 + } + It 'preserves an existing matching route even when tests fail' { + $routeState.Active = @($expectedRoute) + Mock Invoke-LocalBoxAutomationHealthProcess { throw 'Health child failed' } + { Invoke-LocalBoxAutomationRoutedHealth 'source' 'input' 'manifest' 'kubeconfig' } | Should -Throw '*Health child failed*' + Should -Invoke Remove-NetRoute -Times 0 + } + It 'rejects conflicting and ambiguous routes without replacing them' -TestCases @( + @{ Ambiguous = $false }; @{ Ambiguous = $true } + ) { + param($Ambiguous) + $routeState.Active = if ($Ambiguous) { @($expectedRoute, $expectedRoute) } else { + @([pscustomobject]@{ DestinationPrefix = '10.10.0.5/32'; InterfaceIndex = 5; NextHop = '172.16.1.1' }) + } + { Invoke-LocalBoxAutomationRoutedHealth 'source' 'input' 'manifest' 'kubeconfig' } | Should -Throw '*conflicting route*' + Should -Invoke New-NetRoute -Times 0 + Should -Invoke Remove-NetRoute -Times 0 + Should -Invoke Invoke-LocalBoxAutomationHealthProcess -Times 0 + } + It 'fails explicitly on route creation failure without deleting an unowned route' { + Mock New-NetRoute { throw 'Route creation denied' } + { Invoke-LocalBoxAutomationRoutedHealth 'source' 'input' 'manifest' 'kubeconfig' } | Should -Throw '*Route creation denied*' + Should -Invoke Remove-NetRoute -Times 0 + Should -Invoke Invoke-LocalBoxAutomationHealthProcess -Times 0 + } + It 'fails the gate if route removal fails' { + Mock Remove-NetRoute { throw 'Route removal denied' } + { Invoke-LocalBoxAutomationRoutedHealth 'source' 'input' 'manifest' 'kubeconfig' } | Should -Throw '*Route removal denied*' + } + It 'verifies removal rather than trusting a successful removal command' { + Mock Remove-NetRoute {} + { Invoke-LocalBoxAutomationRoutedHealth 'source' 'input' 'manifest' 'kubeconfig' } | Should -Throw '*remains after cleanup*' + } + It 'reports both health and cleanup failures' { + Mock Invoke-LocalBoxAutomationHealthProcess { throw 'Health child failed' } + Mock Remove-NetRoute { throw 'Route removal denied' } + { Invoke-LocalBoxAutomationRoutedHealth 'source' 'input' 'manifest' 'kubeconfig' } | Should -Throw '*Health child failed*Route removal denied*' + } + It 'rejects kubeconfig endpoints that do not match the configured private IPv4 API' -TestCases @( + @{ Endpoint = 'https://10.10.0.6:6443' }; @{ Endpoint = 'http://10.10.0.5:6443' }; @{ Endpoint = 'not a URI' } + ) { + param($Endpoint) + Mock kubectl { $global:LASTEXITCODE = 0; $Endpoint } + { Invoke-LocalBoxAutomationRoutedHealth 'source' 'input' 'manifest' 'kubeconfig' } | Should -Throw '*endpoint does not match*' + Should -Invoke New-NetRoute -Times 0 + } + It 'propagates kubeconfig inspection failure' { + Mock kubectl { $global:LASTEXITCODE = 1 } + { Invoke-LocalBoxAutomationRoutedHealth 'source' 'input' 'manifest' 'kubeconfig' } | Should -Throw '*Cannot read the AKS API endpoint*' + Should -Invoke New-NetRoute -Times 0 + } + It 'rejects invalid IP configuration' -TestCases @( + @{ Key = 'AKSControlPlaneIP'; Value = '10.10.0.0/24' } + @{ Key = 'PhysicalHostInternalIP'; Value = '::1' } + @{ Key = 'SDNLABRoute'; Value = '' } + ) { + param($Key, $Value) + $routeConfig[$Key] = $Value + { Invoke-LocalBoxAutomationRoutedHealth 'source' 'input' 'manifest' 'kubeconfig' } | Should -Throw '*canonical IPv4*' + Should -Invoke New-NetRoute -Times 0 + } + It 'rejects missing or ambiguous internal interfaces' -TestCases @( + @{ Count = 0 }; @{ Count = 2 } + ) { + param($Count) + Mock Get-NetIPAddress { + for ($i = 0; $i -lt $Count; $i++) { @{ IPAddress = '192.168.1.20'; PrefixLength = 24; InterfaceIndex = 42 } } + } + { Invoke-LocalBoxAutomationRoutedHealth 'source' 'input' 'manifest' 'kubeconfig' } | Should -Throw '*uniquely resolve*' + Should -Invoke New-NetRoute -Times 0 + } + It 'rejects unusable internal gateways' -TestCases @( + @{ Gateway = '172.16.1.1' }; @{ Gateway = '192.168.1.20' } + @{ Gateway = '192.168.1.0' }; @{ Gateway = '192.168.1.255' } + ) { + param($Gateway) + $routeConfig.SDNLABRoute = $Gateway + { Invoke-LocalBoxAutomationRoutedHealth 'source' 'input' 'manifest' 'kubeconfig' } | Should -Throw '*usable peer*' + Should -Invoke New-NetRoute -Times 0 + } +} + Describe 'Guest default preparation and health orchestration' { BeforeAll { function Invoke-LocalBoxAz { param($Arguments, $TimeoutSeconds, [switch]$NoOutput) } @@ -478,6 +623,9 @@ Invoke-TestPreparation -SubscriptionId $SubscriptionId -ResourceGroupName $Resou Mock Stop-Transcript {} Mock Initialize-LocalBoxAutomationEnvironment {} Mock Initialize-LocalBoxAutomationKubectl { 'private/kubectl.exe' } + Mock Invoke-LocalBoxAutomationRoutedHealth { + Invoke-LocalBoxAutomationHealthProcess -ResourceRoot $ResourceRoot -InputPath $InputPath -ManifestPath $ManifestPath + } Mock Invoke-TestPreparation {} Mock Send-LocalBoxAutomationFiles {} Mock Write-LocalBoxAutomationStatus {} @@ -512,6 +660,9 @@ Invoke-TestPreparation -SubscriptionId $SubscriptionId -ResourceGroupName $Resou } Should -Invoke Invoke-LocalBoxAz -Times 1 -ParameterFilter { $Arguments[0..1] -join ' ' -eq 'connectedmachine assess-patches' } Should -Invoke Invoke-LocalBoxAutomationHealthProcess -Times 1 + Should -Invoke Invoke-LocalBoxAutomationRoutedHealth -Times 1 -Exactly -ParameterFilter { + $Kubeconfig -eq "$guestDirectory/aks.kubeconfig" -and $ManifestPath -eq $manifestFile + } Should -Invoke Initialize-LocalBoxAutomationKubectl -Times 1 -Exactly -ParameterFilter { $InstanceId -eq '/aks/instance' -and $Directory -eq $guestDirectory } From 33f8f55cb70e262ef885976160b7af5058c71e54 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 3 Nov 2026 19:03:19 +0000 Subject: [PATCH 07/37] Show verified LocalBox test summaries in Console output Print actual suite counts, assessment status, scope and remaining manual checks after all readiness validations; explicitly report failed or unverified gates without claiming success. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../labautomation/localbox-automation.ps1 | 27 ++++++++++- .../resources/hosted-events/readme.md | 11 +++++ .../tests/localbox-automation.tests.ps1 | 47 +++++++++++++++++++ 3 files changed, 84 insertions(+), 1 deletion(-) diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation.ps1 index f8b242e04..e9d5ad2a1 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation.ps1 @@ -199,6 +199,17 @@ function Assert-LocalBoxAutomationXml { finally { $reader.Dispose() } } +function Write-LocalBoxAutomationSuiteSummary { + param([ValidateSet('Upstream', 'Sovereign')][string]$Name, [hashtable]$Counts) + foreach ($field in @('Passed', 'Failed', 'Skipped', 'NotRun')) { + $number = 0L + if (-not $Counts -or -not [long]::TryParse([string]$Counts[$field], [ref]$number) -or $number -lt 0) { + throw "Cannot display $Name test results: missing or invalid $field count." + } + } + Write-Host "[LocalBox] $Name tests: $($Counts.Passed) passed, $($Counts.Failed) failed, $($Counts.Skipped) skipped, $($Counts.NotRun) not run." +} + function Invoke-LocalBoxHostedAutomation { [CmdletBinding()] param( @@ -217,6 +228,7 @@ function Invoke-LocalBoxHostedAutomation { $commandPath = "$vmId/runCommands/sovereign-localbox-prepare" $commandUri = "$commandPath`?api-version=2024-07-01" $temporary = Join-Path ([IO.Path]::GetTempPath()) "localbox-automation-$([guid]::NewGuid())" + $resultUri = $null New-Item -ItemType Directory -Path $temporary | Out-Null try { if (-not $Package) { $Package = New-LocalBoxAutomationPackage -ResourceRoot $ResourceRoot -Directory $temporary } @@ -316,8 +328,21 @@ function Invoke-LocalBoxHostedAutomation { -SubscriptionId $SubscriptionId -ResourceGroupName $ResourceGroupName Assert-LocalBoxAutomationXml -Path "$temporary/health.xml" -Expected $result.Sovereign Assert-LocalBoxAutomationXml -Path "$temporary/upstream.xml" -Expected $result.Upstream - Write-Warning 'Automated infrastructure, MDE installation and update assessment passed. Defender cloud reporting and real participant sign-in still require verification.' + Write-Host '[LocalBox] Infrastructure health: PASSED' -ForegroundColor Green + Write-LocalBoxAutomationSuiteSummary -Name Upstream -Counts $result.Upstream + Write-LocalBoxAutomationSuiteSummary -Name Sovereign -Counts $result.Sovereign + Write-Host '[LocalBox] Reference VM: Running / Arc Connected / MDE installation succeeded.' + Write-Host "[LocalBox] Update assessment: $($result.UpdateAssessment.Status) ($($result.UpdateAssessment.Timestamp))." + Write-Host "[LocalBox] Private test reports: $resultUri (Entra authentication required)." + Write-Host '[LocalBox] Scope: shared LocalBox only; subsequent participant deployment stages are not covered by this result.' + Write-Warning '[LocalBox] Remaining manual checks: Defender cloud reporting and real participant sign-in.' @{ RunId = $runId; ResultsUri = $resultUri } } + catch { + Write-Host '[LocalBox] Readiness gate: FAILED - infrastructure health is NOT VERIFIED.' -ForegroundColor Red + Write-Host '[LocalBox] No successful end-of-run test summary was verified. See the error and diagnostic reports; do not treat deployment as ready.' + if ($resultUri) { Write-Host "[LocalBox] Private diagnostic reports: $resultUri (Entra authentication required)." } + throw + } finally { Remove-Item -LiteralPath $temporary -Recurse -Force -ErrorAction Stop } } diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md index 484045238..f64a99bf7 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md @@ -153,6 +153,17 @@ The [orchestrator](../../labautomation/localbox-automation.ps1) implements: 7. Require a zero guest exit code, matching run/content/group/scope metadata, nonempty passing suites without skipped/not-run tests, and matching NUnit XML before publishing global discovery credentials and returning to Console. + After these validations, Console Live Output prints **Infrastructure health: + PASSED**, actual upstream and Sovereign passed/failed/skipped/not-run counts, + reference-VM onboarding, the timestamped update-assessment result and the + private reports link. This summary covers shared LocalBox, not the subsequent + participant deployment stages. Defender cloud reporting and participant + sign-in remain explicitly listed as manual checks. + A failed execution, unavailable report or failed validation instead prints + **Readiness gate: FAILED - infrastructure health is NOT VERIFIED** and + propagates the error. It does not invent test counts or print a passing + summary when reports could not be verified; inspect the private diagnostics + for individual failed tests. These summaries do not stream guest progress. `Update-MhhToken` runs in the **Console process** on each poll and before Azure operations, including result retrieval. It refreshes Console clients, not guest diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-automation.tests.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-automation.tests.ps1 index e624479d9..8ce8abed3 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-automation.tests.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-automation.tests.ps1 @@ -241,10 +241,34 @@ Describe 'Guest readiness result contract' { } } +Describe 'Console LocalBox test summary formatting' { + BeforeEach { Mock Write-Host {} } + It 'prints supplied counts rather than fixed expected totals' { + Write-LocalBoxAutomationSuiteSummary -Name Sovereign -Counts @{ Passed = 27; Failed = 2; Skipped = 1; NotRun = 3 } + Should -Invoke Write-Host -Times 1 -Exactly -ParameterFilter { + $Object -eq '[LocalBox] Sovereign tests: 27 passed, 2 failed, 1 skipped, 3 not run.' + } + } + It 'does not emit success-stream objects that change the orchestration return value' { + @(Write-LocalBoxAutomationSuiteSummary -Name Upstream -Counts $suite).Count | Should -Be 0 + } + It 'rejects missing, negative and nonnumeric counts without printing fabricated totals' -TestCases @( + @{ Value = $null }; @{ Value = -1 }; @{ Value = 'unknown' } + ) { + param($Value) + $counts = $suite.Clone() + $counts.Failed = $Value + { Write-LocalBoxAutomationSuiteSummary -Name Sovereign -Counts $counts } | Should -Throw '*invalid Failed count*' + Should -Invoke Write-Host -Times 0 + } +} + Describe 'Hosted LocalBox automation lifecycle' { BeforeEach { $tracking = @{ Body = $null; Polls = 0; Existing = $null; ExitCode = 0; ExecutionState = 'Succeeded' } Mock Update-MhhToken {} + Mock Write-Host {} + Mock Write-Warning {} Mock Start-Sleep {} Mock Wait-LocalBoxNestedDeployment {} Mock New-LocalBoxAutomationPackage { @{ Hash = 'package'; Archive = 'UEs=' } } @@ -296,6 +320,27 @@ Describe 'Hosted LocalBox automation lifecycle' { Should -Invoke Wait-LocalBoxNestedDeployment -Times 1 Should -Invoke New-AzRoleAssignment -Times 0 Should -Invoke Invoke-LocalBoxAutomationAz -Times 4 -ParameterFilter { $Arguments[2] -eq 'download' } + Should -Invoke Write-Host -Times 1 -Exactly -ParameterFilter { $Object -eq '[LocalBox] Infrastructure health: PASSED' } + Should -Invoke Write-Host -Times 1 -Exactly -ParameterFilter { $Object -eq '[LocalBox] Upstream tests: 1 passed, 0 failed, 0 skipped, 0 not run.' } + Should -Invoke Write-Host -Times 1 -Exactly -ParameterFilter { $Object -eq '[LocalBox] Sovereign tests: 1 passed, 0 failed, 0 skipped, 0 not run.' } + Should -Invoke Write-Host -Times 1 -Exactly -ParameterFilter { $Object -like '[[]LocalBox] Update assessment: Succeeded*' } + Should -Invoke Write-Host -Times 1 -Exactly -ParameterFilter { $Object -like '*subsequent participant deployment stages are not covered*' } + Should -Invoke Write-Warning -Times 1 -Exactly -ParameterFilter { $Message -like '*Remaining manual checks*Defender cloud reporting*participant sign-in*' } + } + It 'does not claim readiness when report verification fails' -TestCases @( + @{ Gate = 'Assert-LocalBoxAutomationResult' }; @{ Gate = 'Assert-LocalBoxAutomationXml' } + ) { + param($Gate) + Mock $Gate { throw 'Report verification failed' } + { Invoke-LocalBoxHostedAutomation $subscription rg-localbox-shared $group } | Should -Throw '*Report verification failed*' + Should -Invoke Write-Host -Times 0 -ParameterFilter { $Object -like '*Infrastructure health: PASSED*' } + Should -Invoke Write-Host -Times 1 -Exactly -ParameterFilter { $Object -like '*Readiness gate: FAILED*NOT VERIFIED*' } + } + It 'reports an unavailable result download as unverified, not healthy' { + Mock Invoke-LocalBoxAutomationAz { throw 'Diagnostic download denied' } -ParameterFilter { $Arguments[2] -eq 'download' } + { Invoke-LocalBoxHostedAutomation $subscription rg-localbox-shared $group } | Should -Throw '*Diagnostic download denied*' + Should -Invoke Write-Host -Times 0 -ParameterFilter { $Object -like '*Infrastructure health: PASSED*' } + Should -Invoke Write-Host -Times 1 -Exactly -ParameterFilter { $Object -like '*Readiness gate: FAILED*NOT VERIFIED*' } } It 'uses the already validated preflight package without downloading it again' { Invoke-LocalBoxHostedAutomation $subscription rg-localbox-shared $group -Package @{ Hash = 'package'; Archive = 'UEs=' } | Out-Null @@ -310,6 +355,8 @@ Describe 'Hosted LocalBox automation lifecycle' { $tracking.ExecutionState = $State { Invoke-LocalBoxHostedAutomation $subscription rg-localbox-shared $group } | Should -Throw '*LocalBox guest*' Should -Invoke Invoke-LocalBoxAutomationAz -Times 0 -ParameterFilter { $Arguments[2] -eq 'download' } + Should -Invoke Write-Host -Times 0 -ParameterFilter { $Object -like '*Infrastructure health: PASSED*' } + Should -Invoke Write-Host -Times 1 -Exactly -ParameterFilter { $Object -like '*Readiness gate: FAILED*NOT VERIFIED*' } } It 'does not overwrite an active command from another revision or group' { $tracking.Existing = @{ properties = @{ instanceView = @{ executionState = 'Running' }; parameters = @(@{ name = 'PackageHash'; value = 'old' }) } } From 002c28716a4437e417f881ee7851a1c64c67b328 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Wed, 4 Nov 2026 15:58:31 +0000 Subject: [PATCH 08/37] Fix Console parsing of LocalBox result downloads Suppress Blob transfer progress, avoid parsing download metadata and separate native stderr from JSON while preserving failures. Verified against completed test-06 artifacts. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../labautomation/localbox-automation.ps1 | 21 ++++++++++--- .../resources/hosted-events/readme.md | 3 ++ .../tests/localbox-automation.tests.ps1 | 31 +++++++++++++++++++ 3 files changed, 51 insertions(+), 4 deletions(-) diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation.ps1 index e9d5ad2a1..0f2d98dd1 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation.ps1 @@ -1,9 +1,21 @@ function Invoke-LocalBoxAutomationAz { - param([Parameter(Mandatory)][string[]]$Arguments) + param([Parameter(Mandatory)][string[]]$Arguments, [switch]$NoOutput) Update-MhhToken | Out-Null - $output = & az @Arguments --only-show-errors --output json 2>&1 + $PSNativeCommandUseErrorActionPreference = $false + $format = if ($NoOutput) { 'none' } else { 'json' } + $output = @(& az @Arguments --only-show-errors --output $format 2>&1) if ($LASTEXITCODE -ne 0) { throw "LocalBox automation Azure CLI failed: $($output -join "`n")" } - if ($output) { ($output -join "`n") | ConvertFrom-Json -AsHashtable -ErrorAction Stop } + $stderr = @($output | Where-Object { $_ -is [System.Management.Automation.ErrorRecord] }) + foreach ($message in $stderr) { Write-Warning "$message" } + if ($NoOutput) { return } + $stdout = ($output | Where-Object { $_ -isnot [System.Management.Automation.ErrorRecord] }) -join "`n" + if (-not [string]::IsNullOrWhiteSpace($stdout)) { + try { $stdout | ConvertFrom-Json -AsHashtable -ErrorAction Stop } + catch { + $operation = ($Arguments | Select-Object -First 3) -join ' ' + throw "LocalBox automation Azure CLI 'az $operation' returned invalid JSON. Raw output is omitted because it may contain sensitive data." + } + } } function Invoke-LocalBoxAutomationRequest { @@ -321,7 +333,8 @@ function Invoke-LocalBoxHostedAutomation { } | Out-Null foreach ($name in @('status.json', 'health.json', 'health.xml', 'upstream.xml')) { Invoke-LocalBoxAutomationAz @('storage', 'blob', 'download', '--account-name', $account.name, '--container-name', $container, - '--name', "$runId/$name", '--file', (Join-Path $temporary $name), '--auth-mode', 'login', '--overwrite', 'true') | Out-Null + '--name', "$runId/$name", '--file', (Join-Path $temporary $name), '--auth-mode', 'login', '--overwrite', 'true', + '--no-progress') -NoOutput } $result = Get-Content -LiteralPath "$temporary/status.json" -Raw | ConvertFrom-Json -AsHashtable Assert-LocalBoxAutomationResult -Result $result -RunId $runId -PackageHash $package.Hash -GroupId $AksAdminGroupObjectId.ToString() ` diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md index f64a99bf7..0d3fff10f 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md @@ -164,6 +164,9 @@ The [orchestrator](../../labautomation/localbox-automation.ps1) implements: propagates the error. It does not invent test counts or print a passing summary when reports could not be verified; inspect the private diagnostics for individual failed tests. These summaries do not stream guest progress. + Result downloads suppress CLI progress and discard download-command metadata; + validation reads the downloaded JSON/XML files themselves. Native CLI stderr + is kept separate from JSON parsing, and nonzero CLI exits still fail the gate. `Update-MhhToken` runs in the **Console process** on each poll and before Azure operations, including result retrieval. It refreshes Console clients, not guest diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-automation.tests.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-automation.tests.ps1 index 8ce8abed3..d97cebb9a 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-automation.tests.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-automation.tests.ps1 @@ -43,6 +43,34 @@ Describe 'Token-aware LocalBox polling' { Mock az { $global:LASTEXITCODE = 0; 'not JSON' } { Invoke-LocalBoxAutomationAz @('resource', 'list') } | Should -Throw } + It 'separates native stderr progress from JSON and surfaces it as a warning' { + Mock Write-Warning {} + Mock az { + $global:LASTEXITCODE = 0 + [System.Management.Automation.ErrorRecord]::new([Exception]::new('Alive[###] 100%'), 'NativeCommandError', + [System.Management.Automation.ErrorCategory]::NotSpecified, $null) + '{"state":"Succeeded"}' + } + (Invoke-LocalBoxAutomationAz @('resource', 'show')).state | Should -Be 'Succeeded' + Should -Invoke Write-Warning -Times 1 -Exactly -ParameterFilter { $Message -like '*Alive*' } + } + It 'does not parse or return command metadata in explicit no-output mode' { + Mock az { $global:LASTEXITCODE = 0; 'Non-JSON command output' } + @(Invoke-LocalBoxAutomationAz @('storage', 'blob', 'download') -NoOutput).Count | Should -Be 0 + Should -Invoke az -Times 1 -Exactly -ParameterFilter { + $args[[array]::IndexOf($args, '--output') + 1] -eq 'none' + } + } + It 'still propagates failed commands in no-output mode' { + Mock az { $global:LASTEXITCODE = 1; 'AuthorizationPermissionMismatch' } + { Invoke-LocalBoxAutomationAz @('storage', 'blob', 'download') -NoOutput } | + Should -Throw '*AuthorizationPermissionMismatch*' + } + It 'adds operation context to malformed JSON without echoing raw output' { + Mock az { $global:LASTEXITCODE = 0; 'private diagnostic payload' } + $failure = { Invoke-LocalBoxAutomationAz @('resource', 'list') } | Should -Throw '*az resource list*invalid JSON*' -PassThru + $failure.Exception.Message | Should -Not -Match 'private diagnostic payload' + } It 'treats only an allowed 404 as absence' { Mock Invoke-AzRestMethod { @{ StatusCode = 404; Content = '' } } Invoke-LocalBoxAutomationRequest -Path '/test' -AllowNotFound | Should -BeNullOrEmpty @@ -320,6 +348,9 @@ Describe 'Hosted LocalBox automation lifecycle' { Should -Invoke Wait-LocalBoxNestedDeployment -Times 1 Should -Invoke New-AzRoleAssignment -Times 0 Should -Invoke Invoke-LocalBoxAutomationAz -Times 4 -ParameterFilter { $Arguments[2] -eq 'download' } + Should -Invoke Invoke-LocalBoxAutomationAz -Times 4 -Exactly -ParameterFilter { + $Arguments[2] -eq 'download' -and $Arguments -contains '--no-progress' -and $NoOutput + } Should -Invoke Write-Host -Times 1 -Exactly -ParameterFilter { $Object -eq '[LocalBox] Infrastructure health: PASSED' } Should -Invoke Write-Host -Times 1 -Exactly -ParameterFilter { $Object -eq '[LocalBox] Upstream tests: 1 passed, 0 failed, 0 skipped, 0 not run.' } Should -Invoke Write-Host -Times 1 -Exactly -ParameterFilter { $Object -eq '[LocalBox] Sovereign tests: 1 passed, 0 failed, 0 skipped, 0 not run.' } From e738a3d71674d9cc3c163bbeb02843251d286130 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Wed, 4 Nov 2026 18:51:52 +0000 Subject: [PATCH 09/37] Document LocalBox test-06 deployment timings and optimization options Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../hosted-events/test-06-timing-analysis.md | 228 ++++++++++++++++++ 1 file changed, 228 insertions(+) create mode 100644 03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/test-06-timing-analysis.md diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/test-06-timing-analysis.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/test-06-timing-analysis.md new file mode 100644 index 000000000..a5303dd82 --- /dev/null +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/test-06-timing-analysis.md @@ -0,0 +1,228 @@ +# LocalBox test-06: deployment timing and optimization assessment + +## Result and scope + +This report reconstructs the **4 November 2026** run from the live LocalBox, +Azure Resource Manager (ARM), Activity Log, original guest reports, and Console +completion screenshot. It is a single cold-deployment observation, not an SLA. + +| Item | Observed value | +|---|---| +| Console event | `sov-cloud-full-automation-test-06` | +| Environment | `traininglab-23`, resource group `rg-localbox-shared` | +| Guest run ID | `c654cb11-7935-4a68-901a-5864c3ef4e7e` | +| Console start / finish | 05:21:32 / 15:52:16 | +| **Console elapsed time** | **10h 30m 44s (37,844 seconds)** | +| Managed guest start / finish, UTC | 10:27:47.0086285 / 15:51:56.2326884 | +| **Managed guest elapsed time** | **5h 24m 9.224s** | +| Guest outcome | Succeeded, exit code 0; upstream 12/12 and Sovereign 14/14 passed | +| Console outcome | Failed while retrieving guest results; subsequent shared and participant stages were skipped | + +**This is not a successful end-to-end event-deployment benchmark.** The guest +finished successfully, but the Console Azure CLI wrapper attempted to parse +Blob-download progress output as JSON. The resulting failure occurred after the +expensive provisioning work. The host-only fix was subsequently published in +commit `002c28716a4437e417f881ee7851a1c64c67b328` and verified against this run's +actual reports. That verification does not retroactively make this Console job +successful, and a retry is a separate measurement. + +ARM and guest timestamps are UTC. The Console screenshot does not label its +timezone; its displayed times align with the UTC milestones and are treated as +UTC in the combined timeline below. The screenshot's elapsed duration does not +depend on that timezone assumption. + +## Additive timeline + +Boundaries are rounded to the nearest second; each duration is the difference +between those rounded boundaries. Consequently, these rows sum to exactly +**10:30:44**, while a duration calculated from the original fractional timestamps +can differ by a second. + +- **M**: measured operation/report interval. ARM deployment starts are derived + from the deployment's terminal `properties.timestamp` minus `properties.duration`. +- **B**: bounded interval between observed milestones. The elapsed time is known, + but its individual activities are not separately timed; it is not necessarily + idle time or time spent exclusively on the named activity. + +| Start | End | Duration | Work / interval | Evidence | +|---|---|---|---|---| +| 05:21:32 | 05:29:33 | 00:08:01 | Console setup and preflight before initial ARM deployment | B | +| 05:29:33 | 05:49:13 | 00:19:40 | Initial LocalBox ARM deployment | M | +| 05:49:13 | 07:03:43 | 01:14:30 | Post-ARM bootstrap and nested preparation before validation | B | +| 07:03:43 | 07:50:07 | 00:46:24 | Azure Local validation deployment | M | +| 07:50:07 | 07:51:49 | 00:01:42 | Validation-to-deployment transition | B | +| 07:51:49 | 10:26:14 | 02:34:25 | Azure Local cluster deployment | M | +| 10:26:14 | 10:27:47 | 00:01:33 | Readiness detection and managed guest launch | B | +| 10:27:47 | 10:34:51 | 00:07:04 | Guest startup, storage/logical networks and AKS pre-deployment work | B | +| 10:34:51 | 11:45:50 | 01:10:59 | AKS deployment | M | +| 11:45:50 | 11:49:18 | 00:03:28 | AKS post-deployment checks, proxy access/MetalLB and image submission | B | +| 11:49:18 | 14:48:02 | 02:58:44 | Windows Server marketplace image import | M | +| 14:48:02 | 14:48:33 | 00:00:31 | Image-to-reference-VM deployment transition | B | +| 14:48:33 | 14:57:03 | 00:08:30 | Reference VM ARM deployment | M | +| 14:57:03 | 15:41:35 | 00:44:32 | Reference VM completion to MDE extension submission; includes Arc/readiness work | B | +| 15:41:35 | 15:44:47 | 00:03:12 | MDE extension write to successful completion | M | +| 15:44:47 | 15:45:04 | 00:00:17 | Onboarding detection and assessment submission | B | +| 15:45:04 | 15:47:12 | 00:02:08 | Update assessment request to successful result timestamp | M | +| 15:47:12 | 15:48:40 | 00:01:28 | Assessment return, kubectl preparation and test startup | B | +| 15:48:40 | 15:49:02 | 00:00:22 | Upstream tests through XML report write | M | +| 15:49:02 | 15:49:40 | 00:00:38 | Sovereign health startup, including routing/client setup | B | +| 15:49:40 | 15:51:40 | 00:02:00 | Sovereign tests through final health report write | M | +| 15:51:40 | 15:51:56 | 00:00:16 | Finalization and managed command completion | B | +| 15:51:56 | 15:52:16 | 00:00:20 | Console completion detection and failed result retrieval | B | +| | **Total** | **10:30:44** | No downstream shared/participant deployment time included | | + +Before the managed guest started, **5h 6m 15s** had elapsed. The managed guest +then ran for **5h 24m 9s**, followed by approximately **20 seconds** before the +Console job ended. + +## What the detailed evidence establishes + +### Initial ARM and nested Azure Local + +The initial deployment, `localbox-20261104-052907`, took +**19m 40.349s**. Its network module took **9m 32.406s**, host VM module +**9m 52.148s**, staging storage module **31.626s**, and management/policy module +**24.649s**. These modules are inside the parent deployment and can overlap: +**do not add them to the timeline again**. + +Initial ARM success was not nested-cluster readiness. Bootstrap log evidence +helps characterize the following interval: + +| Evidence | UTC timestamps | Interpretation | +|---|---|---| +| Bootstrap transcript/file | 05:41:21 to 05:48:49.646 | Runs inside initial ARM provisioning | +| WinGet provisioning log creation/last write | 05:49:50.508 to 05:58:02.147 | About 8m12s of log lifetime; not a precise exclusive installation timer | +| `New-LocalBoxCluster.log` transcript start | 05:58:28 | Nested cluster workflow had started | +| `Generate-ARM-Template.log` transcript start | 06:50:06 | Template-generation workflow had started | +| Validation deployment | Derived start 07:03:43.061; completion 07:50:06.713 | **46m23.652s** | +| Cluster deployment | Derived start 07:51:48.732; completion 10:26:14.224 | **2h34m25.491s** | + +The cluster deployment's `clusters/deploymentSettings` operation accounts for +**2h34m12.476s** of its elapsed time. Role assignments, secrets and ancillary +resources took seconds and overlap that work. This points to the upstream +cluster deployment workflow, not those small ARM resources, as the major cost. +The collected logs do not establish exclusive durations for every internal +cluster action; further subdivision requires the upstream deployment's +step-level telemetry. + +### AKS and image preparation + +AKS deployment `pciDeployment-localbox-aks` took **1h10m59.340s**. +Its derived start, 10:34:51.026, agrees within about one second with the Activity +Log request start at 10:34:50.010. The terminal deployment timestamp is +11:45:50.366. An early successful ARM write/acceptance event is not evidence that +the whole AKS deployment has completed. + +The Windows Server image `2025-datacenter-azure-edition-smalldisk-01` had: + +- Initial write request: **11:49:18.2769539**. +- Resource creation: **11:49:19.5426047**. +- Successful write completion: **14:48:01.8995519**. +- Import interval: **2h58m43.623s**, approximately **28.3%** of the Console run. +- Reported download size: **16,654 MB**; final progress 100%, state Succeeded. + +Dividing the reported size by the full import interval gives approximately +**1.55 MB/s end-to-end**. This is **not a measured network transfer rate**: +the interval can include download, storage processing, registration and +provider/controller waits. It does not by itself prove a bandwidth bottleneck. +Repeated image-write events during progress updates are not separate imports. + +The current [preparation script](../prepare-localbox.ps1) runs AKS creation, +proxy access/MetalLB, image import, then reference VM creation **serially**. +That ordering is the clearest code-level optimization opportunity. + +### Defender and update assessment + +The reference VM ARM deployment completed at **14:57:02.908717**. The first +MDE extension write started at **15:41:34.8245541** and succeeded at +**15:44:46.5968808**: + +- VM ARM completion to MDE submission: **44m31.916s**. +- MDE submission to extension success: **3m11.772s**. +- Combined interval: **47m43.688s**. + +The first interval includes reference VM/Arc readiness as well as asynchronous +Defender onboarding scheduling. It is not a precisely measured pure Defender +queue time. Nevertheless, installation after submission was much shorter than +the preceding wait. Reinstalling the extension or tightening polling would not +address that larger interval. + +Update assessment started at **15:45:04.491061**. The successful assessment +result timestamp was **15:47:11.7283261**, an interval of **2m7.237s**. +The assessment's `WindowsOsUpdateExtension` provisioning occurred within this +interval and must not be added separately. The result timestamp is not an exact +measurement of when the CLI returned to the guest script. + +### Tests and finalization + +The NUnit reports record **20.7468 seconds** for the 12 upstream tests and +**118.5527 seconds** for the 14 Sovereign tests. Together that is +**2m19.300s**, only **0.37%** of the full Console duration. Test startup, XML/JSON +export, routing and finalization are additional, separately bounded intervals +in the timeline. + +The guest's final status timestamp was **15:51:40.0092555**; managed execution +ended at **15:51:56.2326884**. The final Console failure was therefore a +result-handling problem, not a provisioning or health-test timeout. + +## Optimization priorities + +These are recommendations, **not implemented runtime changes or promised +savings**. Preserve the final readiness gate and compare equivalent cold runs +before declaring an improvement. + +| Priority | Candidate | Potential benefit from this run | Constraints / verification | +|---|---|---|---| +| 1 | Start image import alongside AKS after shared storage/network prerequisites are ready | With unchanged operation durations, overlapping the 71-minute AKS deployment with the 179-minute image import could remove roughly **71 minutes** of serial waiting | Both use the same nested environment. Storage, CPU or network contention could reduce or eliminate the saving. Use controlled concurrency, isolated CLI state, explicit error handling, and wait for both branches before full readiness | +| 2 | Profile and improve the image-import path | Targets the longest individually measured preparation stage: **2h59m** | Separate actual download from storage processing/controller waits; collect supported disk/network/CPU telemetry. No evidence yet justifies a larger VM, arbitrary storage changes, or bypassing the supported import workflow | +| 3 | Pre-provision/reuse a validated shared LocalBox and existing image for an event | Moves substantial cold provisioning off the event's critical path; the existing preparation logic supports reuse | This is scheduling/reuse, not a faster cold deployment. Track ongoing cost, clean up prior participant access, verify configuration/image compatibility, and rerun current health/security checks. Measure warm runs separately | +| 4 | Overlap update assessment and independent infrastructure checks with MDE onboarding | Can hide a few minutes of work inside the roughly 48-minute post-reference interval | Start only after the reference VM is Arc Connected. Keep the final MDE-success requirement; the complete Sovereign suite currently depends on it. Do not equate extension success with verified Defender cloud reporting | +| 5 | Investigate upstream bootstrap, validation and cluster deployment step timings | Targets the 1h14m pre-validation interval, 46-minute validation and 2h34m cluster deployment | Collect per-step evidence first. Do not bypass supported validation or shorten retries just to improve the stopwatch | +| 6 | Add durable timestamped phase/subphase history and surface it in Console | Makes future bottlenecks and apparent stalls observable | Instrumentation improves evidence and UX, not provisioning speed. Record run ID, UTC start/end, elapsed time, outcome and reuse/create decisions; never credentials. Streaming progress remains separate from the already implemented final summary | + +An alternative experiment is to import the image and start the reference VM +before AKS, allowing Defender onboarding to overlap AKS work. This only helps if +the guest does not block on MDE before starting AKS, and it still needs benchmark +validation. It is an alternative/extension to the first recommendation, **not +an additional saving to add mechanically to its 71-minute estimate**. + +Splitting "Infrastructure Ready" from "Defender Ready" could permit some later +work to start earlier, but changes the hosted lifecycle and its readiness +contract. It must explicitly report pending security checks and gate +security-dependent exercises. It is not equivalent to making the existing Full +readiness check faster; simply deleting the MDE wait would still fail the +current reference-VM health test. + +Reducing polling intervals and optimizing Pester are low priority: the measured +test execution is under 0.4% of total time. The already-fixed Blob-output parsing +bug is important for reliability, but does not reduce the hours spent creating +resources. + +## Evidence and measurement limitations + +- ARM deployment metadata supplied terminal timestamps and durations; deployment + operation metadata identified the long-running cluster deployment-settings + operation. Child operation durations are not additional critical-path time. +- Read-only guest diagnostics read the original run's status, transcript, file + metadata and NUnit reports. Live diagnostic executions themselves are not part + of the original deployment timeline. +- Activity Log supplied initial image creation, terminal image completion, MDE + extension start/success and assessment request times. The initial resource-group + query hit its 1,000-event cap and covered only 12:03:51 to 15:59:14. Narrow + resource/time-window queries recovered the earlier image and AKS starts; the + capped export is **not** a complete history. +- The transcript records phase order but not a timestamp for every command; + final status overwrites earlier phases. File creation/last-write times are + supporting evidence, not exclusive phase timers. Bounded intervals must not be + relabeled as precisely timed individual operations. +- Raw transcripts and credentials are not published with this report. Use the + original run ID above to correlate retained private reports; do not substitute + artifacts from a later retry. +- No shared/participant stage ran after the global failure. Their duration, + participant-scale effects and complete event time remain unmeasured. + +For the next baseline, retain the final Console result and all guest reports, +record whether resources were created or reused, and capture phase history. +Benchmark one optimization at a time against the same region, host sizing, +image/version and readiness requirements. A warm retry of this environment is +useful functional verification, but is not a replacement cold-run timing sample. From 212ff976d7a124fd28c08b9b2675736ab2d40e21 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Wed, 4 Nov 2026 18:51:52 +0000 Subject: [PATCH 10/37] Use Developer Bastion for new LocalBox deployments Keep NAT outbound access and Challenge 7 Standard Bastion unchanged. Share the deployment path with manual setup and reject automatic dedicated-Bastion migrations. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../labautomation/deploy-localbox.ps1 | 39 ++- .../labautomation/localbox-network.bicep | 96 ++++++ .../resources/hosted-events/readme.md | 42 +++ .../manual-setup/localbox/deploy-localbox.ps1 | 275 +++--------------- .../tests/localbox-deployment.tests.ps1 | 203 +++++++++++++ .../tests/prepare-localbox.tests.ps1 | 10 +- .../resources/tests/readme.md | 9 + 7 files changed, 435 insertions(+), 239 deletions(-) create mode 100644 03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-network.bicep create mode 100644 03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-deployment.tests.ps1 diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/deploy-localbox.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/deploy-localbox.ps1 index e89a08685..e91c1a417 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/deploy-localbox.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/deploy-localbox.ps1 @@ -3,8 +3,8 @@ .SYNOPSIS Deploys the Azure Arc Jumpstart LocalBox environment used by the Sovereign Cloud lab. .DESCRIPTION -Downloads a coherent set of upstream LocalBox Bicep templates, validates them, and -deploys the LocalBox host, networking, storage, and optional Azure Bastion resources. +Downloads upstream LocalBox Bicep templates and substitutes the lab's Developer +Bastion network module, then validates and deploys the environment. .PARAMETER SubscriptionId The Azure subscription that receives the LocalBox resources. .PARAMETER ResourceGroupName @@ -21,6 +21,9 @@ It is not published as a participant credential. Coach/event-lead-only retrieval requires a supported mechanism from the Console owner. .PARAMETER GithubRef The azure_arc branch, tag, or commit used for both Bicep and runtime artifacts. +.PARAMETER DeployBastion +Deploy Bastion Developer for one browser-based coach connection at a time. +The host remains private and retains NAT Gateway outbound access when enabled. .PARAMETER AzureLocalResourceProviderObjectId Optional tenant-specific object ID of the Microsoft.AzureStackHCI enterprise application. The script resolves it through Microsoft Graph when omitted. @@ -105,6 +108,32 @@ function New-LocalBoxPassword { return -join ($characters | Sort-Object { Get-Random }) } +function Assert-LocalBoxNetworkCompatibility { + param([string]$SubscriptionId, [string]$ResourceGroupName) + + $bastions = @(Invoke-AzJson -Arguments @( + 'network', 'bastion', 'list', '--subscription', $SubscriptionId, '--resource-group', $ResourceGroupName + )) + foreach ($bastion in @($bastions | Where-Object name -eq 'LocalBox-Bastion')) { + if ($bastion.sku.name -ne 'Developer') { + throw "Existing LocalBox-Bastion SKU '$($bastion.sku.name)' cannot be changed to Developer in place. Use a fresh resource group or arrange an explicitly approved delete/recreate migration. No existing Bastion was changed." + } + } + $networks = @(Invoke-AzJson -Arguments @( + 'network', 'vnet', 'list', '--subscription', $SubscriptionId, '--resource-group', $ResourceGroupName + )) + $legacySubnets = @($networks | Where-Object name -eq 'LocalBox-VNet' | + ForEach-Object { $_.subnets } | Where-Object name -eq 'AzureBastionSubnet') + if ($legacySubnets.Count) { + throw 'LocalBox-VNet contains a legacy AzureBastionSubnet. Use a fresh resource group or arrange an explicitly approved migration; this deployment will not automatically remove an existing Bastion subnet.' + } +} + +$networkTemplateFile = Join-Path $PSScriptRoot 'localbox-network.bicep' +if (-not (Test-Path -LiteralPath $networkTemplateFile -PathType Leaf)) { + throw 'Missing localbox-network.bicep. Import the complete labautomation directory before deploying LocalBox.' +} + if (-not (Get-Command az -ErrorAction SilentlyContinue)) { throw 'Azure CLI is required to deploy LocalBox.' } @@ -254,6 +283,7 @@ if ($LASTEXITCODE -ne 0) { } if ($resourceGroupExists -eq 'true') { + Assert-LocalBoxNetworkCompatibility -SubscriptionId $SubscriptionId -ResourceGroupName $ResourceGroupName $resourceGroup = Invoke-AzJson -Arguments @( 'group', 'show', '--subscription', $SubscriptionId, '--name', $ResourceGroupName ) @@ -279,7 +309,6 @@ $bicepFiles = @( 'mgmt/mgmtArtifacts.bicep', 'mgmt/storageAccount.bicep', 'mgmt/customerUsageAttribution.bicep', - 'network/network.bicep', 'host/host.bicep' ) $deploymentName = "localbox-$(Get-Date -Format 'yyyyMMdd-HHmmss')" @@ -292,6 +321,10 @@ try { Write-Host "Downloading LocalBox template: $file" Invoke-WebRequest -Uri "$baseUri/$file" -OutFile (Join-Path $tempDirectory $file) -UseBasicParsing } + Copy-Item -LiteralPath $networkTemplateFile -Destination (Join-Path $tempDirectory 'network/network.bicep') -ErrorAction Stop + if ($DeployBastion) { + Write-Host 'Using Bastion Developer: one browser connection at a time; no native-client tunnels. Regional support is checked by ARM validation.' -ForegroundColor Cyan + } $resourceTags = @{ Project = 'jumpstart_LocalBox' diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-network.bicep b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-network.bicep new file mode 100644 index 000000000..e65c1cfd0 --- /dev/null +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-network.bicep @@ -0,0 +1,96 @@ +@description('Name of the LocalBox virtual network') +param virtualNetworkName string = 'LocalBox-VNet' + +param subnetName string = 'LocalBox-Subnet' +param location string = resourceGroup().location +param deployBastion bool = false +param natGatewayName string = 'LocalBox-NatGateway' +param networkSecurityGroupName string = 'LocalBox-NSG' +param resourceTags object + +resource networkSecurityGroup 'Microsoft.Network/networkSecurityGroups@2024-07-01' = { + name: networkSecurityGroupName + location: location + tags: resourceTags + properties: { + securityRules: [] + } +} + +resource natGatewayPublicIp 'Microsoft.Network/publicIPAddresses@2024-07-01' = if (deployBastion) { + name: '${natGatewayName}-PIP' + location: location + tags: resourceTags + sku: { + name: 'Standard' + } + properties: { + publicIPAllocationMethod: 'Static' + publicIPAddressVersion: 'IPv4' + idleTimeoutInMinutes: 4 + } +} + +resource natGateway 'Microsoft.Network/natGateways@2024-07-01' = if (deployBastion) { + name: natGatewayName + location: location + tags: resourceTags + sku: { + name: 'Standard' + } + properties: { + publicIpAddresses: [ + { + id: natGatewayPublicIp.id + } + ] + idleTimeoutInMinutes: 4 + } +} + +resource arcVirtualNetwork 'Microsoft.Network/virtualNetworks@2024-07-01' = { + name: virtualNetworkName + location: location + tags: resourceTags + properties: { + addressSpace: { + addressPrefixes: [ + '172.16.0.0/16' + ] + } + subnets: [ + { + name: subnetName + properties: { + addressPrefix: '172.16.1.0/24' + privateEndpointNetworkPolicies: 'Enabled' + privateLinkServiceNetworkPolicies: 'Enabled' + networkSecurityGroup: { + id: networkSecurityGroup.id + } + natGateway: deployBastion ? { + id: natGateway.id + } : null + defaultOutboundAccess: false + } + } + ] + } +} + +resource bastionHost 'Microsoft.Network/bastionHosts@2024-07-01' = if (deployBastion) { + name: 'LocalBox-Bastion' + location: location + tags: resourceTags + sku: { + name: 'Developer' + } + properties: { + virtualNetwork: { + id: arcVirtualNetwork.id + } + } +} + +output vnetId string = arcVirtualNetwork.id +output subnetId string = arcVirtualNetwork.properties.subnets[0].id diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md index 0d3fff10f..a5c811895 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md @@ -40,6 +40,39 @@ reference-VM security inspection. No participant Owner, host login or password i granted by this hook. Existing shared VM/AKS access remains a workshop trust boundary, not per-participant isolation. +### LocalBox Bastion + +New LocalBox deployments use **Azure Bastion Developer**, with no hourly Bastion +charge. It supports **one browser-based connection at a time**, intended for a +coach accessing LocalBox-Client. It does not support native-client tunnels, +concurrent sessions or peered-VNet access. Spain Central and Sweden Central are +supported; check the current [SKU and region matrix](https://learn.microsoft.com/azure/bastion/bastion-sku-comparison) +before choosing another host region. ARM validation remains authoritative; the +deployer does not silently substitute a paid SKU. + +The [local network module](../../labautomation/localbox-network.bicep) replaces +only the downloaded upstream network module. It preserves the LocalBox VNet, +Client subnet/NSG and NAT Gateway outbound path, but creates no dedicated Bastion +subnet, Bastion public IP or Bastion NSG. Keep this file beside +`deploy-localbox.ps1` when importing the Console package. The interactive manual +entry point delegates to the same deployer. Other Jumpstart templates and runtime +artifacts still come from the selected upstream revision. + +This change applies only to **LocalBox**. Challenge 7's separate Bastion remains +**Standard** because its K3s workflow requires `az network bastion tunnel`. +Developer avoids dedicated Bastion provisioning, but neither a deployment-time +improvement nor elimination of Azure service errors is guaranteed. + +Existing successful LocalBox deployments are reused without SKU changes. +Azure does not support an in-place downgrade from Basic/Standard to Developer. +The deployer rejects an existing dedicated `LocalBox-Bastion` or legacy +`AzureBastionSubnet` rather than deleting resources or rewriting that subnet. +Use a fresh event/resource group, or arrange a separate approved migration. +Re-importing this change alone will **not** repair a failed dedicated Bastion. +In particular, the failed Basic Bastion on traininglab-44 was left unchanged. + +### Participant access + Participant RBAC setup reuses an existing direct Security Reader assignment on its subscription, including the grant made by the global hook for participants in the LocalBox host subscription. It does not recreate the same grant under a @@ -193,6 +226,15 @@ automation waits separately for installation and assesses updates. See the [timing evidence and planning guidance](../localbox/readme.md#preparation-time); allow time for both preparation and later assessments. +For a full cold-run breakdown, see the +[test-06 timing and optimization assessment](test-06-timing-analysis.md). +The 4 November 2026 run took **10 hours 30 minutes 44 seconds** from Console +start to finish, including **5 hours 24 minutes 9 seconds** of managed guest +execution. All 26 guest tests passed, but Console result retrieval failed and +downstream shared/participant stages were skipped. This is not a successful +end-to-end event-duration benchmark; the report separates measured operations, +bounded intervals and potential optimizations. + ### Diagnostics and reruns If an older run failed with `Could not find a part of the path diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/manual-setup/localbox/deploy-localbox.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/manual-setup/localbox/deploy-localbox.ps1 index 68d8e823d..f032a4468 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/manual-setup/localbox/deploy-localbox.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/manual-setup/localbox/deploy-localbox.ps1 @@ -1,257 +1,64 @@ #Requires -Version 7.0 <# .SYNOPSIS - Deploy Azure Arc Jumpstart LocalBox environment for Azure Local. - +Interactively deploys Azure Arc Jumpstart LocalBox using the shared lab deployer. .DESCRIPTION - This script deploys the LocalBox environment from Azure Arc Jumpstart. - LocalBox provides a virtualized Azure Local environment for testing - Azure Local features without physical hardware. - - The deployment uses the official Arc Jumpstart Bicep templates from: - https://github.com/microsoft/azure_arc - - LocalBox includes: - - Virtualized Azure Local cluster (single-node) - - Arc Resource Bridge integration - - Custom Location for VM deployment - - VM Gallery Images - - AKS on Azure Local support (optional) - -.PARAMETER ResourceGroupName - The name of the resource group to deploy LocalBox to (will be created if it doesn't exist) - -.PARAMETER Location - Azure region for deployment (default: swedencentral) - -.PARAMETER AzureLocalInstanceLocation - Azure Local registration and staging storage region (default: australiaeast). - This lab default is independent of the Azure host region. - -.PARAMETER WindowsAdminUsername - Admin username for Windows VMs (default: arcdemo) - +Uses the same Developer Bastion network module and deployment validation as the +Console path, without enabling hosted subscription policies or Defender plans. +Required resource providers must already be registered. .PARAMETER DeployBastion - Deploy Azure Bastion for secure VM access (default: true) - -.PARAMETER DeployAKSHCI - Deploy AKS on Azure Local (default: false, adds significant deployment time) - - +Deploy Bastion Developer for one browser-based coach connection at a time. +Existing dedicated Bastions are not automatically downgraded or deleted. +.PARAMETER AzureLocalInstanceLocation +Azure Local registration and staging storage region, independent of the host region. .EXAMPLE - .\deploy-localbox.ps1 -ResourceGroupName "rg-localbox" -Location "swedencentral" - - -.NOTES - Author: MicroHack Team - Date: January 2026 - - Prerequisites: - - Azure CLI installed - - Sufficient vCPU quota (32 vCPUs, Standard_E32s_v6 or larger recommended) - - Required resource providers registered - - Deployment time: ~4-6 hours - - IMPORTANT: LocalBox requires significant compute resources. Ensure you have - adequate vCPU quota before deployment. - -.LINK - https://jumpstart.azure.com/azure_jumpstart_localbox -.LINK - https://github.com/microsoft/azure_arc/tree/main/azure_jumpstart_localbox +.\deploy-localbox.ps1 -ResourceGroupName rg-localbox -Location swedencentral #> - [CmdletBinding()] param( [Parameter(Mandatory = $true)] [string]$ResourceGroupName, - - [Parameter(Mandatory = $false)] - [string]$Location = "swedencentral", - - [Parameter(Mandatory = $false)] - [string]$WindowsAdminUsername = "arcdemo", - - [Parameter(Mandatory = $false)] + [string]$Location = 'swedencentral', + [string]$WindowsAdminUsername = 'arcdemo', [bool]$DeployBastion = $true, - - [Parameter(Mandatory = $false)] - [string]$GithubAccount = "microsoft", - - [Parameter(Mandatory = $false)] - [string]$GithubBranch = "main", - - [Parameter(Mandatory = $false)] + [string]$GithubAccount = 'microsoft', + [string]$GithubBranch = 'main', [ValidateSet('australiaeast', 'southcentralus', 'eastus', 'westeurope', 'southeastasia', 'canadacentral', 'japaneast', 'centralindia')] - [string]$AzureLocalInstanceLocation = "australiaeast" + [string]$AzureLocalInstanceLocation = 'australiaeast' ) -Write-Host "`n=== Azure Arc Jumpstart LocalBox Deployment ===" -ForegroundColor Cyan -Write-Host "Location: $Location" -ForegroundColor Gray -Write-Host "Resource Group: $ResourceGroupName" -ForegroundColor Gray -Write-Host "" - -# Check if Azure CLI is installed +$ErrorActionPreference = 'Stop' +$PSNativeCommandUseErrorActionPreference = $false if (-not (Get-Command az -ErrorAction SilentlyContinue)) { - Write-Error "Azure CLI is not installed. Please install it from: https://docs.microsoft.com/cli/azure/install-azure-cli" - exit 1 + throw 'Azure CLI is required. Sign in with az login before running this script.' } - -# Check if logged in -$account = az account show 2>$null | ConvertFrom-Json -if (-not $account) { - Write-Host "Not logged in to Azure CLI. Please login..." -ForegroundColor Yellow - az login - $account = az account show | ConvertFrom-Json +$accountJson = & az account show --output json --only-show-errors +if ($LASTEXITCODE -ne 0) { + Write-Host 'Not logged in to Azure CLI. Please sign in.' -ForegroundColor Yellow + & az login --only-show-errors --output none + if ($LASTEXITCODE -ne 0) { throw 'Azure CLI sign-in failed.' } + $accountJson = & az account show --output json --only-show-errors + if ($LASTEXITCODE -ne 0) { throw 'Unable to read the Azure account after sign-in.' } } +$account = $accountJson | ConvertFrom-Json -ErrorAction Stop -Write-Host "Using subscription: $($account.name)" -ForegroundColor Green - -# Warning about resource requirements -Write-Host "`n" + ("=" * 80) -ForegroundColor Yellow -Write-Host "WARNING: LocalBox requires significant compute resources!" -ForegroundColor Yellow -Write-Host ("=" * 80) -ForegroundColor Yellow -Write-Host "Minimum requirements:" -ForegroundColor Yellow -Write-Host " - vCPU quota: 32 vCPUs (Standard_E32s_v6 or larger)" -ForegroundColor Gray -Write-Host " - Deployment time: 4-6 hours" -ForegroundColor Gray -Write-Host "" - -$confirm = Read-Host "Do you want to proceed? (y/N)" -if ($confirm -ne 'y' -and $confirm -ne 'Y') { - Write-Host "Deployment cancelled." -ForegroundColor Yellow - exit 0 +Write-Host "Deploying LocalBox in $Location using subscription $($account.name)." -ForegroundColor Cyan +Write-Warning 'LocalBox requires at least 32 available Esv6-family vCPUs and several hours of nested provisioning. Bastion Developer supports one browser connection at a time.' +if ((Read-Host 'Continue? (y/N)') -ine 'y') { + Write-Host 'Deployment cancelled.' + return } -# Prompt for password -$WindowsAdminPassword = Read-Host -Prompt "Enter admin password for the host VM" -MaskInput - -# Retrieve the object id of your directory's Azure Local resource provider. -$spnProviderId = az ad sp list --display-name "Microsoft.AzureStackHCI Resource Provider" --output json | ConvertFrom-Json - -if (-not $spnProviderId -or -not $spnProviderId.id) { - Write-Host "`n" + ("=" * 80) -ForegroundColor Red - Write-Host "ERROR: Could not retrieve Azure Local Resource Provider service principal." -ForegroundColor Red - Write-Host ("=" * 80) -ForegroundColor Red - Write-Host "" - Write-Host "This typically happens due to:" -ForegroundColor Yellow - Write-Host " - Stale authentication token (Continuous Access Evaluation)" -ForegroundColor Gray - Write-Host " - Insufficient permissions to query Entra ID" -ForegroundColor Gray - Write-Host "" - Write-Host "To resolve, run the following commands:" -ForegroundColor Cyan - Write-Host " 1. az logout" -ForegroundColor White - Write-Host " 2. Elevate to a privileged account in PIM (e.g., Global Reader or Directory Reader)" -ForegroundColor White - Write-Host " 3. az login" -ForegroundColor White - Write-Host " 4. Re-run this script" -ForegroundColor White - Write-Host "" - exit 1 -} - -$spnTenantId = az account show --output json | ConvertFrom-Json - -# Create resource group if it doesn't exist -Write-Host "`nCreating resource group '$ResourceGroupName' in '$Location'..." -ForegroundColor Cyan -az group create --name $ResourceGroupName --location $Location | Out-Null - -# Arc Jumpstart LocalBox Bicep template URL -$templateUri = "https://raw.githubusercontent.com/microsoft/azure_arc/main/azure_jumpstart_localbox/bicep/main.bicep" - -Write-Host "`n=== Starting LocalBox Deployment ===" -ForegroundColor Cyan -Write-Host "This deployment will take approximately 4-6 hours to complete." -ForegroundColor Yellow -Write-Host "You can monitor progress in the Azure Portal under:" -ForegroundColor Gray -Write-Host " Resource Groups > $ResourceGroupName > Deployments" -ForegroundColor Gray -Write-Host "" - -# Download Bicep template locally (--template-uri does not support remote .bicep files) -Write-Host "Downloading Bicep templates..." -ForegroundColor Cyan -$tempDir = Join-Path ([System.IO.Path]::GetTempPath()) "localbox-bicep" - -# Create directory structure -New-Item -ItemType Directory -Path $tempDir -Force | Out-Null -New-Item -ItemType Directory -Path (Join-Path $tempDir "mgmt") -Force | Out-Null -New-Item -ItemType Directory -Path (Join-Path $tempDir "network") -Force | Out-Null -New-Item -ItemType Directory -Path (Join-Path $tempDir "host") -Force | Out-Null - -$templateFile = Join-Path $tempDir "main.bicep" -$baseUri = "https://raw.githubusercontent.com/microsoft/azure_arc/main/azure_jumpstart_localbox/bicep" - -# List of all required Bicep files -$bicepFiles = @( - "main.bicep", - "mgmt/mgmtArtifacts.bicep", - "mgmt/storageAccount.bicep", - "mgmt/customerUsageAttribution.bicep", - "network/network.bicep", - "host/host.bicep" -) - +$password = Read-Host -Prompt 'Enter admin password for the host VM' -MaskInput try { - foreach ($file in $bicepFiles) { - $sourceUrl = "$baseUri/$file" - $destPath = Join-Path $tempDir $file - Write-Host " Downloading: $file" -ForegroundColor Gray - Invoke-WebRequest -Uri $sourceUrl -OutFile $destPath -UseBasicParsing - } - Write-Host "Templates downloaded to: $tempDir" -ForegroundColor Gray -} catch { - Write-Error "Failed to download Bicep template: $($_.Exception.Message)" - exit 1 + $deployer = Join-Path $PSScriptRoot '../../../labautomation/deploy-localbox.ps1' + $deployment = & $deployer -SubscriptionId $account.id -ResourceGroupName $ResourceGroupName ` + -Location $Location -WindowsAdminUsername $WindowsAdminUsername -WindowsAdminPassword $password ` + -DeployBastion $DeployBastion -GithubAccount $GithubAccount -GithubRef $GithubBranch ` + -AzureLocalInstanceLocation $AzureLocalInstanceLocation -NoWait + Write-Host "Submitted $($deployment.DeploymentName). Monitor the resource group's Deployments page; ARM completion is not nested readiness." -ForegroundColor Green + $deployment +} +finally { + $password = $null } - -# Build deployment parameters -$paramsFile = Join-Path (Split-Path (New-TemporaryFile).FullName) "localbox-params.json" -@{ - "`$schema" = "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#" - contentVersion = "1.0.0.0" - parameters = @{ - windowsAdminUsername = @{ value = $WindowsAdminUsername } - windowsAdminPassword = @{ value = $WindowsAdminPassword } - spnProviderId = @{ value = $spnProviderId.id } - tenantId = @{ value = $spnTenantId.tenantId } - location = @{ value = $Location } - githubAccount = @{ value = $GithubAccount } - githubBranch = @{ value = $GithubBranch } - azureLocalInstanceLocation = @{ value = $AzureLocalInstanceLocation } - } -} | ConvertTo-Json -Depth 10 | Set-Content $paramsFile - -Write-Host "Starting deployment..." -ForegroundColor Cyan -Write-Host "Template: $templateFile" -ForegroundColor Gray -Write-Host "Parameters: $paramsFile" -ForegroundColor Gray -Write-Host "" - -# Deploy using Azure CLI - -$deploymentName = "localbox-deployment-$(Get-Date -Format 'yyyyMMdd-HHmmss')" - -try { - az deployment group create ` - --name $deploymentName ` - --resource-group $ResourceGroupName ` - --template-file $templateFile ` - --parameters $paramsFile ` - --no-wait - - Write-Host "`n=== Deployment Initiated ===" -ForegroundColor Green - Write-Host "The LocalBox deployment has been started in the background." -ForegroundColor Gray - Write-Host "" - Write-Host "To monitor deployment progress:" -ForegroundColor Cyan - Write-Host " 1. Azure Portal: Resource Groups > $ResourceGroupName > Deployments" -ForegroundColor Gray - Write-Host " 2. Azure CLI: az deployment group show -g $ResourceGroupName -n $deploymentName" -ForegroundColor Gray - Write-Host "" - Write-Host "Once deployment completes:" -ForegroundColor Cyan - Write-Host " 1. Connect to LocalBox-Client VM via Azure Bastion or RDP" -ForegroundColor Gray - Write-Host " 2. Azure Local instance will appear in the Azure Portal" -ForegroundColor Gray - Write-Host " 3. Follow https://jumpstart.azure.com/azure_jumpstart_localbox/RB to create a Logical Network for VMs and download a Marketplace Image" -ForegroundColor Gray - Write-Host " 4. You can deploy VMs on Azure Local via the Portal" -ForegroundColor Gray - Write-Host "" - Write-Host "Azure Local Features Available:" -ForegroundColor Yellow - Write-Host " - Arc Resource Bridge for hybrid management" -ForegroundColor Gray - Write-Host " - Custom Location for VM deployment" -ForegroundColor Gray - Write-Host " - VM Gallery Images from Azure Marketplace" -ForegroundColor Gray - Write-Host "" - Write-Host "Documentation: https://jumpstart.azure.com/azure_jumpstart_localbox" -ForegroundColor Yellow -} catch { - Write-Error "Deployment failed: $($_.Exception.Message)" - exit 1 -} \ No newline at end of file diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-deployment.tests.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-deployment.tests.ps1 new file mode 100644 index 000000000..00eb90856 --- /dev/null +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-deployment.tests.ps1 @@ -0,0 +1,203 @@ +BeforeAll { + $root = (Resolve-Path "$PSScriptRoot/../..").Path + $azExecutable = (Get-Command az -CommandType Application -ErrorAction Stop | Select-Object -First 1).Source + $compiledPath = Join-Path $TestDrive 'network.json' + & $azExecutable bicep build --file "$root/labautomation/localbox-network.bicep" --outfile $compiledPath --only-show-errors + if ($LASTEXITCODE -ne 0) { throw 'LocalBox network Bicep compilation failed.' } + $network = Get-Content $compiledPath -Raw | ConvertFrom-Json -AsHashtable + + $fixture = Join-Path $TestDrive 'lab' + New-Item -ItemType Directory -Path "$fixture/labautomation", "$fixture/resources/manual-setup/localbox" -Force | Out-Null + Copy-Item "$root/labautomation/deploy-localbox.ps1" "$fixture/labautomation" + Copy-Item "$root/resources/manual-setup/localbox/deploy-localbox.ps1" "$fixture/resources/manual-setup/localbox" + function az {} +} + +Describe 'Compiled LocalBox Developer network' { + It 'uses the Developer virtual-network binding without dedicated Bastion IP configuration' { + $bastion = @($network.resources | Where-Object type -eq 'Microsoft.Network/bastionHosts') + $bastion.Count | Should -Be 1 + $bastion[0].name | Should -Be 'LocalBox-Bastion' + $bastion[0].sku.name | Should -Be 'Developer' + $bastion[0].properties.virtualNetwork.id | Should -Match 'Microsoft.Network/virtualNetworks' + $bastion[0].properties.ContainsKey('ipConfigurations') | Should -BeFalse + } + It 'retains the NAT public IP and gateway but no dedicated Bastion resources' { + $ips = @($network.resources | Where-Object type -eq 'Microsoft.Network/publicIPAddresses') + $ips.Count | Should -Be 1 + $ips[0].name | Should -Match 'natGatewayName' + $ips[0].sku.name | Should -Be 'Standard' + @($network.resources | Where-Object type -eq 'Microsoft.Network/natGateways').Count | Should -Be 1 + @($network.resources | Where-Object type -eq 'Microsoft.Network/networkSecurityGroups').Count | Should -Be 1 + $vnet = $network.resources | Where-Object type -eq 'Microsoft.Network/virtualNetworks' + $vnet.properties.addressSpace.addressPrefixes | Should -Contain '172.16.0.0/16' + $vnet.properties.subnets.Count | Should -Be 1 + $subnet = $vnet.properties.subnets[0].properties + $subnet.addressPrefix | Should -Be '172.16.1.0/24' + $subnet.defaultOutboundAccess | Should -BeFalse + $subnet.natGateway | Should -Match 'Microsoft.Network/natGateways' + $subnet.networkSecurityGroup.id | Should -Match 'Microsoft.Network/networkSecurityGroups' + } + It 'preserves disabled-Bastion resource conditions and upstream module outputs' { + $conditional = @($network.resources | Where-Object { $_.ContainsKey('condition') }) + $conditional.Count | Should -Be 3 + foreach ($resource in $conditional) { $resource.condition | Should -Be "[parameters('deployBastion')]" } + $network.outputs.vnetId.value | Should -Match 'Microsoft.Network/virtualNetworks' + $network.outputs.subnetId.value | Should -Match 'subnets' + foreach ($resource in $network.resources) { $resource.tags | Should -Be "[parameters('resourceTags')]" } + } +} + +Describe 'LocalBox deployment entry points' { + BeforeEach { + Copy-Item "$root/labautomation/localbox-network.bicep" "$fixture/labautomation" -Force + $deploymentTestState = @{ + GroupExists = $false + Bastions = @() + Networks = @() + QueriesFail = $false + ValidationFails = $false + Submitted = 0 + Builds = 0 + Parameters = $null + NetworkSource = $null + Downloads = [Collections.Generic.List[string]]::new() + } + $arguments = @{ + SubscriptionId = '10000000-0000-0000-0000-000000000001' + ResourceGroupName = 'rg-localbox-test' + Location = 'spaincentral' + WindowsAdminPassword = 'fixture-password-not-a-credential' + AzureLocalResourceProviderObjectId = '20000000-0000-0000-0000-000000000001' + NoWait = $true + } + Mock Invoke-WebRequest { + $deploymentTestState.Downloads.Add([string]$Uri) + Set-Content -LiteralPath $OutFile '// upstream fixture; full compilation is validated separately' + } + Mock az { + $global:LASTEXITCODE = 0 + $operation = ($args | Select-Object -First 3) -join ' ' + switch -Regex ($operation) { + '^account show' { '{"name":"test","id":"10000000-0000-0000-0000-000000000001","tenantId":"30000000-0000-0000-0000-000000000001"}'; break } + '^provider show' { 'Registered'; break } + '^feature show' { 'Registered'; break } + '^provider register' { '{}'; break } + '^vm list-sizes' { 'Standard_E32s_v6'; break } + '^vm list-usage' { '[{"name":{"localizedValue":"Standard Esv6 Family vCPUs"},"limit":64,"currentValue":0}]'; break } + '^ad sp list' { '[{"id":"20000000-0000-0000-0000-000000000001"}]'; break } + '^group exists' { $deploymentTestState.GroupExists.ToString().ToLowerInvariant(); break } + '^network bastion list' { + if ($deploymentTestState.QueriesFail) { throw 'AuthorizationFailed' } + ConvertTo-Json -InputObject @($deploymentTestState.Bastions) -Depth 10 + break + } + '^network vnet list' { ConvertTo-Json -InputObject @($deploymentTestState.Networks) -Depth 10; break } + '^group (create|show|update)' { '{"name":"rg-localbox-test","location":"spaincentral"}'; break } + '^bicep build' { + $deploymentTestState.Builds++ + $path = $args[[array]::IndexOf($args, '--file') + 1] + $deploymentTestState.NetworkSource = Get-Content (Join-Path (Split-Path $path) 'network/network.bicep') -Raw + break + } + '^deployment group validate' { + if ($deploymentTestState.ValidationFails) { throw 'Bastion Developer is not available in the requested region.' } + $path = $args[[array]::IndexOf($args, '--parameters') + 1].TrimStart('@') + $deploymentTestState.Parameters = (Get-Content $path -Raw | ConvertFrom-Json).parameters + '{}' + break + } + '^deployment group create' { $deploymentTestState.Submitted++; '{}'; break } + default { throw "Unexpected az operation in test: $operation" } + } + } + } + It 'packages the local module alongside upstream templates and preserves default Bastion/private-host parameters' { + $result = & "$fixture/labautomation/deploy-localbox.ps1" @arguments + $result.ProvisioningState | Should -Be 'Submitted' + $result.BastionName | Should -Be 'LocalBox-Bastion' + $deploymentTestState.Parameters.deployBastion.value | Should -BeTrue + $deploymentTestState.Parameters.tags.value.SecurityControl | Should -Be 'Ignore' + $deploymentTestState.Parameters.tags.value.CostControl | Should -Be 'Ignore' + $deploymentTestState.NetworkSource | Should -Be (Get-Content "$root/labautomation/localbox-network.bicep" -Raw) + $deploymentTestState.Downloads.Count | Should -Be 5 + @($deploymentTestState.Downloads | Where-Object { $_ -like '*/network/network.bicep' }).Count | Should -Be 0 + $deploymentTestState.Builds | Should -Be 1 + $deploymentTestState.Submitted | Should -Be 1 + } + It 'preserves explicit disabled-Bastion behavior' { + $result = & "$fixture/labautomation/deploy-localbox.ps1" @arguments -DeployBastion $false + $deploymentTestState.Parameters.deployBastion.value | Should -BeFalse + $result.BastionName | Should -BeNullOrEmpty + } + It 'allows a compatible existing Developer deployment' { + $deploymentTestState.GroupExists = $true + $deploymentTestState.Bastions = @(@{ name = 'LocalBox-Bastion'; sku = @{ name = 'Developer' } }) + $deploymentTestState.Networks = @(@{ name = 'LocalBox-VNet'; subnets = @(@{ name = 'LocalBox-Subnet' }) }) + & "$fixture/labautomation/deploy-localbox.ps1" @arguments | Out-Null + $deploymentTestState.Submitted | Should -Be 1 + } + It 'rejects existing Bastion without changing or deleting it' -TestCases @( + @{ Sku = 'Basic' }, @{ Sku = 'Standard' }, @{ Sku = 'Premium' }, @{ Sku = '' } + ) { + param($Sku) + $deploymentTestState.GroupExists = $true + $deploymentTestState.Bastions = @(@{ name = 'LocalBox-Bastion'; sku = @{ name = $Sku }; provisioningState = 'Failed' }) + { & "$fixture/labautomation/deploy-localbox.ps1" @arguments } | Should -Throw '*cannot be changed to Developer in place*' + $deploymentTestState.Submitted | Should -Be 0 + $deploymentTestState.Downloads.Count | Should -Be 0 + Should -Invoke az -Times 0 -ParameterFilter { ($args -join ' ') -match 'delete|group update|deployment group create' } + } + It 'rejects a legacy subnet even when its dedicated Bastion has already been removed' { + $deploymentTestState.GroupExists = $true + $deploymentTestState.Networks = @(@{ name = 'LocalBox-VNet'; subnets = @(@{ name = 'AzureBastionSubnet' }) }) + { & "$fixture/labautomation/deploy-localbox.ps1" @arguments } | Should -Throw '*legacy AzureBastionSubnet*' + $deploymentTestState.Submitted | Should -Be 0 + } + It 'fails closed on existing network discovery errors' { + $deploymentTestState.GroupExists = $true + $deploymentTestState.QueriesFail = $true + { & "$fixture/labautomation/deploy-localbox.ps1" @arguments } | Should -Throw '*AuthorizationFailed*' + $deploymentTestState.Submitted | Should -Be 0 + } + It 'does not fall back to a paid SKU when ARM validation rejects the region' { + $deploymentTestState.ValidationFails = $true + { & "$fixture/labautomation/deploy-localbox.ps1" @arguments } | Should -Throw '*not available*' + $deploymentTestState.Submitted | Should -Be 0 + } + It 'fails preflight when the Console package omits the local network module' { + Remove-Item -LiteralPath "$fixture/labautomation/localbox-network.bicep" + { & "$fixture/labautomation/deploy-localbox.ps1" @arguments } | Should -Throw '*Missing localbox-network.bicep*' + Should -Invoke az -Times 0 + } + It 'uses the same deployer and forwards manual settings with Bastion enabled=' -TestCases @( + @{ Enabled = $true }, @{ Enabled = $false } + ) { + param($Enabled) + Mock Read-Host { 'y' } -ParameterFilter { $Prompt -eq 'Continue? (y/N)' } + Mock Read-Host { 'fixture-password-not-a-credential' } -ParameterFilter { $Prompt -eq 'Enter admin password for the host VM' } + $result = & "$fixture/resources/manual-setup/localbox/deploy-localbox.ps1" ` + -ResourceGroupName 'rg-localbox-test' -Location spaincentral -DeployBastion $Enabled ` + -GithubAccount 'fixture-owner' -GithubBranch 'fixture-ref' -AzureLocalInstanceLocation westeurope + $result.ProvisioningState | Should -Be 'Submitted' + $deploymentTestState.Parameters.deployBastion.value | Should -Be $Enabled + $deploymentTestState.Parameters.githubAccount.value | Should -Be 'fixture-owner' + $deploymentTestState.Parameters.githubBranch.value | Should -Be 'fixture-ref' + $deploymentTestState.Parameters.azureLocalInstanceLocation.value | Should -Be 'westeurope' + @($deploymentTestState.Downloads | Where-Object { $_ -notlike 'https://raw.githubusercontent.com/fixture-owner/azure_arc/fixture-ref/*' }).Count | Should -Be 0 + } + It 'preserves manual cancellation before password collection or deployment' { + Mock Read-Host { 'n' } + & "$fixture/resources/manual-setup/localbox/deploy-localbox.ps1" -ResourceGroupName 'rg-localbox-test' + Should -Invoke Read-Host -Times 1 -Exactly + Should -Invoke Invoke-WebRequest -Times 0 + $deploymentTestState.Submitted | Should -Be 0 + } + It 'offers interactive login when needed and propagates sign-in failure' { + Mock az { $global:LASTEXITCODE = 1 } -ParameterFilter { $args[0] -eq 'account' -or $args[0] -eq 'login' } + { & "$fixture/resources/manual-setup/localbox/deploy-localbox.ps1" -ResourceGroupName 'rg-localbox-test' } | + Should -Throw '*sign-in failed*' + Should -Invoke az -Times 1 -Exactly -ParameterFilter { $args[0] -eq 'login' } + $deploymentTestState.Submitted | Should -Be 0 + } +} diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/prepare-localbox.tests.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/prepare-localbox.tests.ps1 index da16bb3a9..45b77b858 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/prepare-localbox.tests.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/prepare-localbox.tests.ps1 @@ -1140,8 +1140,14 @@ Describe 'LocalBox registration region contract' { $registration.DefaultValue.SafeGetValue() | Should -Be 'australiaeast' $hostRegion = $ast.ParamBlock.Parameters | Where-Object { $_.Name.VariablePath.UserPath -eq 'Location' } $hostRegion.DefaultValue.SafeGetValue() | Should -Be 'swedencentral' - $ast.Extent.Text | Should -Match 'azureLocalInstanceLocation\s*=\s*@\{\s*value\s*=\s*\$AzureLocalInstanceLocation\s*\}' - $ast.Extent.Text | Should -Match 'location\s*=\s*@\{\s*value\s*=\s*\$Location\s*\}' + if ($Path -like 'resources/manual-setup/*') { + $ast.Extent.Text | Should -Match '-AzureLocalInstanceLocation \$AzureLocalInstanceLocation' + $ast.Extent.Text | Should -Match '-Location \$Location' + } + else { + $ast.Extent.Text | Should -Match 'azureLocalInstanceLocation\s*=\s*@\{\s*value\s*=\s*\$AzureLocalInstanceLocation\s*\}' + $ast.Extent.Text | Should -Match 'location\s*=\s*@\{\s*value\s*=\s*\$Location\s*\}' + } } It 'uses Australia East registration while preserving shared host-region selection' { $errors = $null diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md index 9402860b5..9adffa674 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md @@ -45,6 +45,15 @@ required to validate regional service availability and effective inherited polic ## Global event LocalBox checks +Run `Invoke-Pester ./localbox-deployment.tests.ps1 -Output Detailed` for the +LocalBox Developer Bastion deployment contract. These tests require Azure CLI +with Bicep available: they compile the network module locally, then mock Azure +deployment operations. They verify the Developer VNet binding, retained NAT +outbound path, absence of dedicated Bastion resources, complete Console +packaging, manual-entry-point forwarding, and refusal to downgrade an existing +dedicated Bastion or remove its subnet. They do not deploy resources or verify +real portal connectivity; a fresh deployment remains the integration gate. + Run `Invoke-Pester ./localbox-automation.tests.ps1,./global-localbox.tests.ps1 -Output Detailed` for the hosted readiness gate. Offline tests cover token refresh on every poll, nested readiness beyond ARM success, imported-content packaging, event-group From 2f51015d6fbc1b6aafc10f500a0003c4c946bf71 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Wed, 4 Nov 2026 19:11:13 +0000 Subject: [PATCH 11/37] Retry transient Azure Local validation download timeouts once Use a resumable managed command with existing local ARM files, a two-minute backoff and the remaining nested-readiness budget. Require successful correlated validation before deploying; retain terminal errors and avoid duplicate deployments. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../labautomation/localbox-automation.ps1 | 116 ++++++- .../localbox-validation-retry.ps1 | 117 +++++++ .../resources/hosted-events/readme.md | 41 +++ .../tests/localbox-validation-retry.tests.ps1 | 285 ++++++++++++++++++ .../resources/tests/readme.md | 10 + 5 files changed, 566 insertions(+), 3 deletions(-) create mode 100644 03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-validation-retry.ps1 create mode 100644 03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-validation-retry.tests.ps1 diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation.ps1 index 0f2d98dd1..1851c4633 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation.ps1 @@ -44,15 +44,125 @@ function Wait-LocalBoxAutomation { } while ($true) } +function Test-LocalBoxValidationDownloadTimeout { + param($ErrorDetail) + if (-not $ErrorDetail) { return $false } + if ($ErrorDetail.details) { + foreach ($child in $ErrorDetail.details) { + if (-not (Test-LocalBoxValidationDownloadTimeout $child)) { return $false } + } + return $true + } + return $ErrorDetail.code -eq 'UpdateDeploymentSettingsDataFailed' -and + $ErrorDetail.message -match 'Name=Download content for deployment' -and + $ErrorDetail.message -match 'Exception=Timeout of 7200 seconds reached for interface' +} + +function Get-LocalBoxValidationRetryState { + param($Command, [string]$SubscriptionId, [string]$ResourceGroupName) + $metadata = @{} + foreach ($parameter in $Command.properties.parameters) { + if ($metadata.ContainsKey($parameter.name)) { throw 'Duplicate validation recovery metadata. Inspect the existing Run Command; it was not replaced.' } + $metadata[$parameter.name] = $parameter.value + } + $correlation = [guid]::Empty + $startedAfter = [datetimeoffset]::MinValue + if ($metadata.RecoveryVersion -ne '1' -or $metadata.SubscriptionId -ne $SubscriptionId -or + $metadata.ResourceGroupName -ne $ResourceGroupName -or + -not [guid]::TryParse([string]$metadata.OriginalValidationCorrelationId, [ref]$correlation) -or $correlation -eq [guid]::Empty -or + -not [datetimeoffset]::TryParse([string]$metadata.StartedAfter, [ref]$startedAfter)) { + throw 'Unrecognized validation recovery Run Command. Inspect it before retrying; it was not replaced.' + } + if ($Command.properties.provisioningState -in @('Failed', 'Canceled')) { throw 'Validation recovery Run Command provisioning failed. Inspect sovereign-localbox-validation-retry.' } + $state = $Command.properties.instanceView + if (-not $state.startTime -or [datetimeoffset]$state.startTime -lt $startedAfter) { + return @{ Ready = $false; OriginalCorrelationId = $metadata.OriginalValidationCorrelationId + Detail = 'Waiting for validation retry 1/1 to start; ignoring stale instanceView.' } + } + if ($state.executionState -in @('Failed', 'Canceled', 'TimedOut')) { + throw 'Automatic validation retry failed or timed out. No further automatic retry will be submitted; inspect sovereign-localbox-validation-retry and localcluster-validate.' + } + if ($state.executionState -eq 'Succeeded') { + if ($null -eq $state.exitCode -or $state.exitCode -ne 0) { throw 'Validation recovery did not exit successfully.' } + return @{ Ready = $true; OriginalCorrelationId = $metadata.OriginalValidationCorrelationId } + } + @{ Ready = $false; OriginalCorrelationId = $metadata.OriginalValidationCorrelationId + Detail = "Automatic validation retry 1/1: $($state.executionState). Deployment remains gated on successful validation." } +} + function Wait-LocalBoxNestedDeployment { param([string]$SubscriptionId, [string]$ResourceGroupName, [int]$TimeoutSeconds = 28800) + $deadline = [datetime]::UtcNow.AddSeconds($TimeoutSeconds) + $retry = @{ Checked = $false; Command = $null; OriginalCorrelationId = $null } + $vmId = "/subscriptions/$SubscriptionId/resourceGroups/$ResourceGroupName/providers/Microsoft.Compute/virtualMachines/LocalBox-Client" + $retryUri = "$vmId/runCommands/sovereign-localbox-validation-retry?api-version=2024-07-01" Wait-LocalBoxAutomation -Phase 'nested deployment' -TimeoutSeconds $TimeoutSeconds -Probe { $deployments = @(Invoke-LocalBoxAutomationAz @('deployment', 'group', 'list', '--subscription', $SubscriptionId, '--resource-group', $ResourceGroupName)) $clusterDeployment = @($deployments | Where-Object name -eq 'localcluster-deploy') - foreach ($deployment in @($deployments | Where-Object name -in @('localcluster-validate', 'localcluster-deploy'))) { - if ($deployment.properties.provisioningState -in @('Failed', 'Canceled')) { - throw "Nested deployment $($deployment.name) failed. Inspect its deployment operations and Client logs." + $validation = @($deployments | Where-Object name -eq 'localcluster-validate') + if ($clusterDeployment.Count -gt 1 -or $validation.Count -gt 1) { throw 'Ambiguous nested deployment records.' } + if ($clusterDeployment.Count -eq 1 -and $clusterDeployment[0].properties.provisioningState -in @('Failed', 'Canceled')) { + throw 'Nested deployment localcluster-deploy failed. Inspect its deployment operations and Client logs.' + } + if (-not $retry.Checked -and ($clusterDeployment.Count -ne 1 -or $clusterDeployment[0].properties.provisioningState -ne 'Succeeded')) { + $retry.Command = Invoke-LocalBoxAutomationRequest -Path "$retryUri&`$expand=instanceView" -AllowNotFound + $retry.Checked = $true + } + if ($retry.Command) { + $retry.Command = Invoke-LocalBoxAutomationRequest -Path "$retryUri&`$expand=instanceView" + $retryState = Get-LocalBoxValidationRetryState -Command $retry.Command -SubscriptionId $SubscriptionId -ResourceGroupName $ResourceGroupName + if ($retry.OriginalCorrelationId -and $retry.OriginalCorrelationId -ne $retryState.OriginalCorrelationId) { + throw 'Validation recovery correlation changed while monitoring; the command was not replaced.' } + $retry.OriginalCorrelationId = $retryState.OriginalCorrelationId + if (-not $retryState.Ready) { return $retryState } + if ($validation.Count -eq 1 -and $validation[0].properties.correlationId -eq $retryState.OriginalCorrelationId) { + return @{ Ready = $false; Detail = 'Validation retry completed; waiting for updated ARM deployment records.' } + } + } + if ($validation.Count -eq 1 -and $validation[0].properties.provisioningState -in @('Failed', 'Canceled')) { + if ($retry.Command -or $validation[0].properties.provisioningState -eq 'Canceled' -or $clusterDeployment.Count) { + throw 'Nested deployment localcluster-validate failed or was canceled. No further automatic validation retry is allowed.' + } + $operations = @(Invoke-LocalBoxAutomationAz @('deployment', 'operation', 'group', 'list', '--subscription', $SubscriptionId, + '--resource-group', $ResourceGroupName, '--name', 'localcluster-validate')) + $failed = @($operations | Where-Object { $_.properties.provisioningState -eq 'Failed' }) + if ($failed.Count -ne 1 -or $failed[0].properties.targetResource.resourceType -ine 'Microsoft.AzureStackHCI/clusters/deploymentSettings') { + throw 'Nested deployment localcluster-validate failed outside the recognized content-download operation. Manual investigation is required.' + } + $status = $failed[0].properties.statusMessage + if ($status -is [string]) { + try { $status = $status | ConvertFrom-Json -AsHashtable -ErrorAction Stop } + catch { throw 'Unable to parse validation operation details; automatic retry is not safe.' } + } + if (-not (Test-LocalBoxValidationDownloadTimeout $status.error)) { + throw 'Nested deployment localcluster-validate failed with a non-retryable validation error. Inspect its deployment operations.' + } + $correlation = [guid]::Empty + if (-not [guid]::TryParse([string]$validation[0].properties.correlationId, [ref]$correlation) -or $correlation -eq [guid]::Empty) { + throw 'Cannot correlate the failed validation; automatic retry was not submitted.' + } + $remaining = [int][math]::Floor(($deadline - [datetime]::UtcNow).TotalSeconds) + if ($remaining -le 120) { throw 'Insufficient nested-deployment time remaining for the validation retry delay.' } + $source = Get-Content -LiteralPath (Join-Path $PSScriptRoot 'localbox-validation-retry.ps1') -Raw -ErrorAction Stop + $vm = Invoke-LocalBoxAutomationRequest -Path "$vmId`?api-version=2024-07-01" + Write-Warning '[LocalBox] Azure Local validation hit the 7200-second content-download timeout. Scheduling automatic retry 1/1 after 120 seconds; the existing nested wait deadline is unchanged.' + Invoke-LocalBoxAutomationRequest -Path $retryUri -Method PUT -Body @{ + location = $vm.location + properties = @{ + source = @{ script = $source }; asyncExecution = $true; timeoutInSeconds = $remaining + parameters = @( + @{ name = 'SubscriptionId'; value = $SubscriptionId } + @{ name = 'ResourceGroupName'; value = $ResourceGroupName } + @{ name = 'OriginalValidationCorrelationId'; value = $correlation.ToString() } + @{ name = 'StartedAfter'; value = [datetime]::UtcNow.ToString('o') } + @{ name = 'RecoveryVersion'; value = '1' } + ) + } + } | Out-Null + $retry.Command = @{ properties = @{} } + $retry.OriginalCorrelationId = $correlation.ToString() + return @{ Ready = $false; Detail = 'Automatic validation retry 1/1 submitted. Credentials and existing ARM files remain on LocalBox-Client.' } } if ($clusterDeployment.Count -ne 1 -or $clusterDeployment[0].properties.provisioningState -ne 'Succeeded') { return @{ Ready = $false; Detail = 'Waiting for localcluster-deploy to succeed.' } diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-validation-retry.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-validation-retry.ps1 new file mode 100644 index 000000000..de0d8b2ae --- /dev/null +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-validation-retry.ps1 @@ -0,0 +1,117 @@ +param( + [string]$SubscriptionId, + [string]$ResourceGroupName, + [string]$OriginalValidationCorrelationId, + [string]$StartedAfter, + [string]$RecoveryVersion +) + +function Invoke-LocalBoxValidationRetryAz { + param([string[]]$Arguments, [switch]$NoOutput) + $PSNativeCommandUseErrorActionPreference = $false + $format = if ($NoOutput) { 'none' } else { 'json' } + $output = @(& az @Arguments --only-show-errors --output $format 2>&1) + $operation = ($Arguments | Select-Object -First 3) -join ' ' + if ($LASTEXITCODE -ne 0) { + throw "Validation recovery Azure CLI '$operation' failed. Inspect localcluster-validate/localcluster-deploy operations. CLI output is withheld to protect deployment credentials." + } + $stderr = @($output | Where-Object { $_ -is [Management.Automation.ErrorRecord] }) + if ($stderr.Count) { Write-Warning "Azure CLI emitted diagnostics during '$operation'; raw output is withheld to protect deployment credentials." } + if ($NoOutput) { return } + $stdout = ($output | Where-Object { $_ -isnot [Management.Automation.ErrorRecord] }) -join "`n" + try { $stdout | ConvertFrom-Json -ErrorAction Stop } + catch { throw "Validation recovery Azure CLI '$operation' returned invalid JSON; raw output is withheld." } +} + +function Invoke-LocalBoxValidationRetry { + param( + [Parameter(Mandatory)][string]$SubscriptionId, + [Parameter(Mandatory)][string]$ResourceGroupName, + [Parameter(Mandatory)][guid]$OriginalValidationCorrelationId, + [string]$TemplateDirectory = 'C:\LocalBox', + [int]$RetryDelaySeconds = 120 + ) + $ErrorActionPreference = 'Stop' + if ($OriginalValidationCorrelationId -eq [guid]::Empty) { throw 'A nonempty original validation correlation ID is required.' } + $template = Join-Path $TemplateDirectory 'azlocal.json' + $parameters = Join-Path $TemplateDirectory 'azlocal.parameters.json' + foreach ($path in @($template, $parameters)) { + if (-not (Test-Path -LiteralPath $path -PathType Leaf)) { throw "Required existing LocalBox deployment file is missing: $path" } + } + Get-Command az -ErrorAction Stop | Out-Null + Write-Host "[LocalBox] Automatic validation retry 1/1: waiting $RetryDelaySeconds seconds before rechecking the failed deployment." + Start-Sleep -Seconds $RetryDelaySeconds + $previousProfile = $env:AZURE_CONFIG_DIR + $profile = Join-Path ([IO.Path]::GetTempPath()) "localbox-validation-$([guid]::NewGuid())" + $failure = $null + try { + New-Item -ItemType Directory -Path $profile -ErrorAction Stop | Out-Null + $env:AZURE_CONFIG_DIR = $profile + Invoke-LocalBoxValidationRetryAz @('login', '--identity', '--allow-no-subscriptions') -NoOutput + $scope = @('--subscription', $SubscriptionId, '--resource-group', $ResourceGroupName) + $original = Invoke-LocalBoxValidationRetryAz (@('deployment', 'group', 'show') + $scope + @('--name', 'localcluster-validate')) + if ($original.properties.provisioningState -ne 'Failed' -or + $original.properties.correlationId -ne $OriginalValidationCorrelationId.ToString()) { + throw 'Validation changed after recovery was requested. No retry was submitted; inspect possible manual intervention.' + } + $deployments = @(Invoke-LocalBoxValidationRetryAz (@('deployment', 'group', 'list') + $scope)) + if (@($deployments | Where-Object name -eq 'localcluster-deploy').Count) { + throw 'A cluster deployment already exists. Recovery will not replace or overlap it.' + } + try { + $localParameters = (Get-Content -LiteralPath $parameters -Raw -ErrorAction Stop | ConvertFrom-Json -ErrorAction Stop).parameters + $localTemplate = Get-Content -LiteralPath $template -Raw -ErrorAction Stop | ConvertFrom-Json -ErrorAction Stop + } + catch { throw 'Unable to read the existing LocalBox template/parameters; contents are withheld to protect credentials.' } + foreach ($field in @('clusterName', 'keyVaultName', 'diagnosticStorageAccountName', 'arcNodeResourceIds')) { + $expected = $original.properties.parameters.$field.value + $actual = $localParameters.$field.value + if ($null -eq $expected -or $null -eq $actual -or + (ConvertTo-Json -InputObject $actual -Compress) -ne (ConvertTo-Json -InputObject $expected -Compress)) { + throw "LocalBox parameter '$field' does not match the failed validation. No retry was submitted." + } + } + foreach ($mode in @('Validate', 'Deploy')) { + if ($mode -notin $localTemplate.parameters.deploymentMode.allowedValues) { throw 'The LocalBox template does not support the expected Validate/Deploy contract.' } + } + $arguments = @('deployment', 'group', 'create') + $scope + @('--template-file', $template, '--parameters', "@$parameters") + Write-Host '[LocalBox] Retrying Azure Local validation using the existing local deployment files.' + $validation = Invoke-LocalBoxValidationRetryAz ($arguments + @('deploymentMode=Validate', '--name', 'localcluster-validate')) + if ($validation.properties.provisioningState -ne 'Succeeded' -or + -not $validation.properties.correlationId -or $validation.properties.correlationId -eq $OriginalValidationCorrelationId.ToString()) { + throw 'The automatic validation retry did not return a new successful validation. Cluster deployment was not submitted.' + } + $current = Invoke-LocalBoxValidationRetryAz (@('deployment', 'group', 'show') + $scope + @('--name', 'localcluster-validate')) + if ($current.properties.provisioningState -ne 'Succeeded' -or $current.properties.correlationId -ne $validation.properties.correlationId) { + throw 'Validation changed before cluster deployment could be submitted. Inspect possible manual intervention.' + } + $deployments = @(Invoke-LocalBoxValidationRetryAz (@('deployment', 'group', 'list') + $scope)) + if (@($deployments | Where-Object name -eq 'localcluster-deploy').Count) { + throw 'A cluster deployment appeared during validation. Recovery will not replace or overlap it.' + } + Write-Host '[LocalBox] Validation retry succeeded. Submitting Azure Local deployment; nested readiness is still pending.' + Invoke-LocalBoxValidationRetryAz ($arguments + @('deploymentMode=Deploy', '--name', 'localcluster-deploy', '--no-wait')) -NoOutput + } + catch { + $failure = $_ + throw + } + finally { + $env:AZURE_CONFIG_DIR = $previousProfile + try { + if (Test-Path -LiteralPath $profile) { Remove-Item -LiteralPath $profile -Recurse -Force -ErrorAction Stop } + } + catch { + if ($failure) { throw "Validation recovery failed: $($failure.Exception.Message) Cleanup of its private CLI profile also failed." } + throw 'Validation recovery could not remove its private CLI profile.' + } + } +} + +if ($MyInvocation.InvocationName -ne '.') { + try { + Invoke-LocalBoxValidationRetry -SubscriptionId $SubscriptionId -ResourceGroupName $ResourceGroupName ` + -OriginalValidationCorrelationId $OriginalValidationCorrelationId + } + catch { Write-Error $_ -ErrorAction Continue; exit 1 } +} diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md index a5c811895..078aba1f9 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md @@ -237,6 +237,47 @@ bounded intervals and potential optimizations. ### Diagnostics and reruns +#### Automatic Azure Local validation recovery + +The Console can perform **one automatic validation retry**, after a **two-minute +delay**, for the known `UpdateDeploymentSettingsDataFailed` failure in +`Download content for deployment` with `Timeout of 7200 seconds reached for +interface`. It requires exactly one failed deployment-settings operation and +recognized error details. Other validation failures, cancellation, deployment +failures, unreadable diagnostics and ambiguous state still stop the job. + +The host submits the asynchronous Managed Run Command +`sovereign-localbox-validation-retry` on LocalBox-Client. Keep +[localbox-validation-retry.ps1](../../labautomation/localbox-validation-retry.ps1) +in the imported `labautomation` directory. The recovery: + +1. Rechecks the original failed validation correlation after the delay. +2. Uses the Client's managed identity and existing `C:\LocalBox\azlocal.json` and + `azlocal.parameters.json`. Nonsecret environment identifiers must match the + failed deployment. It does not regenerate templates or credentials, + replay bootstrap, or export the parameter file to Console. +3. Reruns `localcluster-validate` with `deploymentMode=Validate`. Only a new, + successful validation permits submission of `localcluster-deploy` with + `deploymentMode=Deploy`. A concurrent/manual deployment is not overwritten. +4. Removes its temporary Azure CLI profile. Console refreshes its own tokens + while polling, reports retry status, and continues the normal cluster, + connectivity, bridge, extension and subsequent Full health gates. + +Recovery uses the **remaining eight-hour nested-readiness budget**, not a new +eight-hour window. A retry can itself take hours, and neither its submission nor +its successful exit means the nested deployment is ready. Ensure the Console +job limit accommodates the complete workflow. + +The Run Command record persists: a restarted Console job monitors an existing +recovery instead of submitting another. A failed/timed-out recovery is terminal; +there is no automatic retry loop across Console restarts. Inspect its +`instanceView` and the ARM deployment operations before manual intervention. +Once a manual repair has produced a successful cluster deployment, the normal +readiness checks can proceed without rerunning that historical recovery. +Do not manually re-trigger validation/deployment while automatic recovery is +running. Cancellation or forced guest termination may prevent CLI-profile +cleanup; inspect the Client before resetting recovery state. + If an older run failed with `Could not find a part of the path '/content/.../resources/prepare-localbox.ps1'`, it reached ARM completion but could not package guest automation. Re-import the corrected branch (including diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-validation-retry.tests.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-validation-retry.tests.ps1 new file mode 100644 index 000000000..d8a926c7e --- /dev/null +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-validation-retry.tests.ps1 @@ -0,0 +1,285 @@ +BeforeAll { + $root = (Resolve-Path "$PSScriptRoot/../..").Path + . "$root/labautomation/localbox-automation.ps1" + . "$root/labautomation/localbox-validation-retry.ps1" + function Update-MhhToken {} + function az {} + $testSubscription = '10000000-0000-0000-0000-000000000001' + $testGroup = 'rg-localbox-test' + $originalId = '20000000-0000-0000-0000-000000000001' + $newId = '20000000-0000-0000-0000-000000000002' + $downloadError = @{ + code = 'UpdateDeploymentSettingsDataFailed' + message = 'Validation status is {Status=Error, Steps={Name=Download content for deployment, Description=Download required content before starting validation., Status=Error, Exception=Timeout of 7200 seconds reached for interface DownloadContent}}' + } +} + +Describe 'Validation failure classification' { + It 'recognizes the reported content-download timeout, including nested ARM errors' { + Test-LocalBoxValidationDownloadTimeout $downloadError | Should -BeTrue + Test-LocalBoxValidationDownloadTimeout @{ code = 'DeploymentFailed'; details = @($downloadError) } | Should -BeTrue + } + It 'rejects unknown, empty and mixed validation errors' { + foreach ($errorDetail in @( + $null, + @{ code = 'AuthorizationFailed'; message = $downloadError.message }, + @{ code = $downloadError.code; message = 'Network validation failed' }, + @{ code = $downloadError.code; message = 'Name=Download content for deployment, Exception=Access denied' }, + @{ details = @($downloadError, @{ code = 'InvalidConfiguration'; message = 'Invalid network' }) } + )) { + Test-LocalBoxValidationDownloadTimeout $errorDetail | Should -BeFalse + } + } +} + +Describe 'Bounded Console validation recovery' { + BeforeEach { + $recoveryTest = @{ Submitted = 0; Reads = 0; Recovered = $false; Existing = $null; Body = $null; FailedState = 'Failed' } + Mock Update-MhhToken {} + Mock Wait-LocalBoxAutomation { + for ($iteration = 0; $iteration -lt 6; $iteration++) { + $probeResult = & $Probe + if ($probeResult.Ready) { return $probeResult } + } + throw 'Test polling budget exhausted.' + } + Mock Invoke-LocalBoxAutomationAz { + if ($Arguments[0] -eq 'deployment' -and $Arguments[1] -eq 'operation') { + @{ properties = @{ provisioningState = 'Failed' + targetResource = @{ resourceType = 'Microsoft.AzureStackHCI/clusters/deploymentSettings' } + statusMessage = @{ error = $downloadError } } } + } + elseif ($Arguments[0] -eq 'deployment') { + if ($recoveryTest.Recovered) { + @{ name = 'localcluster-validate'; properties = @{ provisioningState = 'Succeeded'; correlationId = $newId } } + @{ name = 'localcluster-deploy'; properties = @{ provisioningState = 'Succeeded' } } + } + else { + @{ name = 'localcluster-validate'; properties = @{ provisioningState = $recoveryTest.FailedState; correlationId = $originalId } } + } + } + elseif ($Arguments[0] -eq 'resource' -and $Arguments[1] -eq 'list') { + foreach ($type in @('Microsoft.AzureStackHCI/clusters', 'Microsoft.ResourceConnector/appliances', 'Microsoft.ExtendedLocation/customLocations')) { + @{ type = $type; id = "/$type/test" } + } + } + elseif ($Arguments[0] -eq 'resource') { + @{ properties = @{ provisioningState = 'Succeeded'; connectivityStatus = 'Connected'; status = 'Running' + clusterExtensionIds = @('/test/hybridaksextension') } } + } + else { throw 'Unexpected Azure CLI call.' } + } + Mock Invoke-LocalBoxAutomationRequest { + if ($Method -eq 'PUT') { + $recoveryTest.Submitted++ + $recoveryTest.Body = $Body + $recoveryTest.Existing = @{ properties = @{ provisioningState = 'Succeeded'; parameters = $Body.properties.parameters } } + return + } + if ($Path -notlike '*runCommands*') { return @{ location = 'spaincentral' } } + if (-not $recoveryTest.Existing) { return } + $recoveryTest.Reads++ + $execution = if ($recoveryTest.Reads -lt 2) { 'Running' } else { 'Succeeded' } + $recoveryTest.Existing.properties.instanceView = @{ + executionState = $execution; exitCode = 0; startTime = [datetime]::UtcNow.ToString('o') + } + if ($execution -eq 'Succeeded') { $recoveryTest.Recovered = $true } + $recoveryTest.Existing + } + $retryParameters = @( + @{ name = 'SubscriptionId'; value = $testSubscription } + @{ name = 'ResourceGroupName'; value = $testGroup } + @{ name = 'OriginalValidationCorrelationId'; value = $originalId } + @{ name = 'StartedAfter'; value = [datetime]::UtcNow.AddMinutes(-1).ToString('o') } + @{ name = 'RecoveryVersion'; value = '1' } + ) + } + It 'submits exactly one asynchronous retry inside the existing timeout and still checks nested readiness' { + Wait-LocalBoxNestedDeployment -SubscriptionId $testSubscription -ResourceGroupName $testGroup -TimeoutSeconds 600 + $recoveryTest.Submitted | Should -Be 1 + $recoveryTest.Body.properties.asyncExecution | Should -BeTrue + $recoveryTest.Body.properties.timeoutInSeconds | Should -BeLessOrEqual 600 + $recoveryTest.Body.properties.timeoutInSeconds | Should -BeGreaterThan 120 + $recoveryTest.Body.properties.source.script | Should -Be (Get-Content "$root/labautomation/localbox-validation-retry.ps1" -Raw) + Should -Invoke Invoke-LocalBoxAutomationAz -Times 5 -Exactly -ParameterFilter { $Arguments[0] -eq 'resource' } + Should -Invoke Wait-LocalBoxAutomation -Times 1 -Exactly -ParameterFilter { $TimeoutSeconds -eq 600 } + } + It 'resumes an existing recovery without replacing it or spending another retry' { + $recoveryTest.Existing = @{ properties = @{ provisioningState = 'Succeeded'; parameters = $retryParameters } } + Wait-LocalBoxNestedDeployment -SubscriptionId $testSubscription -ResourceGroupName $testGroup -TimeoutSeconds 600 + $recoveryTest.Submitted | Should -Be 0 + } + It 'never retries a canceled validation' { + $recoveryTest.FailedState = 'Canceled' + { Wait-LocalBoxNestedDeployment -SubscriptionId $testSubscription -ResourceGroupName $testGroup } | Should -Throw '*canceled*' + $recoveryTest.Submitted | Should -Be 0 + } + It 'does not schedule recovery without enough time for its backoff' { + { Wait-LocalBoxNestedDeployment -SubscriptionId $testSubscription -ResourceGroupName $testGroup -TimeoutSeconds 120 } | Should -Throw '*Insufficient*' + $recoveryTest.Submitted | Should -Be 0 + } + It 'rejects unrelated failed operations' { + Mock Invoke-LocalBoxAutomationAz { + @{ properties = @{ provisioningState = 'Failed'; targetResource = @{ resourceType = 'Microsoft.KeyVault/vaults' }; statusMessage = @{ error = $downloadError } } } + } -ParameterFilter { $Arguments[1] -eq 'operation' } + { Wait-LocalBoxNestedDeployment -SubscriptionId $testSubscription -ResourceGroupName $testGroup } | Should -Throw '*outside the recognized*' + $recoveryTest.Submitted | Should -Be 0 + } + It 'surfaces operation-read errors without submitting recovery' { + Mock Invoke-LocalBoxAutomationAz { throw 'AuthorizationFailed' } -ParameterFilter { $Arguments[1] -eq 'operation' } + { Wait-LocalBoxNestedDeployment -SubscriptionId $testSubscription -ResourceGroupName $testGroup } | Should -Throw '*AuthorizationFailed*' + $recoveryTest.Submitted | Should -Be 0 + } + It 'rejects an unrecognized validation failure even when its operation type matches' { + Mock Invoke-LocalBoxAutomationAz { + @{ properties = @{ provisioningState = 'Failed' + targetResource = @{ resourceType = 'Microsoft.AzureStackHCI/clusters/deploymentSettings' } + statusMessage = @{ error = @{ code = 'UpdateDeploymentSettingsDataFailed'; message = 'Invalid configuration' } } } } + } -ParameterFilter { $Arguments[1] -eq 'operation' } + { Wait-LocalBoxNestedDeployment -SubscriptionId $testSubscription -ResourceGroupName $testGroup } | Should -Throw '*non-retryable*' + $recoveryTest.Submitted | Should -Be 0 + } + It 'does not replace a failed recovery when the Console is restarted' { + Mock Invoke-LocalBoxAutomationRequest { + @{ properties = @{ provisioningState = 'Succeeded'; parameters = $retryParameters + instanceView = @{ executionState = 'Failed'; exitCode = 1; startTime = [datetime]::UtcNow.ToString('o') } } } + } + { Wait-LocalBoxNestedDeployment -SubscriptionId $testSubscription -ResourceGroupName $testGroup } | Should -Throw '*Automatic validation retry failed*' + $recoveryTest.Submitted | Should -Be 0 + } + It 'rejects a second failed validation after successful recovery execution rather than looping' { + $recoveryTest.Existing = @{ properties = @{ provisioningState = 'Succeeded'; parameters = $retryParameters } } + Mock Invoke-LocalBoxAutomationAz { + @{ name = 'localcluster-validate'; properties = @{ provisioningState = 'Failed'; correlationId = $newId } } + } -ParameterFilter { $Arguments[0] -eq 'deployment' -and $Arguments[1] -eq 'group' } + { Wait-LocalBoxNestedDeployment -SubscriptionId $testSubscription -ResourceGroupName $testGroup } | Should -Throw '*No further automatic*' + $recoveryTest.Submitted | Should -Be 0 + } + It 'ignores stale execution success and rejects foreign recovery metadata' { + $command = @{ properties = @{ provisioningState = 'Succeeded'; parameters = $retryParameters + instanceView = @{ executionState = 'Succeeded'; exitCode = 0; startTime = [datetime]::UtcNow.AddHours(-1).ToString('o') } } } + (Get-LocalBoxValidationRetryState $command $testSubscription $testGroup).Ready | Should -BeFalse + { Get-LocalBoxValidationRetryState $command $testSubscription 'another-group' } | Should -Throw '*Unrecognized*' + } + It 'rejects nonzero and missing recovery exit codes even when execution says Succeeded' { + foreach ($code in @(1, $null)) { + $command = @{ properties = @{ provisioningState = 'Succeeded'; parameters = $retryParameters + instanceView = @{ executionState = 'Succeeded'; exitCode = $code; startTime = [datetime]::UtcNow.ToString('o') } } } + { Get-LocalBoxValidationRetryState $command $testSubscription $testGroup } | Should -Throw '*did not exit successfully*' + } + } +} + +Describe 'Validation recovery on LocalBox-Client' { + BeforeEach { + $guestRetryTest = @{ Creates = [Collections.Generic.List[object]]::new(); OriginalState = 'Failed'; OriginalId = $originalId + ValidationState = 'Succeeded'; DeploymentExists = $false; FailValidation = $false; Profile = $null } + $templateDir = Join-Path $TestDrive 'localbox' + New-Item -ItemType Directory -Path $templateDir -Force | Out-Null + $fixtureParameters = @{ + clusterName = @{ value = 'localboxcluster' }; keyVaultName = @{ value = 'localbox-kv-test' } + diagnosticStorageAccountName = @{ value = 'localboxdiagtest' } + arcNodeResourceIds = @{ value = @("/subscriptions/$testSubscription/resourceGroups/$testGroup/providers/Microsoft.HybridCompute/machines/node1") } + } + @{ parameters = $fixtureParameters } | ConvertTo-Json -Depth 10 | Set-Content "$templateDir/azlocal.parameters.json" + @{ parameters = @{ deploymentMode = @{ allowedValues = @('Validate', 'Deploy') } } } | + ConvertTo-Json -Depth 10 | Set-Content "$templateDir/azlocal.json" + Mock Start-Sleep {} + Mock Invoke-LocalBoxValidationRetryAz { + $guestRetryTest.Profile = $env:AZURE_CONFIG_DIR + switch ($Arguments[2]) { + 'show' { + if ($guestRetryTest.Creates.Count) { + @{ properties = @{ provisioningState = 'Succeeded'; correlationId = $newId } } + } + else { + @{ properties = @{ provisioningState = $guestRetryTest.OriginalState; correlationId = $guestRetryTest.OriginalId; parameters = $fixtureParameters } } + } + } + 'list' { + if ($guestRetryTest.DeploymentExists) { @{ name = 'localcluster-deploy' } } + } + 'create' { + $guestRetryTest.Creates.Add(@($Arguments)) + if ($Arguments -contains 'deploymentMode=Validate') { + if ($guestRetryTest.FailValidation) { throw 'Validation failed again' } + @{ properties = @{ provisioningState = $guestRetryTest.ValidationState; correlationId = $newId } } + } + } + default { + if ($Arguments[0] -ne 'login') { throw 'Unexpected guest CLI call.' } + } + } + } + $guestArguments = @{ SubscriptionId = $testSubscription; ResourceGroupName = $testGroup + OriginalValidationCorrelationId = $originalId; TemplateDirectory = $templateDir } + } + It 'waits, validates exactly once, then submits deploy without changing local credentials or files' { + $before = (Get-FileHash "$templateDir/azlocal.parameters.json").Hash + $previousProfile = $env:AZURE_CONFIG_DIR + Invoke-LocalBoxValidationRetry @guestArguments + Should -Invoke Start-Sleep -Times 1 -Exactly -ParameterFilter { $Seconds -eq 120 } + $guestRetryTest.Creates.Count | Should -Be 2 + $guestRetryTest.Creates[0] | Should -Contain 'deploymentMode=Validate' + $guestRetryTest.Creates[0] | Should -Not -Contain '--no-wait' + $guestRetryTest.Creates[1] | Should -Contain 'deploymentMode=Deploy' + $guestRetryTest.Creates[1] | Should -Contain '--no-wait' + (Get-FileHash "$templateDir/azlocal.parameters.json").Hash | Should -Be $before + $env:AZURE_CONFIG_DIR | Should -Be $previousProfile + Test-Path $guestRetryTest.Profile | Should -BeFalse + } + It 'never deploys if retry fails or returns a nonsuccess status' { + $guestRetryTest.FailValidation = $true + { Invoke-LocalBoxValidationRetry @guestArguments } | Should -Throw '*Validation failed again*' + $guestRetryTest.Creates.Count | Should -Be 1 + Test-Path $guestRetryTest.Profile | Should -BeFalse + $guestRetryTest.Creates.Clear() + $guestRetryTest.FailValidation = $false + $guestRetryTest.ValidationState = 'Failed' + { Invoke-LocalBoxValidationRetry @guestArguments } | Should -Throw '*did not return a new successful validation*' + $guestRetryTest.Creates.Count | Should -Be 1 + } + It 'rejects changed correlation, changed state, and an existing deployment' { + $guestRetryTest.OriginalId = $newId + { Invoke-LocalBoxValidationRetry @guestArguments } | Should -Throw '*Validation changed*' + $guestRetryTest.OriginalId = $originalId + $guestRetryTest.OriginalState = 'Running' + { Invoke-LocalBoxValidationRetry @guestArguments } | Should -Throw '*Validation changed*' + $guestRetryTest.OriginalState = 'Failed' + $guestRetryTest.DeploymentExists = $true + { Invoke-LocalBoxValidationRetry @guestArguments } | Should -Throw '*already exists*' + $guestRetryTest.Creates.Count | Should -Be 0 + } + It 'rejects parameters from a different generated environment' { + $fixtureParameters.clusterName.value = 'different-cluster' + { Invoke-LocalBoxValidationRetry @guestArguments } | Should -Throw '*does not match*' + $guestRetryTest.Creates.Count | Should -Be 0 + } + It 'does not deploy when another operation changes validation after the retry' { + Mock Invoke-LocalBoxValidationRetryAz { + @{ properties = @{ provisioningState = 'Running'; correlationId = $originalId } } + } -ParameterFilter { $Arguments[2] -eq 'show' -and $guestRetryTest.Creates.Count -gt 0 } + { Invoke-LocalBoxValidationRetry @guestArguments } | Should -Throw '*Validation changed before cluster deployment*' + $guestRetryTest.Creates.Count | Should -Be 1 + } + It 'does not submit a duplicate cluster deployment after successful validation' { + Mock Invoke-LocalBoxValidationRetryAz { @{ name = 'localcluster-deploy' } } -ParameterFilter { + $Arguments[2] -eq 'list' -and $guestRetryTest.Creates.Count -gt 0 + } + { Invoke-LocalBoxValidationRetry @guestArguments } | Should -Throw '*appeared during validation*' + $guestRetryTest.Creates.Count | Should -Be 1 + } + It 'fails explicitly on missing local deployment files' { + Remove-Item "$templateDir/azlocal.parameters.json" + { Invoke-LocalBoxValidationRetry @guestArguments } | Should -Throw '*file is missing*' + Should -Invoke Invoke-LocalBoxValidationRetryAz -Times 0 + } +} + +Describe 'Recovery CLI failure handling' { + It 'withholds credential-bearing native output on failure' { + Mock az { $global:LASTEXITCODE = 1; 'sensitive-fixture-value' } + $failure = { Invoke-LocalBoxValidationRetryAz @('deployment', 'group', 'create') } | Should -Throw '*CLI*failed*' -PassThru + $failure.Exception.Message | Should -Not -Match 'sensitive-fixture-value' + } +} diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md index 9adffa674..89297415a 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md @@ -45,6 +45,16 @@ required to validate regional service availability and effective inherited polic ## Global event LocalBox checks +Run `Invoke-Pester ./localbox-validation-retry.tests.ps1 -Output Detailed` for +the bounded Azure Local validation recovery. These mocked tests cover the exact +content-download timeout classifier, the unchanged nested wait budget, one +asynchronous recovery, Console restart/resume, stale execution state, error +propagation, existing local-file/correlation checks, managed-identity CLI-profile +cleanup, and validation-before-deploy ordering. They never submit Azure +deployments. The reported traininglab-58 failure could not be read with the +available operator permissions; real retry and subsequent cluster deployment +remain a fresh-Console integration check. + Run `Invoke-Pester ./localbox-deployment.tests.ps1 -Output Detailed` for the LocalBox Developer Bastion deployment contract. These tests require Azure CLI with Bicep available: they compile the network module locally, then mock Azure From 853c4e0a3c4921f83e2a7d6a4905cd8aaa1189c5 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Thu, 5 Nov 2026 07:36:53 +0000 Subject: [PATCH 12/37] Surface actionable LocalBox guest failure diagnostics in Console Report correlated guest phases, NUnit failure counts and bounded redacted error details while preserving strict readiness gates and the original failure. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../labautomation/localbox-automation.ps1 | 156 ++++++++++++++++-- .../resources/hosted-events/readme.md | 27 ++- .../tests/localbox-automation.tests.ps1 | 12 +- .../tests/localbox-failure-summary.tests.ps1 | 132 +++++++++++++++ .../resources/tests/readme.md | 13 ++ 5 files changed, 325 insertions(+), 15 deletions(-) create mode 100644 03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-failure-summary.tests.ps1 diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation.ps1 index 1851c4633..19371b76e 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation.ps1 @@ -285,11 +285,19 @@ try { '@.Replace('__CONFIG__', $configurationBase64).Replace('__ARCHIVE__', $Archive) } -function Assert-LocalBoxAutomationResult { +function Assert-LocalBoxAutomationReportIdentity { param([hashtable]$Result, [string]$RunId, [string]$PackageHash, [string]$GroupId, [string]$SubscriptionId, [string]$ResourceGroupName) - if ($Result.RunId -ne $RunId -or $Result.PackageHash -ne $PackageHash -or + if (-not $Result -or $Result.RunId -ne $RunId -or $Result.PackageHash -ne $PackageHash -or $Result.AksAdminGroupObjectId -ne $GroupId -or $Result.SubscriptionId -ne $SubscriptionId -or - $Result.ResourceGroupName -ne $ResourceGroupName -or $Result.State -ne 'Succeeded') { + $Result.ResourceGroupName -ne $ResourceGroupName) { + throw 'LocalBox diagnostic report is missing, stale or belongs to a different run, package, group or Azure scope.' + } +} + +function Assert-LocalBoxAutomationResult { + param([hashtable]$Result, [string]$RunId, [string]$PackageHash, [string]$GroupId, [string]$SubscriptionId, [string]$ResourceGroupName) + Assert-LocalBoxAutomationReportIdentity @PSBoundParameters + if ($Result.State -ne 'Succeeded') { throw 'LocalBox automation result is missing, stale, mismatched or unsuccessful.' } foreach ($suite in @('Upstream', 'Sovereign')) { @@ -303,24 +311,33 @@ function Assert-LocalBoxAutomationResult { } } -function Assert-LocalBoxAutomationXml { - param([string]$Path, [hashtable]$Expected) +function Read-LocalBoxAutomationXml { + param([string]$Path) $settings = [Xml.XmlReaderSettings]::new() $settings.DtdProcessing = [Xml.DtdProcessing]::Prohibit + $settings.XmlResolver = $null + $settings.MaxCharactersInDocument = 10485760 $reader = [Xml.XmlReader]::Create($Path, $settings) try { $document = [Xml.XmlDocument]::new() $document.XmlResolver = $null $document.Load($reader) - $cases = @($document.SelectNodes('//test-case')) - if ($document.DocumentElement.get_LocalName() -ne 'test-results' -or $cases.Count -ne $Expected.Passed -or - @($cases | Where-Object { $_.GetAttribute('result') -ne 'Success' -or $_.GetAttribute('executed') -ne 'True' }).Count) { - throw "Pester XML does not match the successful test summary: $Path" - } + if ($document.DocumentElement.get_LocalName() -ne 'test-results') { throw 'Expected a Pester NUnit report.' } + return ,$document } finally { $reader.Dispose() } } +function Assert-LocalBoxAutomationXml { + param([string]$Path, [hashtable]$Expected) + $document = Read-LocalBoxAutomationXml -Path $Path + $cases = @($document.SelectNodes('//test-case')) + if ($cases.Count -ne $Expected.Passed -or + @($cases | Where-Object { $_.GetAttribute('result') -ne 'Success' -or $_.GetAttribute('executed') -ne 'True' }).Count) { + throw "Pester XML does not match the successful test summary: $Path" + } +} + function Write-LocalBoxAutomationSuiteSummary { param([ValidateSet('Upstream', 'Sovereign')][string]$Name, [hashtable]$Counts) foreach ($field in @('Passed', 'Failed', 'Skipped', 'NotRun')) { @@ -332,6 +349,111 @@ function Write-LocalBoxAutomationSuiteSummary { Write-Host "[LocalBox] $Name tests: $($Counts.Passed) passed, $($Counts.Failed) failed, $($Counts.Skipped) skipped, $($Counts.NotRun) not run." } +function ConvertTo-LocalBoxDiagnosticText { + param([string]$Text, [ValidateRange(40, [int]::MaxValue)][int]$MaximumLength = 800) + if ([string]::IsNullOrWhiteSpace($Text)) { return '(not reported)' } + $text = [regex]::Replace($Text, '\x1b\[[0-?]*[ -/]*[@-~]', '') + $text = [regex]::Replace($text, '(?s)-----BEGIN [^-]*PRIVATE KEY-----.*?-----END [^-]*PRIVATE KEY-----', '[REDACTED PRIVATE KEY]') + $text = [regex]::Replace($text, '(?i)\bBearer\s+[^\s"''<>]+', 'Bearer [REDACTED]') + $text = [regex]::Replace($text, '\beyJ[A-Za-z0-9_-]+\.eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+', '[REDACTED JWT]') + $text = [regex]::Replace($text, '(?i)(https?://)[^/\s@]+@', '$1[REDACTED]@') + $text = [regex]::Replace($text, '(?i)(https?://[^\s?]+)\?[^\s"''<>]+', '$1?[REDACTED QUERY]') + $text = [regex]::Replace($text, + '(?i)(["'']?[\w-]*(?:password|secret|token|accountkey|connectionstring|authorization)["'']?\s*[:=]\s*)(?:"[^"]*"|''[^'']*''|[^\s,;]+)', '$1[REDACTED]') + $text = [regex]::Replace($text, + '(?i)(--?[\w-]*(?:password|secret|token|account-key|connection-string)\s+)(?:"[^"]*"|''[^'']*''|\S+)', '$1[REDACTED]') + $text = [regex]::Replace($text, '[\x00-\x1f\x7f-\x9f]', ' ') + $text = [regex]::Replace($text, '\s+', ' ').Trim() + if ($text.Length -gt $MaximumLength) { + $suffix = ' ... [truncated; see private report]' + return $text.Substring(0, $MaximumLength - $suffix.Length) + $suffix + } + $text +} + +function Get-LocalBoxAutomationFailureSuite { + param([string]$Path) + $document = Read-LocalBoxAutomationXml -Path $Path + $cases = @($document.SelectNodes('//test-case')) + if (-not $cases.Count) { throw 'The test report contains no test cases.' } + $counts = @{ Passed = 0; Failed = 0; Skipped = 0; NotRun = 0 } + $failures = @( + foreach ($case in $cases) { + switch ($case.GetAttribute('result')) { + 'Success' { + if ($case.GetAttribute('executed') -ne 'True') { throw 'An unexecuted test claims success.' } + $counts.Passed++ + } + { $_ -in @('Failure', 'Error') } { + $counts.Failed++ + $message = $case.SelectSingleNode('failure/message') + @{ Name = $case.GetAttribute('name'); Message = if ($message) { $message.InnerText } else { 'No failure message recorded.' } } + } + { $_ -in @('Ignored', 'Skipped') } { $counts.Skipped++ } + { $_ -in @('Inconclusive', 'NotRunnable', 'Invalid') } { $counts.NotRun++ } + default { throw 'The test report contains an unsupported result value.' } + } + } + ) + @{ Counts = $counts; Failures = $failures } +} + +function Write-LocalBoxAutomationFailureReport { + param([string]$Directory, [string]$RunId, [string]$PackageHash, [string]$GroupId, + [string]$SubscriptionId, [string]$ResourceGroupName) + try { + $status = Get-Content -LiteralPath "$Directory/status.json" -Raw -ErrorAction Stop | ConvertFrom-Json -AsHashtable -ErrorAction Stop + Assert-LocalBoxAutomationReportIdentity -Result $status -RunId $RunId -PackageHash $PackageHash -GroupId $GroupId ` + -SubscriptionId $SubscriptionId -ResourceGroupName $ResourceGroupName + } + catch { + Write-Warning '[LocalBox] Detailed report unavailable or unverified: status.json could not be read and correlated. No test counts from these files will be displayed.' + return + } + Write-Host "[LocalBox] Last guest phase: $(ConvertTo-LocalBoxDiagnosticText $status.Phase 120); reported state: $(ConvertTo-LocalBoxDiagnosticText $status.State 80)." + if ($status.Error) { Write-Host "[LocalBox] Guest error: $(ConvertTo-LocalBoxDiagnosticText $status.Error)" } + foreach ($suite in @(@{ Name = 'Upstream'; File = 'upstream.xml' }, @{ Name = 'Sovereign'; File = 'health.xml' })) { + try { + $report = Get-LocalBoxAutomationFailureSuite -Path (Join-Path $Directory $suite.File) + Write-LocalBoxAutomationSuiteSummary -Name $suite.Name -Counts $report.Counts + foreach ($failure in @($report.Failures | Select-Object -First 10)) { + Write-Host "[LocalBox] FAILED TEST ($($suite.Name)): $(ConvertTo-LocalBoxDiagnosticText $failure.Name 300)" + Write-Host "[LocalBox] Reason: $(ConvertTo-LocalBoxDiagnosticText $failure.Message)" + } + if ($report.Failures.Count -gt 10) { Write-Host "[LocalBox] $($report.Failures.Count - 10) additional failed tests are listed in $($suite.File)." } + if (@($report.Failures | Where-Object { + $_.Message -match 'ipaddresspools\.metallb\.io' -and $_.Message -match 'not found|NotFound' + }).Count) { + Write-Host '[LocalBox] Suggested check: compare the Azure-managed MetalLB pool with its Kubernetes IPAddressPool and L2Advertisement. Azure Succeeded does not prove these objects exist; have an organizer reconcile the existing configuration rather than rebuild LocalBox.' + } + } + catch { Write-Warning "[LocalBox] $($suite.Name) test results unavailable or invalid ($($suite.File)); this suite is NOT VERIFIED." } + } + Write-Host '[LocalBox] These diagnostic counts do not establish overall readiness. Fix the reported failure, then retry the existing Console event; downstream stages remain blocked.' +} + +function Write-LocalBoxAutomationFailureDiagnostics { + param([string]$Directory, [string]$StorageAccountName, [string]$Container, [string]$RunId, + [string]$PackageHash, [string]$GroupId, [string]$SubscriptionId, [string]$ResourceGroupName, $InstanceView) + $exitCode = if ($null -eq $InstanceView.exitCode) { 'not reported' } else { ConvertTo-LocalBoxDiagnosticText ([string]$InstanceView.exitCode) 40 } + Write-Host "[LocalBox] Run: $RunId; execution: $(ConvertTo-LocalBoxDiagnosticText $InstanceView.executionState 80); exit code: $exitCode." + if ($InstanceView.error) { Write-Host "[LocalBox] Execution error: $(ConvertTo-LocalBoxDiagnosticText $InstanceView.error)" } + foreach ($name in @('status.json', 'upstream.xml', 'health.xml')) { + $path = Join-Path $Directory $name + if (Test-Path -LiteralPath $path) { Remove-Item -LiteralPath $path -Force -ErrorAction Stop } + try { + Invoke-LocalBoxAutomationAz @('storage', 'blob', 'download', '--account-name', $StorageAccountName, '--container-name', $Container, + '--name', "$RunId/$name", '--file', $path, '--auth-mode', 'login', '--overwrite', 'true', '--no-progress') -NoOutput + } + catch { + if (Test-Path -LiteralPath $path) { Remove-Item -LiteralPath $path -Force -ErrorAction Stop } + Write-Warning "[LocalBox] Could not retrieve $name for this run: $(ConvertTo-LocalBoxDiagnosticText $_.Exception.Message 400)" + } + } + Write-LocalBoxAutomationFailureReport -Directory $Directory -RunId $RunId -PackageHash $PackageHash -GroupId $GroupId ` + -SubscriptionId $SubscriptionId -ResourceGroupName $ResourceGroupName +} + function Invoke-LocalBoxHostedAutomation { [CmdletBinding()] param( @@ -351,6 +473,7 @@ function Invoke-LocalBoxHostedAutomation { $commandUri = "$commandPath`?api-version=2024-07-01" $temporary = Join-Path ([IO.Path]::GetTempPath()) "localbox-automation-$([guid]::NewGuid())" $resultUri = $null + $failureEvidence = @{ InstanceView = $null } New-Item -ItemType Directory -Path $temporary | Out-Null try { if (-not $Package) { $Package = New-LocalBoxAutomationPackage -ResourceRoot $ResourceRoot -Directory $temporary } @@ -433,9 +556,11 @@ function Invoke-LocalBoxHostedAutomation { return @{ Ready = $false; Detail = "Waiting for execution $runId to start; ignoring previous instanceView." } } if ($state.executionState -in @('Failed', 'Canceled', 'TimedOut')) { + $failureEvidence.InstanceView = $state throw "LocalBox guest automation failed. Inspect $commandPath instanceView and $resultUri." } if ($state.executionState -eq 'Succeeded') { + $failureEvidence.InstanceView = $state if ($null -eq $state.exitCode -or $state.exitCode -ne 0) { throw "LocalBox guest exit code was not zero. Inspect $resultUri." } return @{ Ready = $true } } @@ -462,10 +587,19 @@ function Invoke-LocalBoxHostedAutomation { @{ RunId = $runId; ResultsUri = $resultUri } } catch { + $primaryFailure = $_ Write-Host '[LocalBox] Readiness gate: FAILED - infrastructure health is NOT VERIFIED.' -ForegroundColor Red Write-Host '[LocalBox] No successful end-of-run test summary was verified. See the error and diagnostic reports; do not treat deployment as ready.' + if ($failureEvidence.InstanceView -and $resultUri) { + try { + Write-LocalBoxAutomationFailureDiagnostics -Directory $temporary -StorageAccountName $account.name -Container $container ` + -RunId $runId -PackageHash $package.Hash -GroupId $AksAdminGroupObjectId.ToString() ` + -SubscriptionId $SubscriptionId -ResourceGroupName $ResourceGroupName -InstanceView $failureEvidence.InstanceView + } + catch { Write-Warning "[LocalBox] Failure diagnostics could not be completed: $(ConvertTo-LocalBoxDiagnosticText $_.Exception.Message 400)" } + } if ($resultUri) { Write-Host "[LocalBox] Private diagnostic reports: $resultUri (Entra authentication required)." } - throw + throw $primaryFailure } finally { Remove-Item -LiteralPath $temporary -Recurse -Force -ErrorAction Stop } } diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md index 078aba1f9..6d37cad89 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md @@ -194,13 +194,34 @@ The [orchestrator](../../labautomation/localbox-automation.ps1) implements: sign-in remain explicitly listed as manual checks. A failed execution, unavailable report or failed validation instead prints **Readiness gate: FAILED - infrastructure health is NOT VERIFIED** and - propagates the error. It does not invent test counts or print a passing - summary when reports could not be verified; inspect the private diagnostics - for individual failed tests. These summaries do not stream guest progress. + propagates the error. For a correlated terminal guest execution, Console also + attempts to retrieve the original run's status and NUnit reports. It displays + the run ID, execution/exit status, last guest phase, bounded error details, + actual suite counts, and up to ten failed test names/reasons per suite. + Missing or invalid reports are explicitly **unavailable / NOT VERIFIED**; + mismatched run/content/group/scope metadata suppresses all report-derived + counts. A diagnostic-download failure never replaces the original failure. + Output strips terminal control codes, redacts common credential formats, and + does not dump transcripts, XML stack traces or raw guest stdout. Full reports + remain private. These summaries do not stream guest progress. Result downloads suppress CLI progress and discard download-command metadata; validation reads the downloaded JSON/XML files themselves. Native CLI stderr is kept separate from JSON parsing, and nonzero CLI exits still fail the gate. +For example, the original traininglab-02 MetalLB failure now renders: + +```text +[LocalBox] Last guest phase: Pester; reported state: Failed. +[LocalBox] Upstream tests: 12 passed, 0 failed, 0 skipped, 0 not run. +[LocalBox] Sovereign tests: 13 passed, 1 failed, 0 skipped, 0 not run. +[LocalBox] FAILED TEST (Sovereign): ...has an in-cluster MetalLB address pool and L2 advertisement +[LocalBox] Reason: ...ipaddresspools.metallb.io "aks-pool" not found +``` + +That failure also prints a suggested check of the Azure-managed pool against the +Kubernetes IPAddressPool/L2Advertisement, rather than advising an environment +rebuild. Diagnostic counts never change the overall failed readiness outcome. + `Update-MhhToken` runs in the **Console process** on each poll and before Azure operations, including result retrieval. It refreshes Console clients, not guest credentials or job time limits. The guest uses its own managed identity and diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-automation.tests.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-automation.tests.ps1 index d97cebb9a..920513542 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-automation.tests.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-automation.tests.ps1 @@ -380,15 +380,25 @@ Describe 'Hosted LocalBox automation lifecycle' { } It 'fails on nonzero, absent or failed guest execution results' -TestCases @( @{ Code = 1; State = 'Succeeded' }; @{ Code = $null; State = 'Succeeded' }; @{ Code = 0; State = 'Failed' } + @{ Code = 1; State = 'Canceled' }; @{ Code = 1; State = 'TimedOut' } ) { param($Code, $State) $tracking.ExitCode = $Code $tracking.ExecutionState = $State { Invoke-LocalBoxHostedAutomation $subscription rg-localbox-shared $group } | Should -Throw '*LocalBox guest*' - Should -Invoke Invoke-LocalBoxAutomationAz -Times 0 -ParameterFilter { $Arguments[2] -eq 'download' } + Should -Invoke Invoke-LocalBoxAutomationAz -Times 3 -Exactly -ParameterFilter { + $Arguments[2] -eq 'download' -and $Arguments -contains '--no-progress' -and $NoOutput + } Should -Invoke Write-Host -Times 0 -ParameterFilter { $Object -like '*Infrastructure health: PASSED*' } Should -Invoke Write-Host -Times 1 -Exactly -ParameterFilter { $Object -like '*Readiness gate: FAILED*NOT VERIFIED*' } } + It 'preserves the original guest failure if diagnostic retrieval also fails' { + $tracking.ExecutionState = 'Failed' + Mock Invoke-LocalBoxAutomationAz { throw 'Blob access denied' } -ParameterFilter { $Arguments[2] -eq 'download' } + { Invoke-LocalBoxHostedAutomation $subscription rg-localbox-shared $group } | Should -Throw '*LocalBox guest automation failed*' + Should -Invoke Write-Warning -Times 3 -Exactly -ParameterFilter { $Message -like '*Could not retrieve*Blob access denied*' } + Should -Invoke Write-Host -Times 0 -ParameterFilter { $Object -like '*Infrastructure health: PASSED*' } + } It 'does not overwrite an active command from another revision or group' { $tracking.Existing = @{ properties = @{ instanceView = @{ executionState = 'Running' }; parameters = @(@{ name = 'PackageHash'; value = 'old' }) } } { Invoke-LocalBoxHostedAutomation $subscription rg-localbox-shared $group } | Should -Throw '*different content or group*' diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-failure-summary.tests.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-failure-summary.tests.ps1 new file mode 100644 index 000000000..6568a3d9c --- /dev/null +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-failure-summary.tests.ps1 @@ -0,0 +1,132 @@ +BeforeAll { + . "$PSScriptRoot/../../labautomation/localbox-automation.ps1" + $scopeArguments = @{ + RunId = '682105d5-dfa6-4cba-a646-74ee20fe7121'; PackageHash = 'fixture-package' + GroupId = '10000000-0000-0000-0000-000000000001' + SubscriptionId = '20000000-0000-0000-0000-000000000001'; ResourceGroupName = 'rg-localbox-test' + } +} + +Describe 'Actionable failure summaries' { + BeforeEach { + $directory = Join-Path $TestDrive 'reports' + New-Item -ItemType Directory -Path $directory -Force | Out-Null + $status = @{ + RunId = $scopeArguments.RunId; PackageHash = $scopeArguments.PackageHash + AksAdminGroupObjectId = $scopeArguments.GroupId; SubscriptionId = $scopeArguments.SubscriptionId + ResourceGroupName = $scopeArguments.ResourceGroupName; State = 'Failed'; Phase = 'Pester' + Error = 'Sovereign Cloud Full health tests failed.' + } + $status | ConvertTo-Json | Set-Content "$directory/status.json" + $passing = '' + "$($passing * 12)" | Set-Content "$directory/upstream.xml" + $failure = 'private-stack-not-for-console' + "$($passing * 13)$failure" | Set-Content "$directory/health.xml" + Mock Write-Host {} + Mock Write-Warning {} + } + It 'reports the exact failing phase, both suite counts, failed test and underlying Kubernetes error' { + Write-LocalBoxAutomationFailureReport -Directory $directory @scopeArguments + Should -Invoke Write-Host -Times 1 -Exactly -ParameterFilter { $Object -like '*Last guest phase: Pester; reported state: Failed*' } + Should -Invoke Write-Host -Times 1 -Exactly -ParameterFilter { $Object -like '*Upstream tests: 12 passed, 0 failed*' } + Should -Invoke Write-Host -Times 1 -Exactly -ParameterFilter { $Object -like '*Sovereign tests: 13 passed, 1 failed*' } + Should -Invoke Write-Host -Times 1 -Exactly -ParameterFilter { $Object -like '*FAILED TEST*MetalLB address pool*' } + Should -Invoke Write-Host -Times 1 -Exactly -ParameterFilter { $Object -like '*Reason:*aks-pool*not found*' } + Should -Invoke Write-Host -Times 1 -Exactly -ParameterFilter { $Object -like '*Suggested check:*Azure-managed MetalLB*' } + Should -Invoke Write-Host -Times 0 -ParameterFilter { $Object -match 'private-stack-not-for-console|Infrastructure health: PASSED' } + } + It 'rejects a mismatched before displaying test counts' -TestCases @( + @{ Field = 'RunId' }, @{ Field = 'PackageHash' }, @{ Field = 'AksAdminGroupObjectId' }, + @{ Field = 'SubscriptionId' }, @{ Field = 'ResourceGroupName' } + ) { + param($Field) + $status[$Field] = 'different' + $status | ConvertTo-Json | Set-Content "$directory/status.json" + Write-LocalBoxAutomationFailureReport -Directory $directory @scopeArguments + Should -Invoke Write-Host -Times 0 + Should -Invoke Write-Warning -Times 1 -Exactly -ParameterFilter { $Message -like '*unavailable or unverified*' } + } + It 'keeps phase and upstream evidence when Sovereign tests never produced a report' { + Remove-Item "$directory/health.xml" + Write-LocalBoxAutomationFailureReport -Directory $directory @scopeArguments + Should -Invoke Write-Host -Times 1 -Exactly -ParameterFilter { $Object -like '*Upstream tests: 12 passed*' } + Should -Invoke Write-Warning -Times 1 -Exactly -ParameterFilter { $Message -like '*Sovereign*NOT VERIFIED*' } + Should -Invoke Write-Host -Times 0 -ParameterFilter { $Object -like '*Sovereign tests:*' } + } + It 'displays no report-derived evidence for status' -TestCases @( + @{ Kind = 'missing' }, @{ Kind = 'malformed' } + ) { + param($Kind) + if ($Kind -eq 'missing') { Remove-Item "$directory/status.json" } + else { '{broken' | Set-Content "$directory/status.json" } + Write-LocalBoxAutomationFailureReport -Directory $directory @scopeArguments + Should -Invoke Write-Host -Times 0 + Should -Invoke Write-Warning -Times 1 -Exactly -ParameterFilter { $Message -like '*unavailable or unverified*' } + } + It 'rejects malformed, empty, wrong-root and entity-bearing XML' { + foreach ($xml in @( + '', '', + '', + ']>&bad;' + )) { + $xml | Set-Content "$directory/health.xml" + { Get-LocalBoxAutomationFailureSuite "$directory/health.xml" } | Should -Throw + } + } + It 'does not count an unexecuted success as passed' { + '' | Set-Content "$directory/health.xml" + { Get-LocalBoxAutomationFailureSuite "$directory/health.xml" } | Should -Throw '*unexecuted*' + } + It 'counts ignored and inconclusive cases without treating them as successful' { + '' | + Set-Content "$directory/health.xml" + $report = Get-LocalBoxAutomationFailureSuite "$directory/health.xml" + $report.Counts.Passed | Should -Be 0 + $report.Counts.Skipped | Should -Be 1 + $report.Counts.NotRun | Should -Be 1 + } + It 'bounds failed-test details without losing the total failure count' { + "$($failure * 12)" | Set-Content "$directory/health.xml" + Write-LocalBoxAutomationFailureReport -Directory $directory @scopeArguments + Should -Invoke Write-Host -Times 10 -Exactly -ParameterFilter { $Object -like '*FAILED TEST (*' } + Should -Invoke Write-Host -Times 1 -Exactly -ParameterFilter { $Object -like '*Sovereign tests: 0 passed, 12 failed*' } + Should -Invoke Write-Host -Times 1 -Exactly -ParameterFilter { $Object -like '*2 additional failed tests*' } + } + It 'retains a correlated execution error when all report downloads are denied' { + Mock Invoke-LocalBoxAutomationAz { throw 'Permission denied' } + Write-LocalBoxAutomationFailureDiagnostics -Directory $directory -StorageAccountName fixture -Container fixture ` + -InstanceView @{ executionState = 'Failed'; exitCode = 1; error = 'Health runner failed.' } @scopeArguments + Should -Invoke Invoke-LocalBoxAutomationAz -Times 3 -Exactly -ParameterFilter { + $Arguments -contains '--no-progress' -and $Arguments -contains '--auth-mode' -and $NoOutput + } + Should -Invoke Write-Host -Times 1 -Exactly -ParameterFilter { $Object -like '*execution: Failed; exit code: 1*' } + Should -Invoke Write-Host -Times 1 -Exactly -ParameterFilter { $Object -like '*Execution error: Health runner failed*' } + Should -Invoke Write-Host -Times 0 -ParameterFilter { $Object -like '*tests:*' } + Test-Path "$directory/status.json" | Should -BeFalse + } +} + +Describe 'Console diagnostic redaction' { + It 'redacts common credentials and URL credentials before truncating' { + $text = @' +password="private spaced password" --admin-password 'private cli password' +{"access_token":"private-token"} AccountKey=private-key; Bearer private-bearer +https://user:private-userinfo@example.test/path?sig=private-signature&other=value +eyJheader.eyJpayload.privatesignature +-----BEGIN PRIVATE KEY----- +private-key-material +-----END PRIVATE KEY----- +'@ + $rendered = ConvertTo-LocalBoxDiagnosticText $text 4000 + $rendered | Should -Not -Match 'private-|private spaced|private cli|privatesignature' + $rendered | Should -Match 'REDACTED' + } + It 'strips terminal control characters and bounds long errors' { + $text = "$([char]27)[31mError$([char]27)[0m`n" + ('x' * 1000) + $rendered = ConvertTo-LocalBoxDiagnosticText $text 80 + $rendered | Should -Not -Match '[\x00-\x1f]' + $rendered | Should -Match '^Error ' + $rendered | Should -Match '\[truncated; see private report\]$' + $rendered.Length | Should -Be 80 + } +} diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md index 89297415a..258bd0fa9 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md @@ -45,6 +45,19 @@ required to validate regional service availability and effective inherited polic ## Global event LocalBox checks +Run `Invoke-Pester ./localbox-failure-summary.tests.ps1,./localbox-automation.tests.ps1 -Output Detailed` +for Console failure reporting. Tests cover run/content/group/scope correlation, +real NUnit failure counts and messages, missing/malformed reports, prohibited XML +entities, bounded output, credential redaction, and preservation of the original +guest error when report downloads fail. No transcript or raw guest stdout is +printed as a diagnostic fallback. + +The formatter was also verified on traininglab-02 against its original failed +run (`682105d5-dfa6-4cba-a646-74ee20fe7121`): Pester phase, upstream 12/12, +Sovereign 13 passed / 1 failed, and the missing `aks-pool` Kubernetes error. +This read-only report check did not rerun the Console job or change readiness; +the later passing MetalLB repair reports were not used as the failure fixture. + Run `Invoke-Pester ./localbox-validation-retry.tests.ps1 -Output Detailed` for the bounded Azure Local validation recovery. These mocked tests cover the exact content-download timeout classifier, the unchanged nested wait budget, one From a4188387649a52623f510f4d4873253c79d679f6 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Thu, 5 Nov 2026 09:40:31 +0000 Subject: [PATCH 13/37] Recover missing MetalLB runtime objects with one guarded reapply Verify in-cluster state before hosted health tests, allow one unchanged Azure pool PUT after a bounded grace period, persist the attempt before submission and require runtime convergence. Preserve read-only health checks and route cleanup. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../resources/hosted-events/readme.md | 27 +++ .../resources/invoke-localbox-automation.ps1 | 135 ++++++++++- .../resources/prepare-localbox.ps1 | 8 +- .../tests/localbox-automation.tests.ps1 | 24 +- .../tests/localbox-metallb-recovery.tests.ps1 | 223 ++++++++++++++++++ .../resources/tests/readme.md | 13 + 6 files changed, 422 insertions(+), 8 deletions(-) create mode 100644 03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-metallb-recovery.tests.ps1 diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md index 6d37cad89..dcfe2b5c8 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md @@ -183,6 +183,33 @@ The [orchestrator](../../labautomation/localbox-automation.ps1) implements: If the parent process is forcibly terminated, its `finally` cannot run; inspect the API host route before retrying rather than deleting existing routes indiscriminately. ActiveStore routes do not survive a reboot. + + Before starting the Sovereign suite, the hosted runner uses that same + temporary route to check the in-cluster MetalLB IPAddressPool and + L2Advertisement. Missing objects get a **120-second reconciliation grace + period**, then at most **one unchanged Azure-managed pool PUT per guest run**. + Recovery requires a matching deployment manifest and network reservation, + exactly the expected Azure pool, and an already successful Arc networking + extension and ARP pool. Additional pools, conflicting settings, deletion, + authentication/connectivity errors, missing CRDs and malformed responses stop + the run rather than trigger a repair. No Kubernetes objects are directly + applied or deleted. + + After the PUT, the runner polls for up to **300 seconds**, requiring both Azure + success and the expected Kubernetes objects before continuing with all Full + health tests. Individual Kubernetes requests retain their 20-second timeout; + recovery ARM reads use 30 seconds and the single PUT uses 120 seconds. + An in-flight request can finish after a polling deadline. + The attempt is recorded **before submission** in the private + `metallb-recovery.json` report. A timeout or restart cannot submit another PUT + for that run. A resumed run may proceed if objects have since become healthy; + otherwise it stops for organizer review. An explicitly retriggered Console + deployment with a new run ID has a new one-attempt budget. + Route cleanup still runs on recovery failure. Standalone health checks and + `-HealthOnly` remain read-only; this recovery is enabled by hosted deployment, + not by the health tests. It mitigates the observed Azure/Kubernetes state + mismatch, but does not establish the underlying provider cause or guarantee + against future drift. 7. Require a zero guest exit code, matching run/content/group/scope metadata, nonempty passing suites without skipped/not-run tests, and matching NUnit XML before publishing global discovery credentials and returning to Console. diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/invoke-localbox-automation.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/invoke-localbox-automation.ps1 index 2859168e6..b5f1f5a41 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/invoke-localbox-automation.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/invoke-localbox-automation.ps1 @@ -62,7 +62,7 @@ function Assert-LocalBoxAutomationTests { function Send-LocalBoxAutomationFiles { param([hashtable]$Configuration, [string]$Directory) - foreach ($name in @('status.json', 'health.json', 'health.xml', 'upstream.xml', 'automation.log')) { + foreach ($name in @('status.json', 'health.json', 'health.xml', 'upstream.xml', 'automation.log', 'metallb-recovery.json')) { $path = Join-Path $Directory $name if (Test-Path -LiteralPath $path) { Invoke-LocalBoxAz @('storage', 'blob', 'upload', '--account-name', $Configuration.StorageAccountName, @@ -99,8 +99,132 @@ function Invoke-LocalBoxAutomationHealthProcess { if ($LASTEXITCODE -ne 0) { throw 'Sovereign Cloud Full health tests failed. Inspect health.xml and health.json.' } } +function Test-LocalBoxAutomationMetalLbRuntime { + param([string]$Kubeconfig, [hashtable]$Plan) + $pools = Invoke-SovereignKubectl $Kubeconfig @('get', 'ipaddresspools.metallb.io', '--all-namespaces') + $advertisements = Invoke-SovereignKubectl $Kubeconfig @('get', 'l2advertisements.metallb.io', '--all-namespaces') + foreach ($list in @($pools, $advertisements)) { + if ($list -isnot [System.Collections.IDictionary] -or $list.items -isnot [System.Collections.IList]) { + throw 'Invalid MetalLB Kubernetes list response; missing objects cannot be determined safely.' + } + } + $expected = @($pools.items | Where-Object { $_.metadata.name -eq 'aks-pool' -and $_.metadata.namespace -eq 'kube-system' }) + if ($pools.items.Count -ne $expected.Count -or $expected.Count -gt 1) { + throw 'Conflicting MetalLB IPAddressPool objects exist. Organizer review is required; no automatic repair is allowed.' + } + if ($expected.Count) { + $pool = $expected[0] + Assert-LocalBoxProperties $pool.spec @{ addresses = $Plan.PoolExpected.properties.addresses } + if ($pool.spec.autoAssign -eq $false -or $pool.metadata.deletionTimestamp) { + throw 'The MetalLB pool disables auto-assignment or is being deleted; no automatic repair is allowed.' + } + } + $matching = @($advertisements.items | Where-Object { + $_.metadata.namespace -eq 'kube-system' -and $_.spec.ipAddressPools -contains 'aks-pool' + }) + foreach ($advertisement in @($advertisements.items | Where-Object { + $_.metadata.namespace -eq 'kube-system' -and + ($_.metadata.name -eq 'aks-pool' -or $_.spec.ipAddressPools -contains 'aks-pool') + })) { + Assert-LocalBoxProperties $advertisement.spec @{ ipAddressPools = @('aks-pool') } + foreach ($key in @('interfaces', 'nodeSelectors', 'ipAddressPoolSelectors')) { + if ($advertisement.spec[$key].Count -gt 0) { + throw "The MetalLB L2Advertisement has custom $key configured; organizer review is required." + } + } + } + if (@($matching | Where-Object { $_.metadata.deletionTimestamp }).Count) { + throw 'A MetalLB L2Advertisement is being deleted; no automatic repair is allowed.' + } + return ($expected.Count -eq 1 -and $matching.Count -gt 0) +} + +function Sync-LocalBoxAutomationMetalLbRuntime { + param( + [hashtable]$Manifest, [hashtable]$Configuration, [string]$Kubeconfig, + [Parameter(Mandatory)][string]$StatePath, + [ValidateRange(0, 600)][int]$GraceSeconds = 120, + [ValidateRange(0, 600)][int]$RecoverySeconds = 300 + ) + $ErrorActionPreference = 'Stop' + $aksId = "/subscriptions/$env:subscriptionId/resourceGroups/$env:resourceGroup/providers/Microsoft.Kubernetes/connectedClusters/localbox-aks" + if (-not $env:subscriptionId -or -not $env:resourceGroup -or $Manifest.AksId -ine $aksId -or + $Manifest.SubscriptionId -ine $env:subscriptionId -or $Manifest.ResourceGroupName -ine $env:resourceGroup) { + throw 'MetalLB recovery manifest does not match this hosted LocalBox scope.' + } + $plan = Get-LocalBoxMetalLbPlan -Configuration $Configuration -ClusterId $aksId + Assert-LocalBoxProperties $Manifest.MetalLb $plan + $deadline = [datetime]::UtcNow.AddSeconds($GraceSeconds) + do { + if (Test-LocalBoxAutomationMetalLbRuntime -Kubeconfig $Kubeconfig -Plan $plan) { + Write-Host 'MetalLB runtime objects are verified; no recovery PUT is needed.' + return + } + if ([datetime]::UtcNow -ge $deadline) { break } + Write-Host 'MetalLB runtime pool or L2 advertisement is missing; waiting for initial reconciliation.' + Start-Sleep -Seconds ([math]::Min(15, [math]::Max(0, ($deadline - [datetime]::UtcNow).TotalSeconds))) + } while ($true) + if (Test-Path -LiteralPath $StatePath) { + throw 'MetalLB runtime objects remain missing and automatic recovery was already recorded for this run. No second PUT will be submitted; inspect the private metallb-recovery.json report.' + } + $extension = Get-LocalBoxResource $plan.ExtensionId -TimeoutSeconds 30 + Assert-LocalBoxProperties $extension @{ properties = @{ provisioningState = 'Succeeded'; extensionType = 'microsoft.arcnetworking' } } + $url = "https://management.azure.com$aksId/providers/Microsoft.KubernetesRuntime/loadBalancers?api-version=2024-08-01" + $pools = @(Get-LocalBoxArmCollection $url -TimeoutSeconds 30) + if ($pools.Count -ne 1 -or $pools[0].id -ine $plan.PoolId) { + throw 'The existing Azure-managed MetalLB pool is missing or additional pools exist; no automatic repair is allowed.' + } + $pool = $pools[0] + Assert-LocalBoxProperties $pool $plan.PoolExpected + if ($pool.properties.provisioningState -ne 'Succeeded') { + throw "The Azure-managed MetalLB pool is $($pool.properties.provisioningState), not Succeeded; no automatic repair is allowed." + } + foreach ($key in @('bgpPeers', 'communities', 'serviceSelector')) { + if ($pool.properties[$key].Count -gt 0) { + throw "The Azure-managed MetalLB pool has $key configured; organizer review is required." + } + } + if (Test-LocalBoxAutomationMetalLbRuntime -Kubeconfig $Kubeconfig -Plan $plan) { + Write-Host 'MetalLB runtime objects converged before recovery; no PUT was submitted.' + return + } + $body = @{ properties = @{ + addresses = $pool.properties.addresses; advertiseMode = $pool.properties.advertiseMode + bgpPeers = $pool.properties.bgpPeers; communities = $pool.properties.communities + serviceSelector = $pool.properties.serviceSelector + } } | ConvertTo-Json -Depth 10 -Compress + $record = @{ PoolId = $plan.PoolId; Attempt = 1; State = 'Attempted'; StartedAt = [datetime]::UtcNow.ToString('o') } + # Record before submitting: a timeout or guest restart must not cause a second PUT. + New-Item -Path $StatePath -ItemType File -Value ($record | ConvertTo-Json) -ErrorAction Stop | Out-Null + Write-Host 'MetalLB automatic recovery 1/1: reapplying the unchanged Azure-managed ARP pool; no resources will be deleted.' + Invoke-LocalBoxAz @('rest', '--method', 'put', '--url', + "https://management.azure.com$($plan.PoolId)?api-version=2024-08-01", '--body', $body) -TimeoutSeconds 120 -NoOutput + $deadline = [datetime]::UtcNow.AddSeconds($RecoverySeconds) + do { + $actual = Get-LocalBoxResource $plan.PoolId -TimeoutSeconds 30 + Assert-LocalBoxProperties $actual $plan.PoolExpected + $state = $actual.properties.provisioningState + if ([string]::IsNullOrWhiteSpace($state) -or $state -in @('Failed', 'Canceled', 'Deleting')) { + throw "MetalLB automatic recovery failed: Azure provisioning state is '$state'. No further automatic attempt is allowed." + } + $ready = Test-LocalBoxAutomationMetalLbRuntime -Kubeconfig $Kubeconfig -Plan $plan + if ($state -eq 'Succeeded' -and $ready) { + $record.State = 'Succeeded' + $record.CompletedAt = [datetime]::UtcNow.ToString('o') + $record | ConvertTo-Json | Set-Content -LiteralPath $StatePath -ErrorAction Stop + Write-Host 'MetalLB automatic recovery succeeded: Kubernetes pool and L2 advertisement verified. Full health tests must still pass.' + return + } + if ([datetime]::UtcNow -ge $deadline) { + throw "MetalLB automatic recovery exhausted after one PUT: Azure state=$state; Kubernetes pool/L2 advertisement ready=$ready. Inspect the Arc networking controller and private reports; readiness remains unverified." + } + Write-Host "Waiting for MetalLB recovery: Azure state=$state; Kubernetes pool/L2 advertisement ready=$ready." + Start-Sleep -Seconds ([math]::Min(15, [math]::Max(0, ($deadline - [datetime]::UtcNow).TotalSeconds))) + } while ($true) +} + function Invoke-LocalBoxAutomationRoutedHealth { - param([string]$ResourceRoot, [string]$InputPath, [string]$ManifestPath, [string]$Kubeconfig) + param([string]$ResourceRoot, [string]$InputPath, [string]$ManifestPath, [string]$Kubeconfig, [switch]$RecoverMetalLb) $ErrorActionPreference = 'Stop' $config = Import-PowerShellDataFile -LiteralPath $env:LocalBoxConfigFile foreach ($key in @('AKSControlPlaneIP', 'PhysicalHostInternalIP', 'SDNLABRoute')) { @@ -147,6 +271,11 @@ function Invoke-LocalBoxAutomationRoutedHealth { -NextHop $config.SDNLABRoute -PolicyStore ActiveStore -ErrorAction Stop } else { Write-Host "Reusing the existing AKS API route for $prefix without modifying it." } + if ($RecoverMetalLb) { + $manifest = Get-Content -LiteralPath $ManifestPath -Raw -ErrorAction Stop | ConvertFrom-Json -AsHashtable + Sync-LocalBoxAutomationMetalLbRuntime -Manifest $manifest -Configuration $config -Kubeconfig $Kubeconfig ` + -StatePath (Join-Path (Split-Path -Parent $InputPath) 'metallb-recovery.json') + } Invoke-LocalBoxAutomationHealthProcess -ResourceRoot $ResourceRoot -InputPath $InputPath -ManifestPath $ManifestPath } catch { $healthError = $_.Exception.Message; throw } @@ -256,7 +385,7 @@ function Invoke-LocalBoxGuestAutomation { '--resource-group', $Configuration.ResourceGroupName, '--name', 'localbox-aks', '--admin', '--file', "$Directory/aks.kubeconfig", '--overwrite-existing') -NoOutput Invoke-LocalBoxAutomationRoutedHealth -ResourceRoot $ResourceRoot -InputPath $InputPath -ManifestPath $ManifestPath ` - -Kubeconfig "$Directory/aks.kubeconfig" + -Kubeconfig "$Directory/aks.kubeconfig" -RecoverMetalLb $health = Get-Content -LiteralPath "$Directory/health.json" -Raw | ConvertFrom-Json -AsHashtable if (-not $health.FullReadiness) { throw 'Sovereign Cloud tests did not establish full infrastructure health.' } $status.Sovereign = @{ Passed = $health.Passed; Failed = $health.Failed; Skipped = $health.Skipped; NotRun = $health.NotRun } diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/prepare-localbox.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/prepare-localbox.ps1 index ca72bce6d..d28725f87 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/prepare-localbox.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/prepare-localbox.ps1 @@ -200,7 +200,7 @@ function Initialize-LocalBoxCliExtension { } function Get-LocalBoxResource { - param([string]$Id) + param([string]$Id, [int]$TimeoutSeconds = 300) $arguments = @('resource', 'show', '--ids', $Id) if ($Id -match '/providers/Microsoft\.HybridContainerService/provisionedClusterInstances/') { $arguments += @('--api-version', '2024-01-01') @@ -217,7 +217,7 @@ function Get-LocalBoxResource { elseif ($Id -match '/providers/Microsoft\.HybridCompute/machines/') { $arguments += @('--api-version', '2024-07-10') } - Invoke-LocalBoxAz $arguments + Invoke-LocalBoxAz $arguments -TimeoutSeconds $TimeoutSeconds } function New-LocalBoxVmTemplate { @@ -660,9 +660,9 @@ function Get-LocalBoxMetalLbPlan { } function Get-LocalBoxArmCollection { - param([Parameter(Mandatory)][string]$Url) + param([Parameter(Mandatory)][string]$Url, [int]$TimeoutSeconds = 300) do { - $page = Invoke-LocalBoxAz @('rest', '--method', 'get', '--url', $Url) + $page = Invoke-LocalBoxAz @('rest', '--method', 'get', '--url', $Url) -TimeoutSeconds $TimeoutSeconds # Background-job deserialization turns nested JSON arrays into ArrayList. if ($page -isnot [System.Collections.IDictionary] -or -not $page.Contains('value') -or $page.value -isnot [System.Collections.IList]) { diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-automation.tests.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-automation.tests.ps1 index 920513542..adc9884f4 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-automation.tests.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-automation.tests.ps1 @@ -569,11 +569,33 @@ Describe 'Temporary AKS API routing for hosted health checks' { $routeState.Active.Count | Should -Be 1 $routeState.HealthCalls++ } + Mock Sync-LocalBoxAutomationMetalLbRuntime { $routeState.Active.Count | Should -Be 1 } } AfterEach { $env:LocalBoxConfigFile = $previousLocalConfig } + It 'runs optional recovery within the owned route before invoking health tests' { + $manifestPath = "$TestDrive/manifest.json" + '{}' | Set-Content $manifestPath + Invoke-LocalBoxAutomationRoutedHealth 'source' "$TestDrive/input.json" $manifestPath 'private.kubeconfig' -RecoverMetalLb + Should -Invoke Sync-LocalBoxAutomationMetalLbRuntime -Times 1 -Exactly -ParameterFilter { + $StatePath -eq "$TestDrive/metallb-recovery.json" -and $Kubeconfig -eq 'private.kubeconfig' + } + Should -Invoke Invoke-LocalBoxAutomationHealthProcess -Times 1 -Exactly + $routeState.Active.Count | Should -Be 0 + } + It 'removes the owned route and skips health when recovery fails' { + $manifestPath = "$TestDrive/manifest.json" + '{}' | Set-Content $manifestPath + Mock Sync-LocalBoxAutomationMetalLbRuntime { throw 'MetalLB recovery exhausted' } + { Invoke-LocalBoxAutomationRoutedHealth 'source' "$TestDrive/input.json" $manifestPath 'private.kubeconfig' -RecoverMetalLb } | + Should -Throw '*MetalLB recovery exhausted*' + Should -Invoke Remove-NetRoute -Times 1 -Exactly + Should -Invoke Invoke-LocalBoxAutomationHealthProcess -Times 0 + $routeState.Active.Count | Should -Be 0 + } It 'adds only a nonpersistent API host route while health checks run and verifies cleanup' { Invoke-LocalBoxAutomationRoutedHealth 'source' 'input.json' 'manifest.json' 'private.kubeconfig' Should -Invoke kubectl -Times 1 + Should -Invoke Sync-LocalBoxAutomationMetalLbRuntime -Times 0 Should -Invoke New-NetRoute -Times 1 -Exactly -ParameterFilter { $DestinationPrefix -eq '10.10.0.5/32' -and $InterfaceIndex -eq 42 -and $NextHop -eq '192.168.1.1' -and $PolicyStore -eq 'ActiveStore' @@ -749,7 +771,7 @@ Invoke-TestPreparation -SubscriptionId $SubscriptionId -ResourceGroupName $Resou Should -Invoke Invoke-LocalBoxAz -Times 1 -ParameterFilter { $Arguments[0..1] -join ' ' -eq 'connectedmachine assess-patches' } Should -Invoke Invoke-LocalBoxAutomationHealthProcess -Times 1 Should -Invoke Invoke-LocalBoxAutomationRoutedHealth -Times 1 -Exactly -ParameterFilter { - $Kubeconfig -eq "$guestDirectory/aks.kubeconfig" -and $ManifestPath -eq $manifestFile + $Kubeconfig -eq "$guestDirectory/aks.kubeconfig" -and $ManifestPath -eq $manifestFile -and $RecoverMetalLb } Should -Invoke Initialize-LocalBoxAutomationKubectl -Times 1 -Exactly -ParameterFilter { $InstanceId -eq '/aks/instance' -and $Directory -eq $guestDirectory diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-metallb-recovery.tests.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-metallb-recovery.tests.ps1 new file mode 100644 index 000000000..00239f899 --- /dev/null +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-metallb-recovery.tests.ps1 @@ -0,0 +1,223 @@ +BeforeAll { + . "$PSScriptRoot/../prepare-localbox.ps1" + . "$PSScriptRoot/../test-sovereign-cloud.ps1" + . "$PSScriptRoot/../invoke-localbox-automation.ps1" +} + +Describe 'Bounded hosted MetalLB runtime recovery' { + BeforeEach { + $oldSubscription = $env:subscriptionId + $oldGroup = $env:resourceGroup + $env:subscriptionId = '10000000-0000-0000-0000-000000000001' + $env:resourceGroup = 'rg-localbox-test' + $aksId = "/subscriptions/$env:subscriptionId/resourceGroups/$env:resourceGroup/providers/Microsoft.Kubernetes/connectedClusters/localbox-aks" + $configuration = @{ + AKSIPPrefix = '10.10.0.0/24'; AKSVIPStartIP = '10.10.0.10'; AKSVIPEndIP = '10.10.0.100' + AKSGWIP = '10.10.0.1'; AKSControlPlaneIP = '10.10.0.5' + AKSNodeStartIP = '10.10.0.101'; AKSNodeEndIP = '10.10.0.199' + } + $plan = Get-LocalBoxMetalLbPlan $configuration $aksId + $manifest = @{ + SubscriptionId = $env:subscriptionId; ResourceGroupName = $env:resourceGroup + AksId = $aksId; MetalLb = $plan + } + $pool = @{ id = $plan.PoolId; properties = @{ + addresses = @('10.10.0.10-10.10.0.100'); advertiseMode = 'ARP'; provisioningState = 'Succeeded' + bgpPeers = $null; communities = $null; serviceSelector = $null + } } + $runtimePool = @{ metadata = @{ name = 'aks-pool'; namespace = 'kube-system' }; spec = @{ + addresses = @('10.10.0.10-10.10.0.100'); autoAssign = $true + } } + $advertisement = @{ metadata = @{ name = 'aks-pool'; namespace = 'kube-system' }; spec = @{ ipAddressPools = @('aks-pool') } } + $probe = @{ Ready = $false; PoolVisible = $false; AdvertisementVisible = $false } + $statePath = Join-Path $TestDrive "$([guid]::NewGuid()).json" + $arguments = @{ + Manifest = $manifest; Configuration = $configuration; Kubeconfig = 'private.kubeconfig' + StatePath = $statePath; GraceSeconds = 0; RecoverySeconds = 0 + } + Mock Invoke-SovereignKubectl { + if ($Arguments[1] -eq 'ipaddresspools.metallb.io') { + return @{ items = @(if ($probe.Ready -or $probe.PoolVisible) { $runtimePool }) } + } + return @{ items = @(if ($probe.Ready -or $probe.AdvertisementVisible) { $advertisement }) } + } + Mock Get-LocalBoxResource { + if ($Id -eq $plan.ExtensionId) { return @{ properties = @{ provisioningState = 'Succeeded'; extensionType = 'microsoft.arcnetworking' } } } + return $pool + } + Mock Get-LocalBoxArmCollection { $pool } + Mock Invoke-LocalBoxAz { $probe.Ready = $true } + Mock Start-Sleep {} + } + AfterEach { + $env:subscriptionId = $oldSubscription + $env:resourceGroup = $oldGroup + } + It 'does nothing to Azure when both runtime objects already exist' { + $probe.Ready = $true + Sync-LocalBoxAutomationMetalLbRuntime @arguments + Should -Invoke Invoke-LocalBoxAz -Times 0 + Should -Invoke Get-LocalBoxArmCollection -Times 0 + Test-Path $statePath | Should -BeFalse + } + It 'gives initial reconciliation a grace period without submitting a PUT' { + $arguments.GraceSeconds = 60 + Mock Start-Sleep { $probe.Ready = $true } + Sync-LocalBoxAutomationMetalLbRuntime @arguments + Should -Invoke Start-Sleep -Times 1 -Exactly -ParameterFilter { $Seconds -gt 0 -and $Seconds -le 15 } + Should -Invoke Invoke-LocalBoxAz -Times 0 + } + It 'reapplies exactly the unchanged ARP configuration once and verifies convergence' { + Sync-LocalBoxAutomationMetalLbRuntime @arguments + Should -Invoke Invoke-LocalBoxAz -Times 1 -Exactly -ParameterFilter { + $body = $Arguments[$Arguments.IndexOf('--body') + 1] | ConvertFrom-Json -AsHashtable + $Arguments[0..2] -join ' ' -eq 'rest --method put' -and + $Arguments[$Arguments.IndexOf('--url') + 1] -eq "https://management.azure.com$($plan.PoolId)?api-version=2024-08-01" -and + $body.properties.Count -eq 5 -and $body.properties.addresses[0] -eq '10.10.0.10-10.10.0.100' -and + $body.properties.advertiseMode -eq 'ARP' -and $null -eq $body.properties.serviceSelector -and + $null -eq $body.properties.bgpPeers -and $null -eq $body.properties.communities -and $TimeoutSeconds -eq 120 -and $NoOutput + } + (Get-Content $statePath -Raw | ConvertFrom-Json).State | Should -Be 'Succeeded' + Should -Invoke Get-LocalBoxResource -Times 2 -Exactly -ParameterFilter { $TimeoutSeconds -eq 30 } + } + It 'also recovers when only the advertisement is absent' { + $probe.PoolVisible = $true + Sync-LocalBoxAutomationMetalLbRuntime @arguments + Should -Invoke Invoke-LocalBoxAz -Times 1 -Exactly + } + It 'also recovers when only the pool is absent' { + $probe.AdvertisementVisible = $true + Sync-LocalBoxAutomationMetalLbRuntime @arguments + Should -Invoke Invoke-LocalBoxAz -Times 1 -Exactly + } + It 'waits for Azure success as well as runtime readiness after the PUT' { + $arguments.RecoverySeconds = 60 + Mock Invoke-LocalBoxAz { $probe.Ready = $true; $pool.properties.provisioningState = 'Updating' } + Mock Start-Sleep { $pool.properties.provisioningState = 'Succeeded' } + Sync-LocalBoxAutomationMetalLbRuntime @arguments + Should -Invoke Start-Sleep -Times 1 -Exactly + Should -Invoke Invoke-LocalBoxAz -Times 1 -Exactly + } + It 'stops at the recovery deadline even when Azure still claims success' { + Mock Invoke-LocalBoxAz {} + { Sync-LocalBoxAutomationMetalLbRuntime @arguments } | Should -Throw '*exhausted after one PUT*ready=False*' + Should -Invoke Invoke-LocalBoxAz -Times 1 -Exactly + Should -Invoke Start-Sleep -Times 0 + (Get-Content $statePath -Raw | ConvertFrom-Json).State | Should -Be 'Attempted' + } + It 'never resubmits after an ambiguous PUT failure or guest restart' { + Mock Invoke-LocalBoxAz { throw 'Azure CLI timed out; submission may still be running.' } + { Sync-LocalBoxAutomationMetalLbRuntime @arguments } | Should -Throw '*timed out*' + { Sync-LocalBoxAutomationMetalLbRuntime @arguments } | Should -Throw '*already recorded*' + Should -Invoke Invoke-LocalBoxAz -Times 1 -Exactly + } + It 'permits a healthy resumed run without repeating the recorded recovery' { + Set-Content $statePath '{"State":"Attempted"}' + $probe.Ready = $true + Sync-LocalBoxAutomationMetalLbRuntime @arguments + Should -Invoke Invoke-LocalBoxAz -Times 0 + } + It 'does not submit if the attempt marker cannot be written' { + $arguments.StatePath = Join-Path $TestDrive 'absent-parent/attempt.json' + { Sync-LocalBoxAutomationMetalLbRuntime @arguments } | Should -Throw + Should -Invoke Invoke-LocalBoxAz -Times 0 + } + It 'rejects wrong scope before accessing Azure or Kubernetes' -TestCases @( + @{ Field = 'SubscriptionId' }, @{ Field = 'ResourceGroupName' }, @{ Field = 'AksId' } + ) { + param($Field) + $manifest[$Field] = 'other' + { Sync-LocalBoxAutomationMetalLbRuntime @arguments } | Should -Throw '*scope*' + Should -Invoke Invoke-LocalBoxAz -Times 0 + Should -Invoke Invoke-SovereignKubectl -Times 0 + } + It 'rejects a manifest pool that differs from the installed network reservation' { + $manifest.MetalLb.PoolExpected.properties.addresses = @('10.10.0.200-10.10.0.210') + { Sync-LocalBoxAutomationMetalLbRuntime @arguments } | Should -Throw '*Conflicting*' + Should -Invoke Invoke-LocalBoxAz -Times 0 + } + It 'does not interpret Kubernetes transport, permission or missing-CRD errors as missing objects' -TestCases @( + @{ ErrorText = 'connection refused' }, @{ ErrorText = 'Forbidden' }, @{ ErrorText = 'server does not have resource type ipaddresspools' } + ) { + param($ErrorText) + Mock Invoke-SovereignKubectl { throw $ErrorText } + { Sync-LocalBoxAutomationMetalLbRuntime @arguments } | Should -Throw "*$ErrorText*" + Should -Invoke Invoke-LocalBoxAz -Times 0 + } + It 'rejects malformed Kubernetes list responses' { + Mock Invoke-SovereignKubectl { @{ items = $null } } + { Sync-LocalBoxAutomationMetalLbRuntime @arguments } | Should -Throw '*Invalid MetalLB*' + Should -Invoke Invoke-LocalBoxAz -Times 0 + } + It 'refuses runtime conflicts: ' -TestCases @( + @{ Conflict = 'range' }, @{ Conflict = 'autoAssign' }, @{ Conflict = 'namespace' }, @{ Conflict = 'deleting' } + ) { + param($Conflict) + $probe.PoolVisible = $true + switch ($Conflict) { + 'range' { $runtimePool.spec.addresses = @('10.10.0.200-10.10.0.210') } + 'autoAssign' { $runtimePool.spec.autoAssign = $false } + 'namespace' { $runtimePool.metadata.namespace = 'other' } + 'deleting' { $runtimePool.metadata.deletionTimestamp = '2026-11-05T00:00:00Z' } + } + { Sync-LocalBoxAutomationMetalLbRuntime @arguments } | Should -Throw + Should -Invoke Invoke-LocalBoxAz -Times 0 + } + It 'rejects extra Azure pools rather than risking overlapping ranges' { + Mock Get-LocalBoxArmCollection { $pool; @{ id = 'another-pool' } } + { Sync-LocalBoxAutomationMetalLbRuntime @arguments } | Should -Throw '*additional pools*' + Should -Invoke Invoke-LocalBoxAz -Times 0 + } + It 'rejects a missing Azure pool instead of creating a replacement' { + Mock Get-LocalBoxArmCollection { @() } + { Sync-LocalBoxAutomationMetalLbRuntime @arguments } | Should -Throw '*existing Azure-managed MetalLB pool is missing*' + Should -Invoke Invoke-LocalBoxAz -Times 0 + } + It 'rejects an unsuccessful extension before reapplying the pool' { + Mock Get-LocalBoxResource { @{ properties = @{ provisioningState = 'Failed'; extensionType = 'microsoft.arcnetworking' } } } + { Sync-LocalBoxAutomationMetalLbRuntime @arguments } | Should -Throw '*Conflicting*' + Should -Invoke Invoke-LocalBoxAz -Times 0 + } + It 'does not overwrite a customized L2 advertisement: ' -TestCases @( + @{ Field = 'interfaces'; Value = @('eth0') }, + @{ Field = 'nodeSelectors'; Value = @(@{ matchLabels = @{ role = 'custom' } }) }, + @{ Field = 'ipAddressPoolSelectors'; Value = @(@{ matchLabels = @{ app = 'custom' } }) }, + @{ Field = 'ipAddressPools'; Value = @('other-pool') } + ) { + param($Field, $Value) + $probe.AdvertisementVisible = $true + $advertisement.spec[$Field] = $Value + { Sync-LocalBoxAutomationMetalLbRuntime @arguments } | Should -Throw + Should -Invoke Invoke-LocalBoxAz -Times 0 + } + It 'refuses to race a deleting advertisement' { + $probe.AdvertisementVisible = $true + $advertisement.metadata.deletionTimestamp = '2026-11-05T00:00:00Z' + { Sync-LocalBoxAutomationMetalLbRuntime @arguments } | Should -Throw '*being deleted*' + Should -Invoke Invoke-LocalBoxAz -Times 0 + } + It 'rejects changed Azure configuration' -TestCases @( + @{ Field = 'addresses'; Value = @('10.10.0.200-10.10.0.210') }, + @{ Field = 'advertiseMode'; Value = 'BGP' }, + @{ Field = 'provisioningState'; Value = 'Updating' }, + @{ Field = 'serviceSelector'; Value = @{ app = 'other' } }, + @{ Field = 'bgpPeers'; Value = @('peer') }, + @{ Field = 'communities'; Value = @('community') } + ) { + param($Field, $Value) + $pool.properties[$Field] = $Value + { Sync-LocalBoxAutomationMetalLbRuntime @arguments } | Should -Throw + Should -Invoke Invoke-LocalBoxAz -Times 0 + } + It 'propagates Azure authorization errors without writing an attempt marker' { + Mock Get-LocalBoxArmCollection { throw 'AuthorizationFailed' } + { Sync-LocalBoxAutomationMetalLbRuntime @arguments } | Should -Throw '*AuthorizationFailed*' + Test-Path $statePath | Should -BeFalse + Should -Invoke Invoke-LocalBoxAz -Times 0 + } + It 'stops when the recovery PUT reaches a failed Azure state' { + Mock Invoke-LocalBoxAz { $pool.properties.provisioningState = 'Failed' } + { Sync-LocalBoxAutomationMetalLbRuntime @arguments } | Should -Throw '*recovery failed*Failed*' + Should -Invoke Invoke-LocalBoxAz -Times 1 -Exactly + } +} diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md index 258bd0fa9..49880cda0 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md @@ -18,6 +18,19 @@ A fresh Codespace smoke test is still required to validate the devcontainer buil ## MetalLB preparation checks +Run `Invoke-Pester ./localbox-metallb-recovery.tests.ps1,./localbox-automation.tests.ps1 -Output Detailed` +for hosted runtime recovery. Coverage includes healthy no-op and delayed initial +convergence, pool-only/advertisement-only readiness, one unchanged Azure PUT, +post-PUT convergence/exhaustion, persistent attempt recording before ambiguous +submission failure, scope/configuration conflicts, API errors, and route cleanup. +Recovery runs before the normal Full suite; the tests themselves remain read-only. +Offline fixtures simulate missing objects without deleting healthy live resources. +Live verification on traininglab-02 confirmed that the recovery preflight performs +no PUT when the objects exist, followed by **14/14 Full health tests passing** and +verified temporary-route cleanup. The missing-object/retry branches are tested +with fixtures; no healthy live objects were removed to manufacture a failure. +A fresh deployment exercising automatic recovery remains a separate live gate. + MetalLB preparation tests are included in `prepare-localbox.tests.ps1`. Run `Invoke-Pester ./prepare-localbox.tests.ps1 -FullName 'LocalBox MetalLB preparation*','MetalLB health evidence*','CLI output streams*' -Output Detailed` for the focused offline checks. They cover reserved VIP validation, idempotent From 29048658b5b4527ae22534e838a735155d950466 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Thu, 5 Nov 2026 09:41:03 +0000 Subject: [PATCH 14/37] Pin hosted LocalBox guest package with bounded MetalLB recovery Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../labautomation/localbox-automation-source.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation-source.json b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation-source.json index babe74e21..5ca470992 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation-source.json +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation-source.json @@ -1,9 +1,9 @@ { "repository": "janegilring/MicroHack", - "commit": "948c71fb412b29b64070457ddf5489bcc16cfb06", + "commit": "a4188387649a52623f510f4d4873253c79d679f6", "files": { - "prepare-localbox.ps1": "516c4f832b0b93b987ed595ea082aac3b27958c8ca12d8a1e73c0f353d36c75e", - "invoke-localbox-automation.ps1": "90efd7ba0d1973fa411ba9c037af23aeaa223bf9992fa65ff109810f73020edd", + "prepare-localbox.ps1": "57725d1b9d71bb93f93ac0768cc7f72e76da733633cebe904a03a20a5a800232", + "invoke-localbox-automation.ps1": "cbcb8dbee2523497a3d6b8c0586154c01e579bce3183de9aeb10d4e3ccb7c425", "test-sovereign-cloud.ps1": "e23583ac0a3747261d57469a7ae3e31438c0cb70beb9bf1c751a6244e4ea5117", "tests/localbox.health.tests.ps1": "92f922dff75b1bcecc7c3e9a523c613930bd1a538ef8bb0fd50dd2ee1267b4d9", "tests/sovereign-lab.health.tests.ps1": "2916dd41ea0e53b4599bfb72a8f21ac6d152a476aff72593bd33e075fdde7ada" From 2c7fad3e6e56818869bb33c07f08aa0cafd64071 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Thu, 5 Nov 2026 09:56:03 +0000 Subject: [PATCH 15/37] Overlap fresh AKS provisioning with LocalBox VM image import Submit AKS asynchronously after validation, import the image while Azure provisions AKS, and join strict AKS/Arc/MetalLB readiness before reference VM creation. Preserve existing-cluster ordering and add a sequential fallback. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../resources/hosted-events/readme.md | 18 +++++ .../hosted-events/test-06-timing-analysis.md | 7 ++ .../resources/prepare-localbox.ps1 | 75 ++++++++++++++----- .../tests/prepare-localbox.tests.ps1 | 72 +++++++++++++++--- .../resources/tests/readme.md | 14 +++- 5 files changed, 156 insertions(+), 30 deletions(-) diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md index dcfe2b5c8..3f0e9286b 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md @@ -165,6 +165,24 @@ The [orchestrator](../../labautomation/localbox-automation.ps1) implements: also installs its bundled `kubelogin` dependency there. Only the runner and its child health process receive the private tools PATH; existing installations and the machine PATH are not changed. Retries validate and reuse this client. + For a **new AKS cluster**, preparation validates AKS synchronously, then uses + [`az aksarc create --no-wait`](https://learn.microsoft.com/cli/azure/aksarc#az-aksarc-create) + to let Azure provision it while the Windows VM image imports. Storage and + logical networks are prepared first. CLI commands remain serial with the + existing isolated authentication profile; no competing CLI workers or shared + token-cache writers are introduced. After image completion, preparation + requires AKS provisioning success, the expected configuration, Arc connectivity, + proxy access and MetalLB before creating the reference VM. The final Full + readiness gate is unchanged. + + Existing AKS deployments retain verification-before-image ordering on retries. + Manual preparation supports `-SequentialProvisioning` to restore serial + provisioning if contention is suspected. `-SkipAks` and `-WhatIf` retain their + existing behavior. If preparation fails after asynchronous submission, AKS may + continue provisioning in Azure; the error is propagated and nothing is rolled + back. Inspect both resource states before retrying rather than deleting the + environment. The parent guest timeout still applies. Compare equivalent cold + deployments before treating the potential ~71-minute saving as measured. 5. Wait for the reference VM to be Running, Arc Connected and MDE installation Succeeded. Request an Update Manager assessment and require a successful, timestamped result. This assesses updates; it does **not** install patches. diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/test-06-timing-analysis.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/test-06-timing-analysis.md index a5303dd82..38957fa7b 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/test-06-timing-analysis.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/test-06-timing-analysis.md @@ -171,6 +171,13 @@ These are recommendations, **not implemented runtime changes or promised savings**. Preserve the final readiness gate and compare equivalent cold runs before declaring an improvement. +**2026-11-05 implementation update:** Fresh AKS creation now uses the CLI's +`--no-wait` option before importing the VM image, and joins AKS/Arc/MetalLB +readiness afterwards. This implements the overlap in candidate 1 without +concurrent local CLI sessions. Existing-AKS retries remain serial, and manual +preparation offers `-SequentialProvisioning` as a fallback. The estimates below +remain historical estimates, not measured savings from this implementation. + | Priority | Candidate | Potential benefit from this run | Constraints / verification | |---|---|---|---| | 1 | Start image import alongside AKS after shared storage/network prerequisites are ready | With unchanged operation durations, overlapping the 71-minute AKS deployment with the 179-minute image import could remove roughly **71 minutes** of serial waiting | Both use the same nested environment. Storage, CPU or network contention could reduce or eliminate the saving. Use controlled concurrency, isolated CLI state, explicit error handling, and wait for both branches before full readiness | diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/prepare-localbox.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/prepare-localbox.ps1 index d28725f87..8cae15df9 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/prepare-localbox.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/prepare-localbox.ps1 @@ -23,6 +23,9 @@ ranges and known reservations are validated; live DHCP leases are not inspected. New image imports resolve latest from this cluster's Edge Marketplace catalog, not the Azure Compute catalog; existing working images are reused unchanged. Dot-source this file to load its functions without running provisioning. +Fresh AKS creation runs asynchronously while the VM image imports. Both must +succeed before reference VM creation. Use SequentialProvisioning for serial +execution; existing AKS deployments retain verification-before-image ordering. #> [CmdletBinding(SupportsShouldProcess)] param( @@ -32,6 +35,7 @@ param( [string]$AksAdminGroupObjectId, [string]$KubernetesRuntimeObjectId, [switch]$SkipAks, + [switch]$SequentialProvisioning, [switch]$IncludeReferenceVm = $true, [pscredential]$ReferenceVmCredential, [pscredential]$NodeCredential, @@ -969,6 +973,30 @@ function Initialize-LocalBoxStorage { } } +function Complete-LocalBoxAksPreparation { + [CmdletBinding(SupportsShouldProcess)] + param( + [string]$ClusterId, [hashtable]$Expected, [string]$GroupId, + [hashtable]$MetalLbPlan, [string]$RuntimeObjectId, [int]$TimeoutSeconds = 21600 + ) + if (-not $WhatIfPreference) { + $instanceId = "$ClusterId/providers/Microsoft.HybridContainerService/provisionedClusterInstances/default" + $instance = Wait-LocalBoxResource $instanceId -TimeoutSeconds $TimeoutSeconds + Assert-LocalBoxProperties $instance $Expected + $connected = Wait-LocalBoxResource $ClusterId -TimeoutSeconds $TimeoutSeconds + Assert-LocalBoxProperties $connected @{ properties = @{ aadProfile = @{ adminGroupObjectIDs = @($GroupId) } } } + $deadline = [datetime]::UtcNow.AddSeconds($TimeoutSeconds) + while ($connected.properties.connectivityStatus -ne 'Connected') { + if ([datetime]::UtcNow -ge $deadline) { throw 'AKS exists but Azure Arc is not connected.' } + Start-Sleep -Seconds 15 + $connected = Get-LocalBoxResource $ClusterId + } + } + Sync-LocalBoxAksProxyRole -ClusterId $ClusterId -GroupObjectId $GroupId -WhatIf:$WhatIfPreference + Sync-LocalBoxMetalLb -ClusterId $ClusterId -Plan $MetalLbPlan -RuntimeObjectId $RuntimeObjectId ` + -TimeoutSeconds $TimeoutSeconds -WhatIf:$WhatIfPreference +} + function Invoke-LocalBoxPreparation { [CmdletBinding(SupportsShouldProcess)] param([hashtable]$Settings) @@ -997,6 +1025,7 @@ function Invoke-LocalBoxPreparation { $profile = Join-Path ([IO.Path]::GetTempPath()) "microhack-az-$([guid]::NewGuid())" $lock = [Threading.Mutex]::new($false, 'Global\MicroHackLocalBoxPreparation') $locked = $false + $aksVerificationPending = $false try { $locked = $lock.WaitOne(0) if (-not $locked) { throw 'Another LocalBox preparation is running.' } @@ -1100,25 +1129,23 @@ function Invoke-LocalBoxPreparation { Write-Host "Validating AKS on Azure Local $($Settings.AksClusterName)..." Invoke-LocalBoxAz ($arguments + @('--validate')) -TimeoutSeconds $timeout -NoOutput Write-Host "Creating AKS on Azure Local $($Settings.AksClusterName)..." - Invoke-LocalBoxAz $arguments -TimeoutSeconds $timeout -NoOutput + if (-not $Settings.SequentialProvisioning) { + # Azure provisions asynchronously; local CLI commands still run serially. + $aksVerificationPending = $true + Invoke-LocalBoxAz ($arguments + @('--no-wait')) -TimeoutSeconds $timeout -NoOutput + Write-Host "AKS creation submitted at $([datetime]::UtcNow.ToString('o')); provisioning will overlap VM image import. AKS readiness is not yet verified." + } + else { + Invoke-LocalBoxAz $arguments -TimeoutSeconds $timeout -NoOutput + } } } - if (-not $Settings.SkipAks -and -not $WhatIfPreference) { - $instance = Wait-LocalBoxResource $instanceId -TimeoutSeconds $timeout - Assert-LocalBoxProperties $instance $aksExpected - $connected = Wait-LocalBoxResource $aksId -TimeoutSeconds $timeout - Assert-LocalBoxProperties $connected @{ properties = @{ aadProfile = @{ adminGroupObjectIDs = @($groupId) } } } - $deadline = [datetime]::UtcNow.AddSeconds($timeout) - while ($connected.properties.connectivityStatus -ne 'Connected') { - if ([datetime]::UtcNow -ge $deadline) { throw 'AKS exists but Azure Arc is not connected.' } - Start-Sleep -Seconds 15 - $connected = Get-LocalBoxResource $aksId - } + $aksCompletion = @{ + ClusterId = $aksId; Expected = $aksExpected; GroupId = $groupId + MetalLbPlan = $metalLb; RuntimeObjectId = $Settings.KubernetesRuntimeObjectId; TimeoutSeconds = $timeout } - if (-not $Settings.SkipAks) { - Sync-LocalBoxAksProxyRole -ClusterId $aksId -GroupObjectId $groupId -WhatIf:$WhatIfPreference - Sync-LocalBoxMetalLb -ClusterId $aksId -Plan $metalLb -RuntimeObjectId $Settings.KubernetesRuntimeObjectId ` - -TimeoutSeconds $timeout -WhatIf:$WhatIfPreference + if (-not $Settings.SkipAks -and -not $aksVerificationPending) { + Complete-LocalBoxAksPreparation @aksCompletion -WhatIf:$WhatIfPreference } $provider = Invoke-LocalBoxAz @('provider', 'show', '--namespace', 'Microsoft.EdgeMarketplace') if ($provider.registrationState -ne 'Registered') { throw 'Ask the subscription administrator to register Microsoft.EdgeMarketplace before importing the image.' } @@ -1129,10 +1156,15 @@ function Invoke-LocalBoxPreparation { } } } if ($Settings.ImageVersion -ne 'latest') { $imageExpected.properties.version = @{ name = $Settings.ImageVersion } } - Write-Host "Preparing VM image $($Settings.ImageName); download can take several hours..." + Write-Host "Preparing VM image $($Settings.ImageName) at $([datetime]::UtcNow.ToString('o')); download can take several hours..." $image = Sync-LocalBoxImage -Id $imageId -Expected $imageExpected -ClusterId $cluster[0].id -TimeoutSeconds $timeout -CreateArguments (@('stack-hci-vm', 'image', 'create') + $common + @( '--name', $Settings.ImageName, '--os-type', 'Windows', '--publisher', 'microsoftwindowsserver', '--offer', 'windowsserver', '--sku', '2025-datacenter-azure-edition-smalldisk', '--storage-path-id', $storageId)) + if ($aksVerificationPending) { + Write-Host "VM image import completed at $([datetime]::UtcNow.ToString('o')); joining AKS provisioning and verifying Arc connectivity and MetalLB." + Complete-LocalBoxAksPreparation @aksCompletion -WhatIf:$WhatIfPreference + $aksVerificationPending = $false + } $referenceVm = Sync-LocalBoxReferenceVm -Settings $Settings -Location $location -CustomLocationId $custom.id ` -ImageId $imageId -NetworkId $networks[0].Id -StorageId $storageId -WhatIf:$WhatIfPreference $manifest = @{ @@ -1159,6 +1191,12 @@ function Invoke-LocalBoxPreparation { if ($Settings.SkipAks) { Write-Warning 'AKS was explicitly skipped. Rerun with an admin-group ID before full health validation; this environment is not event-ready.' } Write-Host 'Preparation finished. Run test-sovereign-cloud.ps1 before declaring the environment ready.' } + catch { + if ($aksVerificationPending) { + Write-Warning 'Asynchronous AKS creation was attempted and readiness has not been verified. The Azure operation may still be running even though preparation failed. No resources were rolled back; inspect AKS and image provisioning state before retrying.' + } + throw + } finally { $env:AZURE_CONFIG_DIR = $previousConfig $env:AZURE_EXTENSION_DIR = $previousExtensions @@ -1173,7 +1211,8 @@ if ($MyInvocation.InvocationName -ne '.') { $settings = @{ SubscriptionId = $SubscriptionId; ResourceGroupName = $ResourceGroupName; ConfigPath = $ConfigPath AksAdminGroupObjectId = $AksAdminGroupObjectId; KubernetesRuntimeObjectId = $KubernetesRuntimeObjectId - SkipAks = [bool]$SkipAks; NodeCredential = $NodeCredential; VmSwitchName = $VmSwitchName + SkipAks = [bool]$SkipAks; SequentialProvisioning = [bool]$SequentialProvisioning + NodeCredential = $NodeCredential; VmSwitchName = $VmSwitchName IncludeReferenceVm = [bool]$IncludeReferenceVm; ReferenceVmCredential = $ReferenceVmCredential ImageName = $ImageName; ImageVersion = $ImageVersion; VmPoolStart = $VmPoolStart; VmPoolEnd = $VmPoolEnd AksClusterName = $AksClusterName; KubernetesVersion = $KubernetesVersion; NodeVmSize = $NodeVmSize diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/prepare-localbox.tests.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/prepare-localbox.tests.ps1 index 45b77b858..aa273e2dc 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/prepare-localbox.tests.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/prepare-localbox.tests.ps1 @@ -1537,14 +1537,18 @@ Describe 'AKS Local Arc proxy RBAC' { $ast = [Management.Automation.Language.Parser]::ParseFile("$PSScriptRoot/../prepare-localbox.ps1", [ref]$null, [ref]$null) $guard = $ast.Find({ param($node) $node -is [Management.Automation.Language.IfStatementAst] -and - $node.Clauses[0].Item1.Extent.Text -eq '-not $Settings.SkipAks' -and - $node.Extent.Text -match 'Sync-LocalBoxAksProxyRole' + $node.Clauses[0].Item1.Extent.Text -eq '-not $Settings.SkipAks -and -not $aksVerificationPending' -and + $node.Extent.Text -match 'Complete-LocalBoxAksPreparation' }, $true) $guard | Should -Not -BeNullOrEmpty $reconcile = [scriptblock]::Create($guard.Extent.Text) $aksId = $clusterId $Settings = @{ SkipAks = $true; KubernetesRuntimeObjectId = $groupId } $metalLb = @{} + $aksVerificationPending = $false + $aksCompletion = @{ ClusterId = $clusterId; Expected = @{}; GroupId = $groupId; MetalLbPlan = @{}; RuntimeObjectId = $groupId } + Mock Wait-LocalBoxResource { @{ properties = @{ connectivityStatus = 'Connected' } } } + Mock Assert-LocalBoxProperties {} Mock Sync-LocalBoxAksProxyRole {} Mock Sync-LocalBoxMetalLb {} & $reconcile @@ -2048,6 +2052,7 @@ Describe 'LocalBox preparation dependency order' { $imageId = 'image' $groupId = 'group' $metalLb = @{} + $aksVerificationPending = $false Mock Sync-LocalBoxResource { $events.Add("network:$($Id.Split('/')[-1])") } Mock Invoke-LocalBoxAz { if ($Arguments[0] -eq 'aksarc') { @@ -2062,18 +2067,65 @@ Describe 'LocalBox preparation dependency order' { Mock Sync-LocalBoxImage { $events.Add('image') } Mock Sync-LocalBoxReferenceVm { $events.Add('reference') } } - It 'completes networks, AKS and MetalLB before importing the image and reference VM' { + It 'submits AKS asynchronously after networks and imports the image before joining AKS and MetalLB' { & $prepareWorkloads -Confirm:$false - $events -join ',' | Should -Be 'network:localbox-vm-lnet-vlan200,network:localbox-aks-lnet-vlan110,aks-validate,aks-create,proxy,metallb,image,reference' + $events -join ',' | Should -Be 'network:localbox-vm-lnet-vlan200,network:localbox-aks-lnet-vlan110,aks-validate,aks-create,image,proxy,metallb,reference' + Should -Invoke Invoke-LocalBoxAz -Times 1 -Exactly -ParameterFilter { + $Arguments[0] -eq 'aksarc' -and $Arguments -contains '--no-wait' -and $Arguments -notcontains '--validate' + } + Should -Invoke Invoke-LocalBoxAz -Times 1 -Exactly -ParameterFilter { + $Arguments[0] -eq 'aksarc' -and $Arguments -contains '--validate' -and $Arguments -notcontains '--no-wait' + } Should -Invoke Sync-LocalBoxImage -Times 1 -Exactly -ParameterFilter { $ClusterId -eq "$scope/providers/Microsoft.AzureStackHCI/clusters/localboxcluster" -and $CreateArguments -notcontains '--version' } } - It 'preserves completed infrastructure when the late image stage fails and does not start the reference VM' { + It 'retains serial provisioning as an explicit fallback' { + $Settings.SequentialProvisioning = $true + & $prepareWorkloads -Confirm:$false + $events -join ',' | Should -Be 'network:localbox-vm-lnet-vlan200,network:localbox-aks-lnet-vlan110,aks-validate,aks-create,proxy,metallb,image,reference' + Should -Invoke Invoke-LocalBoxAz -Times 0 -ParameterFilter { $Arguments -contains '--no-wait' } + } + It 'does not start a reference VM or claim AKS completion when image import fails' { Mock Sync-LocalBoxImage { $events.Add('image-failed'); throw 'Import failed' } { & $prepareWorkloads -Confirm:$false } | Should -Throw '*Import failed*' - $events -join ',' | Should -Be 'network:localbox-vm-lnet-vlan200,network:localbox-aks-lnet-vlan110,aks-validate,aks-create,proxy,metallb,image-failed' + $events -join ',' | Should -Be 'network:localbox-vm-lnet-vlan200,network:localbox-aks-lnet-vlan110,aks-validate,aks-create,image-failed' Should -Invoke Sync-LocalBoxReferenceVm -Times 0 + Should -Invoke Wait-LocalBoxResource -Times 0 + } + It 'does not start a reference VM when AKS fails after the image completes' { + Mock Wait-LocalBoxResource { throw 'AKS provisioning Failed' } + { & $prepareWorkloads -Confirm:$false } | Should -Throw '*AKS provisioning Failed*' + Should -Invoke Sync-LocalBoxImage -Times 1 -Exactly + Should -Invoke Sync-LocalBoxReferenceVm -Times 0 + Should -Invoke Sync-LocalBoxMetalLb -Times 0 + } + It 'does not import the image when asynchronous submission fails' { + Mock Invoke-LocalBoxAz { throw 'AKS submission failed' } -ParameterFilter { $Arguments -contains '--no-wait' } + { & $prepareWorkloads -Confirm:$false } | Should -Throw '*AKS submission failed*' + Should -Invoke Sync-LocalBoxImage -Times 0 + Should -Invoke Sync-LocalBoxReferenceVm -Times 0 + } + It 'does not import the image or submit AKS after failed AKS validation' { + Mock Invoke-LocalBoxAz { throw 'AKS validation failed' } -ParameterFilter { $Arguments -contains '--validate' } + { & $prepareWorkloads -Confirm:$false } | Should -Throw '*AKS validation failed*' + Should -Invoke Invoke-LocalBoxAz -Times 0 -ParameterFilter { $Arguments -contains '--no-wait' } + Should -Invoke Sync-LocalBoxImage -Times 0 + } + It 'preserves verification-before-image ordering when AKS already exists' { + Mock Invoke-LocalBoxAz { + if ($Arguments[0] -eq 'resource') { @{ id = "$scope/providers/Microsoft.Kubernetes/connectedClusters/localbox-aks" } } + elseif ($Arguments[0] -eq 'provider') { @{ registrationState = 'Registered' } } + } + & $prepareWorkloads -Confirm:$false + $events -join ',' | Should -Be 'network:localbox-vm-lnet-vlan200,network:localbox-aks-lnet-vlan110,proxy,metallb,image,reference' + Should -Invoke Invoke-LocalBoxAz -Times 0 -ParameterFilter { $Arguments[0] -eq 'aksarc' } + } + It 'does not submit asynchronous operations during WhatIf' { + & $prepareWorkloads -WhatIf + Should -Invoke Invoke-LocalBoxAz -Times 0 -ParameterFilter { $Arguments[0] -eq 'aksarc' } + Should -Invoke Wait-LocalBoxResource -Times 0 + Should -Invoke Sync-LocalBoxImage -Times 1 -Exactly } It 'preserves the explicit SkipAks behavior while keeping the VM stages last' { $Settings.SkipAks = $true @@ -2256,9 +2308,11 @@ exit 0 $source = ${function:Invoke-LocalBoxPreparation}.ToString() $source | Should -Match 'Invoke-LocalBoxAz \(\$arguments \+ @\(''--validate''\)\) -TimeoutSeconds \$timeout -NoOutput' $source | Should -Match 'Invoke-LocalBoxAz \$arguments -TimeoutSeconds \$timeout -NoOutput' - $source | Should -Match 'Wait-LocalBoxResource \$instanceId' - $source | Should -Match 'Assert-LocalBoxProperties \$instance \$aksExpected' - $source | Should -Match 'Assert-LocalBoxProperties \$connected' + $source | Should -Match 'Invoke-LocalBoxAz \(\$arguments \+ @\(''--no-wait''\)\) -TimeoutSeconds \$timeout -NoOutput' + $completion = ${function:Complete-LocalBoxAksPreparation}.ToString() + $completion | Should -Match 'Wait-LocalBoxResource \$instanceId' + $completion | Should -Match 'Assert-LocalBoxProperties \$instance \$Expected' + $completion | Should -Match 'Assert-LocalBoxProperties \$connected' } } diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md index 49880cda0..ddcc457ee 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md @@ -202,9 +202,17 @@ malformed/unavailable catalogs, discovery before deletion, unchanged reuse of working images, and verification of the actual imported version. The resolver was also run read-only against the live cluster catalog on 2026-10-21; it selected `26100.33438.260905`, while Azure Compute exposed the newer `26100.33650.261013`. -It executes the preparation workload sequence -with mocked Azure calls to verify that networks, AKS and MetalLB precede image -import and the optional reference VM, including when image import fails. +It executes the preparation workload sequence with mocked Azure calls to verify +that shared storage and networks precede workload provisioning. Fresh AKS creation +uses `--no-wait` after synchronous validation, overlaps the image import, and then +joins AKS readiness/Arc connectivity/MetalLB before creating the reference VM. +Tests cover submission, validation, image and AKS failures; existing-AKS reuse; +`-SkipAks`; `-WhatIf`; and the explicit `-SequentialProvisioning` fallback. +Image import still uses the existing version, placement and failed-image recovery +guards. A cold live deployment is needed to measure actual overlap and contention; +mocked ordering is not a performance benchmark. +Read-only verification on traininglab-02 confirmed that the installed `aksarc` +extension advertises `create --no-wait`; no provisioning was invoked by that check. Storage tests verify no repeated disk/partition mutations on a completed rerun, resumption of an interrupted partition expansion, and detection of unallocated From ada1bfba1288882f0397b9faa20cbfb3e7a6dae8 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Thu, 5 Nov 2026 09:56:15 +0000 Subject: [PATCH 16/37] Pin guest package with overlapping AKS and image provisioning Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../labautomation/localbox-automation-source.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation-source.json b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation-source.json index 5ca470992..6f72cf677 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation-source.json +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation-source.json @@ -1,8 +1,8 @@ { "repository": "janegilring/MicroHack", - "commit": "a4188387649a52623f510f4d4873253c79d679f6", + "commit": "2c7fad3e6e56818869bb33c07f08aa0cafd64071", "files": { - "prepare-localbox.ps1": "57725d1b9d71bb93f93ac0768cc7f72e76da733633cebe904a03a20a5a800232", + "prepare-localbox.ps1": "8d965246fefc72ad89652054261b0f343cb1dea1bc1d68038a8abb20cba513fd", "invoke-localbox-automation.ps1": "cbcb8dbee2523497a3d6b8c0586154c01e579bce3183de9aeb10d4e3ccb7c425", "test-sovereign-cloud.ps1": "e23583ac0a3747261d57469a7ae3e31438c0cb70beb9bf1c751a6244e4ea5117", "tests/localbox.health.tests.ps1": "92f922dff75b1bcecc7c3e9a523c613930bd1a538ef8bb0fd50dd2ee1267b4d9", From bfb3d176099afc7a7f0d047b9107faa964bfe5b8 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Thu, 5 Nov 2026 20:30:37 +0000 Subject: [PATCH 17/37] Guard new LocalBox startup downloads and surface early failures Add bounded verified base-image staging, immutable new-VM bootstrap protection, replay prevention and correlated Console startup monitoring. Preserve existing unprotected instances and all final readiness gates. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../labautomation/deploy-localbox.ps1 | 25 ++ .../labautomation/localbox-automation.ps1 | 76 +++- .../labautomation/localbox-startup-watch.ps1 | 69 ++++ .../labautomation/localbox-startup.ps1 | 301 ++++++++++++++ .../resources/hosted-events/readme.md | 40 ++ .../tests/localbox-deployment.tests.ps1 | 42 +- .../tests/localbox-startup.tests.ps1 | 382 ++++++++++++++++++ .../resources/tests/readme.md | 12 + 8 files changed, 945 insertions(+), 2 deletions(-) create mode 100644 03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-startup-watch.ps1 create mode 100644 03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-startup.ps1 create mode 100644 03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-startup.tests.ps1 diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/deploy-localbox.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/deploy-localbox.ps1 index e91c1a417..7972f6289 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/deploy-localbox.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/deploy-localbox.ps1 @@ -133,6 +133,8 @@ $networkTemplateFile = Join-Path $PSScriptRoot 'localbox-network.bicep' if (-not (Test-Path -LiteralPath $networkTemplateFile -PathType Leaf)) { throw 'Missing localbox-network.bicep. Import the complete labautomation directory before deploying LocalBox.' } +$startupSource = Join-Path $PSScriptRoot 'localbox-startup.ps1' +. $startupSource if (-not (Get-Command az -ErrorAction SilentlyContinue)) { throw 'Azure CLI is required to deploy LocalBox.' @@ -331,6 +333,29 @@ try { SecurityControl = 'Ignore' CostControl = 'Ignore' } + $clients = @(Invoke-AzJson -Arguments @('resource', 'list', '--subscription', $SubscriptionId, + '--resource-group', $ResourceGroupName, '--resource-type', 'Microsoft.Compute/virtualMachines') | + Where-Object name -eq 'LocalBox-Client') + if ($clients.Count -gt 1) { throw 'Cannot uniquely identify the existing LocalBox Client.' } + $newClient = $clients.Count -eq 0 + $startupId = [guid]::Empty + if ($newClient) { $startupId = [guid]::NewGuid() } + elseif ($clients[0].tags.'microhack-startup-version' -or $clients[0].tags.'microhack-startup-id') { + if ($clients[0].tags.'microhack-startup-version' -ne '1' -or + -not [guid]::TryParse([string]$clients[0].tags.'microhack-startup-id', [ref]$startupId) -or $startupId -eq [guid]::Empty) { + throw 'Existing LocalBox startup-protection metadata is unrecognized. No bootstrap migration was attempted.' + } + } + if ($startupId -ne [guid]::Empty) { + $resourceTags['microhack-startup-version'] = '1' + $resourceTags['microhack-startup-id'] = $startupId.ToString() + Set-LocalBoxStartupTemplate -HostTemplate (Join-Path $tempDirectory 'host/host.bicep') ` + -StartupSource $startupSource -Id $startupId -NewVm:$newClient + Write-Host 'Using correlated startup protection with bounded base-image downloads. Existing VM custom data is never replaced.' + } + else { + Write-Warning 'This existing Client predates startup protection. Its bootstrap is left unchanged; use a new VM to enable guarded startup.' + } @{ '$schema' = 'https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#' diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation.ps1 index 19371b76e..f905c6e31 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation.ps1 @@ -90,10 +90,77 @@ function Get-LocalBoxValidationRetryState { Detail = "Automatic validation retry 1/1: $($state.executionState). Deployment remains gated on successful validation." } } +function Get-LocalBoxStartupProgress { + param([hashtable]$Tracking, [string]$SubscriptionId, [string]$ResourceGroupName, [int]$TimeoutSeconds) + $vmId = "/subscriptions/$SubscriptionId/resourceGroups/$ResourceGroupName/providers/Microsoft.Compute/virtualMachines/LocalBox-Client" + $uri = "$vmId/runCommands/sovereign-localbox-startup-watch?api-version=2024-07-01" + if (-not $Tracking.Checked) { + $vm = Invoke-LocalBoxAutomationRequest -Path "$vmId`?api-version=2024-07-01" + $Tracking.Checked = $true + if (-not $vm.tags.'microhack-startup-version' -and -not $vm.tags.'microhack-startup-id') { return } + $id = [guid]::Empty + if ($vm.tags.'microhack-startup-version' -ne '1' -or + -not [guid]::TryParse([string]$vm.tags.'microhack-startup-id', [ref]$id) -or $id -eq [guid]::Empty) { + throw 'Invalid VM startup-protection metadata. Startup readiness cannot be verified.' + } + $Tracking.Id = $id.ToString() + $command = Invoke-LocalBoxAutomationRequest -Path "$uri&`$expand=instanceView" -AllowNotFound + if (-not $command) { + if ($TimeoutSeconds -le 0) { throw 'No nested-deployment budget remains for startup monitoring.' } + $startedAfter = [datetime]::UtcNow.ToString('o') + Invoke-LocalBoxAutomationRequest -Path $uri -Method PUT -Body @{ + location = $vm.location + properties = @{ + source = @{ script = Get-Content (Join-Path $PSScriptRoot 'localbox-startup-watch.ps1') -Raw -ErrorAction Stop } + asyncExecution = $true; timeoutInSeconds = $TimeoutSeconds + parameters = @( + @{ name = 'BootstrapId'; value = $Tracking.Id } + @{ name = 'SubscriptionId'; value = $SubscriptionId } + @{ name = 'ResourceGroupName'; value = $ResourceGroupName } + @{ name = 'StartedAfter'; value = $startedAfter } + @{ name = 'TimeoutSeconds'; value = [string]$TimeoutSeconds } + ) + } + } | Out-Null + } + } + if (-not $Tracking.Id) { return } + $command = Invoke-LocalBoxAutomationRequest -Path "$uri&`$expand=instanceView" + $metadata = @{} + foreach ($parameter in $command.properties.parameters) { + if ($metadata.ContainsKey($parameter.name)) { throw 'Duplicate startup-monitor metadata.' } + $metadata[$parameter.name] = $parameter.value + } + $started = [datetimeoffset]::MinValue + if ($metadata.BootstrapId -ne $Tracking.Id -or $metadata.SubscriptionId -ne $SubscriptionId -or + $metadata.ResourceGroupName -ne $ResourceGroupName -or + -not [datetimeoffset]::TryParse([string]$metadata.StartedAfter, [ref]$started)) { + throw 'An existing startup monitor does not match this VM deployment; it was not replaced.' + } + if ($command.properties.provisioningState -in @('Failed', 'Canceled')) { throw 'The LocalBox startup monitor could not be provisioned.' } + $instance = $command.properties.instanceView + if (-not $instance.startTime -or [datetimeoffset]$instance.startTime -lt $started -or + $instance.executionState -notin @('Succeeded', 'Failed', 'Canceled', 'TimedOut')) { return } + $matches = [regex]::Matches([string]$instance.output, '(?m)^LOCALBOX_STARTUP_REPORT:(.+)\r?$') + if ($matches.Count -ne 1) { throw 'LocalBox startup monitoring stopped without a verified report. Inspect sovereign-localbox-startup-watch.' } + try { $report = $matches[0].Groups[1].Value | ConvertFrom-Json -AsHashtable -ErrorAction Stop } + catch { throw 'Invalid LocalBox startup report. Inspect sovereign-localbox-startup-watch.' } + if ($report.BootstrapId -ne $Tracking.Id -or $report.SubscriptionId -ne $SubscriptionId -or $report.ResourceGroupName -ne $ResourceGroupName) { + throw 'LocalBox startup report identity mismatch; no report details were accepted.' + } + if ($report.State -eq 'Failed') { + throw "Upstream LocalBox startup failed during $(ConvertTo-LocalBoxDiagnosticText $report.Phase 80) [$(ConvertTo-LocalBoxDiagnosticText $report.Code 80)]: $(ConvertTo-LocalBoxDiagnosticText $report.Message). Azure Local validation/deployment has not started; no infrastructure readiness is claimed." + } + if ($report.State -ne 'Succeeded' -or $instance.executionState -ne 'Succeeded' -or $null -eq $instance.exitCode -or $instance.exitCode -ne 0) { + throw 'The LocalBox startup monitor did not finish successfully; readiness remains unverified.' + } +} + function Wait-LocalBoxNestedDeployment { param([string]$SubscriptionId, [string]$ResourceGroupName, [int]$TimeoutSeconds = 28800) $deadline = [datetime]::UtcNow.AddSeconds($TimeoutSeconds) $retry = @{ Checked = $false; Command = $null; OriginalCorrelationId = $null } + $startup = @{ Checked = $false; Id = $null } $vmId = "/subscriptions/$SubscriptionId/resourceGroups/$ResourceGroupName/providers/Microsoft.Compute/virtualMachines/LocalBox-Client" $retryUri = "$vmId/runCommands/sovereign-localbox-validation-retry?api-version=2024-07-01" Wait-LocalBoxAutomation -Phase 'nested deployment' -TimeoutSeconds $TimeoutSeconds -Probe { @@ -101,6 +168,10 @@ function Wait-LocalBoxNestedDeployment { $clusterDeployment = @($deployments | Where-Object name -eq 'localcluster-deploy') $validation = @($deployments | Where-Object name -eq 'localcluster-validate') if ($clusterDeployment.Count -gt 1 -or $validation.Count -gt 1) { throw 'Ambiguous nested deployment records.' } + if (-not $clusterDeployment.Count -and -not $validation.Count) { + $remaining = [int][math]::Max(0, [math]::Floor(($deadline - [datetime]::UtcNow).TotalSeconds)) + Get-LocalBoxStartupProgress -Tracking $startup -SubscriptionId $SubscriptionId -ResourceGroupName $ResourceGroupName -TimeoutSeconds $remaining + } if ($clusterDeployment.Count -eq 1 -and $clusterDeployment[0].properties.provisioningState -in @('Failed', 'Canceled')) { throw 'Nested deployment localcluster-deploy failed. Inspect its deployment operations and Client logs.' } @@ -165,7 +236,10 @@ function Wait-LocalBoxNestedDeployment { return @{ Ready = $false; Detail = 'Automatic validation retry 1/1 submitted. Credentials and existing ARM files remain on LocalBox-Client.' } } if ($clusterDeployment.Count -ne 1 -or $clusterDeployment[0].properties.provisioningState -ne 'Succeeded') { - return @{ Ready = $false; Detail = 'Waiting for localcluster-deploy to succeed.' } + $detail = if (-not $validation.Count -and -not $clusterDeployment.Count) { + 'Azure Local validation/deployment records are absent; waiting for upstream Client startup.' + } else { 'Waiting for localcluster-deploy to succeed.' } + return @{ Ready = $false; Detail = $detail } } $resources = @(Invoke-LocalBoxAutomationAz @('resource', 'list', '--subscription', $SubscriptionId, '--resource-group', $ResourceGroupName)) foreach ($type in @('Microsoft.AzureStackHCI/clusters', 'Microsoft.ResourceConnector/appliances', 'Microsoft.ExtendedLocation/customLocations')) { diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-startup-watch.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-startup-watch.ps1 new file mode 100644 index 000000000..5ce55d41e --- /dev/null +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-startup-watch.ps1 @@ -0,0 +1,69 @@ +[CmdletBinding()] +param( + [string]$BootstrapId, [string]$SubscriptionId, [string]$ResourceGroupName, + [string]$StartedAfter, [int]$TimeoutSeconds = 28800 +) + +function Get-LocalBoxStartupReport { + param([string]$Path, [string]$Id, [string]$Subscription, [string]$Group) + if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) { return } + try { $status = Get-Content -LiteralPath $Path -Raw -ErrorAction Stop | ConvertFrom-Json -ErrorAction Stop } + catch { throw 'The upstream startup report could not be read or parsed.' } + if ($status.BootstrapId -ne $Id -or $status.SubscriptionId -ne $Subscription -or $status.ResourceGroupName -ne $Group -or + $status.State -notin @('Running', 'Succeeded', 'Failed') -or + $status.Phase -notin @('Startup', 'BaseImages', 'NestedDeployment', 'Complete')) { + throw 'The upstream startup report does not match this VM or has invalid state metadata.' + } + @{ + BootstrapId = $Id; SubscriptionId = $Subscription; ResourceGroupName = $Group + State = $status.State; Phase = $status.Phase; Code = [string]$status.Code + Message = ([string]$status.Message).Substring(0, [math]::Min(1000, ([string]$status.Message).Length)) + } +} + +function Wait-LocalBoxStartupReport { + param([string]$Id, [string]$Subscription, [string]$Group, [int]$TimeoutSeconds) + $ErrorActionPreference = 'Stop' + $started = (Get-Date).ToUniversalTime() + $deadline = $started.AddSeconds($TimeoutSeconds) + $report = @{ + BootstrapId = $Id; SubscriptionId = $Subscription; ResourceGroupName = $Group + State = 'Failed'; Phase = 'Startup'; Code = 'StartupMonitorFailed' + Message = 'Upstream startup could not be verified.' + } + try { + do { + $status = Get-LocalBoxStartupReport -Path 'C:\LocalBox\SovereignStartup\state.json' -Id $Id -Subscription $Subscription -Group $Group + if ($status -and $status.State -in @('Succeeded', 'Failed')) { return $status } + if (-not $status -and (Get-Date).ToUniversalTime() -ge $started.AddHours(1)) { + $report.Code = 'StartupNotStarted' + $report.Message = 'No correlated startup report appeared within one hour. Inspect WinGetLogonScript and LocalBoxLogonScript scheduled tasks; Azure Local validation has not started.' + return $report + } + if ($status -and (Get-Date).ToUniversalTime() -ge $started.AddMinutes(2)) { + $task = Get-ScheduledTask -TaskName LocalBoxLogonScript -ErrorAction Stop + if ($task.State -ne 'Running') { + $report.Code = 'StartupExited' + $report.Message = 'The startup task stopped without a successful report. Inspect LocalBox logs; do not repeat storage initialization blindly.' + return $report + } + } + if ((Get-Date).ToUniversalTime() -ge $deadline) { + $report.Code = 'StartupTimedOut' + $report.Message = 'Upstream startup did not finish within the nested-deployment budget.' + return $report + } + Start-Sleep -Seconds 15 + } while ($true) + } + catch { + $report.Message = 'Startup monitoring failed to verify the local report or scheduled task. Inspect sovereign-localbox-startup-watch and LocalBox logs.' + return $report + } +} + +if ($MyInvocation.InvocationName -ne '.') { + $report = Wait-LocalBoxStartupReport -Id $BootstrapId -Subscription $SubscriptionId -Group $ResourceGroupName -TimeoutSeconds $TimeoutSeconds + Write-Output ('LOCALBOX_STARTUP_REPORT:' + ($report | ConvertTo-Json -Depth 5 -Compress)) + if ($report.State -ne 'Succeeded') { exit 1 } +} diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-startup.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-startup.ps1 new file mode 100644 index 000000000..aad6aa656 --- /dev/null +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-startup.ps1 @@ -0,0 +1,301 @@ +#Requires -Version 5.1 +[CmdletBinding()] +param( + [switch]$Install, + [switch]$DownloadImages, + [string]$BootstrapId = '__LOCALBOX_BOOTSTRAP_ID__' +) + +function Set-LocalBoxStartupTemplate { + param([string]$HostTemplate, [string]$StartupSource, [guid]$Id, [switch]$NewVm) + $text = Get-Content -LiteralPath $HostTemplate -Raw -ErrorAction Stop + $pattern = '(?m)^([ \t]*)commandToExecute: ''(powershell\.exe -ExecutionPolicy Bypass -File Bootstrap\.ps1 [^''\r\n]+)''[ \t]*\r?$' + $matches = [regex]::Matches($text, $pattern) + if ($matches.Count -ne 1 -or $text -match 'customData:') { + throw 'Unsupported upstream LocalBox host template; startup protection cannot be installed safely.' + } + $command = 'cmd.exe /d /c "copy /y C:\\AzureData\\CustomData.bin SovereignStartup.ps1 >nul && powershell.exe -NoProfile -ExecutionPolicy Bypass -File SovereignStartup.ps1 -Install && ' + + $matches[0].Groups[2].Value + '"' + $text = $text.Replace($matches[0].Value, $matches[0].Groups[1].Value + "commandToExecute: '$command'") + if ($NewVm) { + if ([regex]::Matches($text, 'osProfile: \{').Count -ne 1) { throw 'Cannot uniquely locate the upstream VM osProfile.' } + $source = (Get-Content -LiteralPath $StartupSource -Raw -ErrorAction Stop).Replace('__LOCALBOX_BOOTSTRAP_ID__', $Id.ToString()) + if ([Text.Encoding]::UTF8.GetByteCount($source) -gt 65535) { throw 'LocalBox startup protection exceeds the VM custom-data limit.' } + $destination = Join-Path (Split-Path -Parent $HostTemplate) 'localbox-startup.ps1' + Set-Content -LiteralPath $destination -Value $source -Encoding utf8NoBOM -NoNewline -ErrorAction Stop + $text = $text.Replace('osProfile: {', "osProfile: {`n customData: base64(loadTextContent('localbox-startup.ps1'))") + } + Set-Content -LiteralPath $HostTemplate -Value $text -Encoding utf8 -ErrorAction Stop +} + +function Install-LocalBoxStartupGuard { + param([string]$Path = '.\Bootstrap.ps1') + $text = Get-Content -LiteralPath $Path -Raw -ErrorAction Stop + $action = '$Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $LocalBoxPath\LocalBoxLogonScript.ps1' + if ([regex]::Matches($text, [regex]::Escape($action)).Count -ne 1) { + throw 'Unsupported upstream Bootstrap task action; refusing to start an unguarded deployment.' + } + $replacement = @' +$guardRoot = Join-Path $LocalBoxPath 'SovereignStartup' +New-Item -ItemType Directory -Path $guardRoot -Force -ErrorAction Stop | Out-Null +Set-Acl -LiteralPath $guardRoot -AclObject (Get-Acl -LiteralPath 'C:\AzureData' -ErrorAction Stop) -ErrorAction Stop +Copy-Item -LiteralPath 'C:\AzureData\CustomData.bin' -Destination "$guardRoot\startup.ps1" -Force -ErrorAction Stop +$Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument "$guardRoot\startup.ps1" +'@ + Set-Content -LiteralPath $Path -Value $text.Replace($action, $replacement) -Encoding utf8 -ErrorAction Stop +} + +function Save-LocalBoxStartupStatus { + param([hashtable]$Status, [string]$Path) + $Status.UpdatedAt = [datetime]::UtcNow.ToString('o') + $temporary = "$Path.$PID.tmp" + try { + $Status | ConvertTo-Json -Depth 5 | Set-Content -LiteralPath $temporary -Encoding utf8 -ErrorAction Stop + Move-Item -LiteralPath $temporary -Destination $Path -Force -ErrorAction Stop + } + finally { + if (Test-Path -LiteralPath $temporary) { Remove-Item -LiteralPath $temporary -Force -ErrorAction Stop } + } +} + +function Set-LocalBoxBaseImageGuard { + param([string]$Path, [string]$PlanPath, [string]$Id) + $text = Get-Content -LiteralPath $Path -Raw -ErrorAction Stop + $start = '# Download LocalBox VHDs' + $end = '# Set credentials' + if ([regex]::Matches($text, [regex]::Escape($start)).Count -ne 1 -or + [regex]::Matches($text, [regex]::Escape($end)).Count -ne 1 -or $text.IndexOf($end) -le $text.IndexOf($start)) { + throw 'Unsupported upstream base-image download block; no cluster startup was attempted.' + } + $block = $text.Substring($text.IndexOf($start), $text.IndexOf($end) - $text.IndexOf($start)) + $pattern = '(?m)^[ \t]*azcopy cp (?:''(?https://[^'']+)''|"(?https://[^"]+)"|(?https://\S+)) "\$\(\$LocalBoxConfig\.Paths\.VHDDir\)\\(?[^"]+)"[^\r\n]*' + $artifacts = @{} + foreach ($match in [regex]::Matches($block, $pattern)) { + $name = $match.Groups['name'].Value + $uri = [uri]$match.Groups['url'].Value + if ($name -notin @('AzL-node.vhdx', 'AzL-node.sha256', 'GUI.vhdx', 'GUI.sha256') -or $artifacts.ContainsKey($name) -or + $uri.Host -notin @('azlocalvhds.blob.core.windows.net', 'jumpstartprodsg.blob.core.windows.net') -or + $uri.Query -or $uri.Fragment -or $uri.UserInfo) { + throw 'Unrecognized or duplicate upstream base-image source; organizer review is required.' + } + $artifacts[$name] = $uri.AbsoluteUri + } + if ($artifacts.Count -ne 4) { throw 'Expected exactly four upstream VHD/checksum sources; refusing a partial download plan.' } + @{ BootstrapId = $Id; Artifacts = $artifacts } | ConvertTo-Json | + Set-Content -LiteralPath $PlanPath -Encoding utf8 -ErrorAction Stop + $replacement = @' +# Sovereign guarded base-image downloads +& "$Env:LocalBoxDir\SovereignStartup\startup.ps1" -DownloadImages +if (-not $?) { throw 'Base-image staging failed; nested deployment was not started.' } + +'@ + Set-Content -LiteralPath $Path -Value $text.Replace($block, $replacement) -Encoding utf8 -ErrorAction Stop +} + +function Test-LocalBoxBaseImage { + param([string]$Image, [string]$Checksum) + if (-not (Test-Path -LiteralPath $Image -PathType Leaf) -or -not (Test-Path -LiteralPath $Checksum -PathType Leaf)) { return $false } + if ((Get-Item -LiteralPath $Checksum -ErrorAction Stop).Length -gt 256) { return $false } + $expected = ([string](Get-Content -LiteralPath $Checksum -Raw -ErrorAction Stop)).Trim() + if ($expected -notmatch '^[a-fA-F0-9]{64}$' -or (Get-Item -LiteralPath $Image -ErrorAction Stop).Length -le 0) { return $false } + return (Get-FileHash -LiteralPath $Image -Algorithm SHA256 -ErrorAction Stop).Hash -eq $expected +} + +function Invoke-LocalBoxBaseImageTransfer { + param([string]$Uri, [string]$Destination, [string]$LogDirectory, [int]$TimeoutSeconds) + $client = Get-Command azcopy -CommandType Application -ErrorAction Stop + $stdout = Join-Path $LogDirectory 'stdout.log' + $stderr = Join-Path $LogDirectory 'stderr.log' + $oldLogs = $env:AZCOPY_LOG_LOCATION + $oldPlans = $env:AZCOPY_JOB_PLAN_LOCATION + $process = $null + try { + $env:AZCOPY_LOG_LOCATION = $LogDirectory + $env:AZCOPY_JOB_PLAN_LOCATION = $LogDirectory + $process = Start-Process -FilePath $client.Path -ArgumentList @('copy', "`"$Uri`"", "`"$Destination`"", + '--recursive=true', '--check-length=false', '--log-level=ERROR') -PassThru -NoNewWindow ` + -RedirectStandardOutput $stdout -RedirectStandardError $stderr -ErrorAction Stop + if (-not $process.WaitForExit($TimeoutSeconds * 1000)) { + throw "AzCopy transfer timed out after $TimeoutSeconds seconds." + } + $process.WaitForExit() + if ($process.ExitCode -ne 0) { + $http = @( + foreach ($file in Get-ChildItem -LiteralPath $LogDirectory -Filter '*.log' -File) { + foreach ($line in Get-Content -LiteralPath $file.FullName -Tail 80) { + if ($line -match '(?:RESPONSE(?: Status:)?|StatusCode[:=])\s*(4\d\d|5\d\d)') { $Matches[1] } + } + } + ) | Select-Object -Unique + $detail = if ($http) { " HTTP $($http -join ', ')." } else { '' } + throw "AzCopy failed with exit code $($process.ExitCode).$detail Verify source availability and access; no authentication bypass is attempted." + } + if (-not (Test-Path -LiteralPath $Destination -PathType Leaf)) { throw 'AzCopy reported success without creating the requested file.' } + } + finally { + try { + if ($process) { + try { + if (-not $process.HasExited) { + $process.Kill($true) + if (-not $process.WaitForExit(10000)) { throw 'The timed-out AzCopy process did not stop.' } + } + } + finally { $process.Dispose() } + } + } + finally { + $env:AZCOPY_LOG_LOCATION = $oldLogs + $env:AZCOPY_JOB_PLAN_LOCATION = $oldPlans + } + } +} + +function Sync-LocalBoxBaseImages { + param( + [hashtable]$Artifacts, [string]$Directory, [hashtable]$Status, [string]$StatusPath, + [ValidateRange(1, 3)][int]$Attempts = 3, [ValidateRange(0, 120)][int]$RetryDelaySeconds = 60, + [ValidateRange(1, 7200)][int]$TimeoutSeconds = 7200 + ) + $ErrorActionPreference = 'Stop' + $expectedNames = @('AzL-node.vhdx', 'AzL-node.sha256', 'GUI.vhdx', 'GUI.sha256') + if ($Artifacts.Count -ne 4) { throw 'Invalid base-image download plan: exactly four artifacts are required.' } + foreach ($name in $expectedNames) { + $uri = $null + if (-not [uri]::TryCreate([string]$Artifacts[$name], [UriKind]::Absolute, [ref]$uri) -or + $uri.Scheme -ne 'https' -or $uri.Host -notin @('azlocalvhds.blob.core.windows.net', 'jumpstartprodsg.blob.core.windows.net') -or + $uri.Query -or $uri.Fragment -or $uri.UserInfo -or -not $uri.IsDefaultPort) { + throw 'Invalid base-image download plan: an artifact source is not recognized.' + } + } + $deadline = [datetime]::UtcNow.AddSeconds($TimeoutSeconds) + $Status.Phase = 'BaseImages' + Save-LocalBoxStartupStatus $Status $StatusPath + New-Item -ItemType Directory -Path $Directory -Force -ErrorAction Stop | Out-Null + foreach ($name in @('AzL-node', 'GUI')) { + $image = Join-Path $Directory "$name.vhdx" + $checksum = Join-Path $Directory "$name.sha256" + if (Test-LocalBoxBaseImage $image $checksum) { + Write-Host "Reusing SHA256-verified base image $name." + continue + } + for ($attempt = 1; $attempt -le $Attempts; $attempt++) { + $temporary = Join-Path $Directory ".download-$name-$([guid]::NewGuid())" + New-Item -ItemType Directory -Path $temporary -ErrorAction Stop | Out-Null + try { + $Status.Message = "Downloading base image $name; attempt $attempt/$Attempts." + Save-LocalBoxStartupStatus $Status $StatusPath + Write-Host $Status.Message + foreach ($extension in @('sha256', 'vhdx')) { + $remaining = [int][math]::Floor(($deadline - [datetime]::UtcNow).TotalSeconds) + if ($remaining -le 0) { throw 'The two-hour base-image download budget is exhausted.' } + if (-not $Artifacts["$name.$extension"]) { throw "Missing source for $name.$extension." } + Invoke-LocalBoxBaseImageTransfer -Uri $Artifacts["$name.$extension"] -Destination "$temporary/$name.$extension" ` + -LogDirectory $temporary -TimeoutSeconds ([math]::Min(1800, $remaining)) + } + if (-not (Test-LocalBoxBaseImage "$temporary/$name.vhdx" "$temporary/$name.sha256")) { + throw "The downloaded $name image or SHA256 file is missing, empty, malformed or does not match." + } + if ([datetime]::UtcNow -ge $deadline) { throw 'The base-image verification exceeded its download budget.' } + Move-Item -LiteralPath "$temporary/$name.vhdx" -Destination $image -Force -ErrorAction Stop + Move-Item -LiteralPath "$temporary/$name.sha256" -Destination $checksum -Force -ErrorAction Stop + break + } + catch { + $message = "Base image $name failed on attempt $attempt/$Attempts. $($_.Exception.Message)" + Write-Warning $message + if ($attempt -eq $Attempts -or [datetime]::UtcNow.AddSeconds($RetryDelaySeconds) -ge $deadline) { + $Status.State = 'Failed' + $Status.Code = 'BaseImageDownloadFailed' + $Status.Message = $message + Save-LocalBoxStartupStatus $Status $StatusPath + throw $message + } + Start-Sleep -Seconds $RetryDelaySeconds + } + finally { + Remove-Item -LiteralPath $temporary -Recurse -Force -ErrorAction Stop + } + } + } + $Status.Phase = 'NestedDeployment' + $Status.Message = 'Both base images have verified SHA256 checksums; upstream cluster construction may proceed.' + Save-LocalBoxStartupStatus $Status $StatusPath +} + +function Invoke-LocalBoxGuardedStartup { + param([string]$Id, [string]$Root = 'C:\LocalBox') + $ErrorActionPreference = 'Stop' + $statusPath = Join-Path $Root 'SovereignStartup/state.json' + $lock = [Threading.Mutex]::new($false, 'Global\MicroHackLocalBoxStartup') + $locked = $false + $status = $null + try { + $locked = $lock.WaitOne(0) + if (-not $locked) { throw 'Another LocalBox startup is active; storage initialization will not be repeated.' } + if (Test-Path -LiteralPath $statusPath) { + $previous = Get-Content -LiteralPath $statusPath -Raw | ConvertFrom-Json -AsHashtable -ErrorAction Stop + if ($previous.BootstrapId -eq $Id -and $previous.State -eq 'Succeeded') { + Write-Host 'This LocalBox startup already completed; not repeating initialization.' + return + } + throw 'An earlier LocalBox startup is incomplete or failed. Automatic replay of storage initialization is disabled; organizer review is required.' + } + New-Item -ItemType Directory -Path (Split-Path -Parent $statusPath) -Force | Out-Null + $status = @{ + BootstrapId = $Id; SubscriptionId = $env:subscriptionId; ResourceGroupName = $env:resourceGroup + State = 'Running'; Phase = 'Startup'; Code = ''; Message = 'Upstream LocalBox startup is beginning.' + StartedAt = [datetime]::UtcNow.ToString('o') + } + Save-LocalBoxStartupStatus $status $statusPath + Set-LocalBoxBaseImageGuard -Path "$Root/New-LocalBoxCluster.ps1" -PlanPath "$Root/SovereignStartup/images.json" -Id $Id + & "$Root/LocalBoxLogonScript.ps1" + if (-not $?) { throw 'Upstream LocalBox startup returned unsuccessfully.' } + $status = Get-Content -LiteralPath $statusPath -Raw | ConvertFrom-Json -AsHashtable -ErrorAction Stop + if ($status.State -eq 'Failed') { throw 'Upstream startup returned after a reported base-image failure.' } + $status.State = 'Succeeded' + $status.Phase = 'Complete' + $status.Message = 'Upstream startup returned; Azure deployment and Full health gates are still required.' + Save-LocalBoxStartupStatus $status $statusPath + } + catch { + $failure = $_ + if ($status) { + try { + $latest = Get-Content -LiteralPath $statusPath -Raw | ConvertFrom-Json -AsHashtable -ErrorAction Stop + if ($latest.State -ne 'Failed') { + $latest.State = 'Failed' + $latest.Code = 'StartupFailed' + $latest.Message = 'Upstream LocalBox startup failed. Inspect LocalBoxLogonScript.log and New-LocalBoxCluster.log; do not blindly rerun storage initialization.' + Save-LocalBoxStartupStatus $latest $statusPath + } + } + catch { Write-Warning 'Could not persist the startup failure report. The original failure is preserved; inspect local startup logs.' } + } + throw $failure + } + finally { + if ($locked) { $lock.ReleaseMutex() } + $lock.Dispose() + } +} + +if ($MyInvocation.InvocationName -ne '.') { + $ErrorActionPreference = 'Stop' + if ($Install) { + Install-LocalBoxStartupGuard + } + else { + if ($PSVersionTable.PSVersion.Major -lt 7) { throw 'Guarded LocalBox startup requires PowerShell 7.' } + if ($BootstrapId -notmatch '^[a-fA-F0-9-]{36}$') { throw 'Missing deployment-specific LocalBox startup identity.' } + if ($DownloadImages) { + $statusPath = 'C:\LocalBox\SovereignStartup\state.json' + $status = Get-Content -LiteralPath $statusPath -Raw | ConvertFrom-Json -AsHashtable -ErrorAction Stop + $plan = Get-Content 'C:\LocalBox\SovereignStartup\images.json' -Raw | ConvertFrom-Json -AsHashtable -ErrorAction Stop + if ($status.BootstrapId -ne $BootstrapId -or $plan.BootstrapId -ne $BootstrapId) { throw 'Base-image download metadata does not match this startup.' } + $config = Import-PowerShellDataFile -LiteralPath $env:LocalBoxConfigFile + Sync-LocalBoxBaseImages -Artifacts $plan.Artifacts -Directory $config.Paths.VHDDir -Status $status -StatusPath $statusPath + } + else { Invoke-LocalBoxGuardedStartup -Id $BootstrapId } + } +} diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md index 3f0e9286b..ecfe145bb 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md @@ -303,6 +303,46 @@ bounded intervals and potential optimizations. ### Diagnostics and reruns +#### Guarded startup for new Client VMs + +Re-import the complete `labautomation` directory, including +[localbox-startup.ps1](../../labautomation/localbox-startup.ps1) and +[localbox-startup-watch.ps1](../../labautomation/localbox-startup-watch.ps1). +New Client VMs receive a deployment-specific startup guard through immutable VM +custom data. Existing unprotected VMs are **not migrated or repaired**; existing +guarded VMs keep their original payload and startup identity. + +The guard replaces the recognized upstream base-VHD download/checksum block: + +- Each base image gets at most **three attempts**, with **60 seconds** between + retries. The shared download-phase budget is **two hours**; each AzCopy process + is capped at 30 minutes or the remaining budget, whichever is shorter. +- AzCopy exit codes, file existence, nonempty images and valid SHA256 checksums + are mandatory. Missing image and checksum files can never count as a match. +- Verified cached pairs are reused. Downloads are staged separately and promoted + only after verification; failed downloads do not replace canonical cached files. +- Only recognized upstream public sources are accepted. A persistent 401 fails + explicitly; there is no authentication bypass or fallback to obsolete images. +- An incomplete or failed startup is **not automatically replayed**, because the + upstream storage initialization is not safe to repeat blindly. Unsupported + upstream template/script changes also fail explicitly rather than dropping protection. + +While neither `localcluster-validate` nor `localcluster-deploy` exists, Console +uses the read-only `sovereign-localbox-startup-watch` Managed Run Command. +It verifies the startup identity and scope, then reports the failure phase, code +and bounded diagnostic message (for example, `BaseImages`, +`BaseImageDownloadFailed`, and `HTTP 401`) instead of waiting eight hours for a +deployment that was never started. No startup report within **one hour** fails +the gate; a Running report with a stopped task fails after a **two-minute +monitoring grace period**. Inspect the protected +`C:\LocalBox\SovereignStartup\state.json` and the LocalBox startup logs. + +Startup success is **not infrastructure readiness**. All existing ARM, cluster, +connectivity and Full health checks remain required. Monitoring uses the remaining +nested-readiness budget. Offline tests and compilation against the real upstream +templates/scripts validate this integration; a fresh Windows/Console deployment +is still required to verify the complete new-VM path. + #### Automatic Azure Local validation recovery The Console can perform **one automatic validation retry**, after a **two-minute diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-deployment.tests.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-deployment.tests.ps1 index 00eb90856..083d264ab 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-deployment.tests.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-deployment.tests.ps1 @@ -9,6 +9,7 @@ BeforeAll { $fixture = Join-Path $TestDrive 'lab' New-Item -ItemType Directory -Path "$fixture/labautomation", "$fixture/resources/manual-setup/localbox" -Force | Out-Null Copy-Item "$root/labautomation/deploy-localbox.ps1" "$fixture/labautomation" + Copy-Item "$root/labautomation/localbox-startup.ps1" "$fixture/labautomation" Copy-Item "$root/resources/manual-setup/localbox/deploy-localbox.ps1" "$fixture/resources/manual-setup/localbox" function az {} } @@ -55,12 +56,14 @@ Describe 'LocalBox deployment entry points' { GroupExists = $false Bastions = @() Networks = @() + Clients = @() QueriesFail = $false ValidationFails = $false Submitted = 0 Builds = 0 Parameters = $null NetworkSource = $null + HostSource = $null Downloads = [Collections.Generic.List[string]]::new() } $arguments = @{ @@ -73,7 +76,19 @@ Describe 'LocalBox deployment entry points' { } Mock Invoke-WebRequest { $deploymentTestState.Downloads.Add([string]$Uri) - Set-Content -LiteralPath $OutFile '// upstream fixture; full compilation is validated separately' + if ($Uri -like '*/host/host.bicep') { + @' +var profile = { + osProfile: { + computerName: 'LocalBox-Client' + } +} +var settings = { + commandToExecute: 'powershell.exe -ExecutionPolicy Bypass -File Bootstrap.ps1 -adminPassword opaque' +} +'@ | Set-Content -LiteralPath $OutFile + } + else { Set-Content -LiteralPath $OutFile '// upstream fixture; full compilation is validated separately' } } Mock az { $global:LASTEXITCODE = 0 @@ -93,11 +108,13 @@ Describe 'LocalBox deployment entry points' { break } '^network vnet list' { ConvertTo-Json -InputObject @($deploymentTestState.Networks) -Depth 10; break } + '^resource list' { ConvertTo-Json -InputObject @($deploymentTestState.Clients) -Depth 10; break } '^group (create|show|update)' { '{"name":"rg-localbox-test","location":"spaincentral"}'; break } '^bicep build' { $deploymentTestState.Builds++ $path = $args[[array]::IndexOf($args, '--file') + 1] $deploymentTestState.NetworkSource = Get-Content (Join-Path (Split-Path $path) 'network/network.bicep') -Raw + $deploymentTestState.HostSource = Get-Content (Join-Path (Split-Path $path) 'host/host.bicep') -Raw break } '^deployment group validate' { @@ -112,6 +129,26 @@ Describe 'LocalBox deployment entry points' { } } } + It 'preserves the bootstrap of an existing unprotected VM' { + $deploymentTestState.Clients = @(@{ name = 'LocalBox-Client'; tags = @{} }) + & "$fixture/labautomation/deploy-localbox.ps1" @arguments + $deploymentTestState.HostSource | Should -Not -Match 'customData|SovereignStartup' + $deploymentTestState.Parameters.tags.value.PSObject.Properties.Name | Should -Not -Contain 'microhack-startup-id' + } + It 'retains the immutable startup identity on a guarded existing VM' { + $deploymentTestState.Clients = @(@{ name = 'LocalBox-Client'; tags = @{ + 'microhack-startup-version' = '1'; 'microhack-startup-id' = '40000000-0000-0000-0000-000000000001' + } }) + & "$fixture/labautomation/deploy-localbox.ps1" @arguments + $deploymentTestState.HostSource | Should -Match 'SovereignStartup' + $deploymentTestState.HostSource | Should -Not -Match 'customData:' + $deploymentTestState.Parameters.tags.value.'microhack-startup-id' | Should -Be '40000000-0000-0000-0000-000000000001' + } + It 'rejects unknown protection metadata without submitting deployment' { + $deploymentTestState.Clients = @(@{ name = 'LocalBox-Client'; tags = @{ 'microhack-startup-version' = '2' } }) + { & "$fixture/labautomation/deploy-localbox.ps1" @arguments } | Should -Throw '*No bootstrap migration*' + $deploymentTestState.Submitted | Should -Be 0 + } It 'packages the local module alongside upstream templates and preserves default Bastion/private-host parameters' { $result = & "$fixture/labautomation/deploy-localbox.ps1" @arguments $result.ProvisioningState | Should -Be 'Submitted' @@ -119,6 +156,9 @@ Describe 'LocalBox deployment entry points' { $deploymentTestState.Parameters.deployBastion.value | Should -BeTrue $deploymentTestState.Parameters.tags.value.SecurityControl | Should -Be 'Ignore' $deploymentTestState.Parameters.tags.value.CostControl | Should -Be 'Ignore' + $deploymentTestState.Parameters.tags.value.'microhack-startup-version' | Should -Be '1' + $deploymentTestState.Parameters.tags.value.'microhack-startup-id' | Should -Match '^[a-f0-9-]{36}$' + $deploymentTestState.HostSource | Should -Match 'customData:' $deploymentTestState.NetworkSource | Should -Be (Get-Content "$root/labautomation/localbox-network.bicep" -Raw) $deploymentTestState.Downloads.Count | Should -Be 5 @($deploymentTestState.Downloads | Where-Object { $_ -like '*/network/network.bicep' }).Count | Should -Be 0 diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-startup.tests.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-startup.tests.ps1 new file mode 100644 index 000000000..3f5f8b0b7 --- /dev/null +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-startup.tests.ps1 @@ -0,0 +1,382 @@ +BeforeAll { + $root = (Resolve-Path "$PSScriptRoot/../..").Path + . "$root/labautomation/localbox-startup.ps1" + . "$root/labautomation/localbox-startup-watch.ps1" + . "$root/labautomation/localbox-automation.ps1" + $id = '10000000-0000-0000-0000-000000000001' + $subscription = '20000000-0000-0000-0000-000000000001' + function Update-MhhToken {} + function Get-ScheduledTask { param($TaskName) } + $imageScript = @' +Write-Host 'before' +# Download LocalBox VHDs +#azcopy cp 'https://ignored.example/old.vhdx' "$($LocalBoxConfig.Paths.VHDDir)\AzL-node.vhdx" +azcopy cp 'https://azlocalvhds.blob.core.windows.net/images/AzLocal2608.vhdx' "$($LocalBoxConfig.Paths.VHDDir)\AzL-node.vhdx" --recursive=true +azcopy cp 'https://azlocalvhds.blob.core.windows.net/images/AzLocal2608.sha256' "$($LocalBoxConfig.Paths.VHDDir)\AzL-node.sha256" --recursive=true +azcopy cp https://jumpstartprodsg.blob.core.windows.net/hcibox23h2/WinServerApril2024.vhdx "$($LocalBoxConfig.Paths.VHDDir)\GUI.vhdx" --recursive=true +azcopy cp https://jumpstartprodsg.blob.core.windows.net/hcibox23h2/WinServerApril2024.sha256 "$($LocalBoxConfig.Paths.VHDDir)\GUI.sha256" --recursive=true +# Set credentials +Write-Host 'after' +'@ +} + +Describe 'Future-only bootstrap installation' { + BeforeEach { + $hostPath = "$TestDrive/host.bicep" + @' +resource vm 'Microsoft.Compute/virtualMachines@2024-07-01' = { + properties: { + osProfile: { + computerName: 'LocalBox-Client' + } + } +} +var command = { + commandToExecute: 'powershell.exe -ExecutionPolicy Bypass -File Bootstrap.ps1 -adminPassword opaque' +} +'@ | Set-Content $hostPath + } + It 'embeds bounded custom data only for a new VM and preserves protected arguments' { + Set-LocalBoxStartupTemplate $hostPath "$root/labautomation/localbox-startup.ps1" $id -NewVm + $text = Get-Content $hostPath -Raw + $text | Should -Match "customData: base64\(loadTextContent\('localbox-startup.ps1'\)\)" + $text | Should -Match '&& powershell.exe.*-Install && powershell.exe.*-adminPassword opaque' + Get-Content "$TestDrive/localbox-startup.ps1" -Raw | Should -Match $id + Get-Content "$TestDrive/localbox-startup.ps1" -Raw | Should -Not -Match '__LOCALBOX_BOOTSTRAP_ID__' + } + It 'does not replace immutable custom data for an already guarded VM' { + Set-LocalBoxStartupTemplate $hostPath "$root/labautomation/localbox-startup.ps1" $id + Get-Content $hostPath -Raw | Should -Not -Match 'customData:' + } + It 'rejects an incompatible upstream template without silently removing protection' { + Add-Content $hostPath 'customData: other' + { Set-LocalBoxStartupTemplate $hostPath "$root/labautomation/localbox-startup.ps1" $id -NewVm } | Should -Throw '*Unsupported*' + } + It 'rejects custom data larger than the API limit' { + ('x' * 65536) | Set-Content "$TestDrive/oversized.ps1" + { Set-LocalBoxStartupTemplate $hostPath "$TestDrive/oversized.ps1" $id -NewVm } | Should -Throw '*custom-data limit*' + } + It 'wraps the existing startup task and protects the destination directory ACL' { + '$Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument $LocalBoxPath\LocalBoxLogonScript.ps1' | + Set-Content "$TestDrive/Bootstrap.ps1" + Install-LocalBoxStartupGuard "$TestDrive/Bootstrap.ps1" + $text = Get-Content "$TestDrive/Bootstrap.ps1" -Raw + $text | Should -Match 'Set-Acl' + $text | Should -Match '\$guardRoot\\startup.ps1' + { Install-LocalBoxStartupGuard "$TestDrive/Bootstrap.ps1" } | Should -Throw '*Unsupported*' + } +} + +Describe 'Upstream image download patch' { + BeforeEach { $imageScript | Set-Content "$TestDrive/cluster.ps1" } + It 'captures only active trusted sources and inserts a terminating image gate' { + Set-LocalBoxBaseImageGuard "$TestDrive/cluster.ps1" "$TestDrive/images.json" $id + $plan = Get-Content "$TestDrive/images.json" -Raw | ConvertFrom-Json -AsHashtable + $plan.Artifacts.Count | Should -Be 4 + $plan.Artifacts['AzL-node.vhdx'] | Should -Be 'https://azlocalvhds.blob.core.windows.net/images/AzLocal2608.vhdx' + $text = Get-Content "$TestDrive/cluster.ps1" -Raw + $text | Should -Match '-DownloadImages' + $text | Should -Match 'if \(-not \$\?\) \{ throw' + $text | Should -Match "Write-Host 'after'" + $text | Should -Not -Match '(?m)^azcopy' + } + It 'rejects untrusted or incomplete upstream sources' { + $imageScript.Replace('azlocalvhds.blob.core.windows.net', 'untrusted.example') | Set-Content "$TestDrive/cluster.ps1" + { Set-LocalBoxBaseImageGuard "$TestDrive/cluster.ps1" "$TestDrive/images.json" $id } | Should -Throw '*Unrecognized*' + $imageScript.Replace('AzL-node.sha256', 'unexpected.sha256') | Set-Content "$TestDrive/cluster.ps1" + { Set-LocalBoxBaseImageGuard "$TestDrive/cluster.ps1" "$TestDrive/images.json" $id } | Should -Throw + } +} + +Describe 'Bounded verified base-image staging' { + BeforeEach { + $directory = Join-Path $TestDrive ([guid]::NewGuid().ToString()) + New-Item -ItemType Directory -Path $directory | Out-Null + $statusPath = "$directory/status.json" + $status = @{ BootstrapId = $id; State = 'Running'; Phase = 'Startup' } + $artifacts = @{} + foreach ($name in @('AzL-node', 'GUI')) { + $artifacts["$name.vhdx"] = "https://azlocalvhds.blob.core.windows.net/images/$name.vhdx" + $artifacts["$name.sha256"] = "https://azlocalvhds.blob.core.windows.net/images/$name.sha256" + } + Set-Content "$directory/fixture" 'fixture-image' + $digest = (Get-FileHash "$directory/fixture").Hash + Mock Start-Sleep {} + Mock Invoke-LocalBoxBaseImageTransfer { + if ($Destination.EndsWith('.sha256')) { Set-Content $Destination $digest } + else { Set-Content $Destination 'fixture-image' } + } + $arguments = @{ Artifacts = $artifacts; Directory = $directory; Status = $status; StatusPath = $statusPath; RetryDelaySeconds = 0 } + } + It 'never considers two absent files a valid checksum' { + Test-LocalBoxBaseImage "$directory/missing.vhdx" "$directory/missing.sha256" | Should -BeFalse + } + It 'rejects an empty checksum without skipping download recovery' { + Set-Content "$directory/AzL-node.vhdx" 'fixture-image' + Set-Content "$directory/AzL-node.sha256" '' + Test-LocalBoxBaseImage "$directory/AzL-node.vhdx" "$directory/AzL-node.sha256" | Should -BeFalse + Sync-LocalBoxBaseImages @arguments + Should -Invoke Invoke-LocalBoxBaseImageTransfer -Times 4 -Exactly + } + It 'rejects a corrupt source plan before making any network request' { + $artifacts['GUI.vhdx'] = 'https://untrusted.example/image' + { Sync-LocalBoxBaseImages @arguments } | Should -Throw '*Invalid base-image download plan*' + Should -Invoke Invoke-LocalBoxBaseImageTransfer -Times 0 + } + It 'downloads and verifies both images before reporting nested deployment may proceed' { + Sync-LocalBoxBaseImages @arguments + Should -Invoke Invoke-LocalBoxBaseImageTransfer -Times 4 -Exactly -ParameterFilter { $TimeoutSeconds -gt 0 -and $TimeoutSeconds -le 1800 } + Test-LocalBoxBaseImage "$directory/AzL-node.vhdx" "$directory/AzL-node.sha256" | Should -BeTrue + $status.Phase | Should -Be 'NestedDeployment' + @(Get-ChildItem $directory -Directory -Force).Count | Should -Be 0 + } + It 'reuses valid cached images without transferring or changing them' { + foreach ($name in @('AzL-node', 'GUI')) { + Set-Content "$directory/$name.vhdx" 'fixture-image' + Set-Content "$directory/$name.sha256" $digest + } + Sync-LocalBoxBaseImages @arguments + Should -Invoke Invoke-LocalBoxBaseImageTransfer -Times 0 + } + It 'retries a transient 401 within the fixed attempt count' { + $transferCounter = @{ Count = 0 } + Mock Invoke-LocalBoxBaseImageTransfer { + $transferCounter.Count++ + if ($transferCounter.Count -eq 1) { throw 'AzCopy failed with exit code 1. HTTP 401.' } + if ($Destination.EndsWith('.sha256')) { Set-Content $Destination $digest } + else { Set-Content $Destination 'fixture-image' } + } + Sync-LocalBoxBaseImages @arguments + Should -Invoke Invoke-LocalBoxBaseImageTransfer -Times 5 -Exactly + Should -Invoke Start-Sleep -Times 1 -Exactly + } + It 'fails after three denied downloads, preserving existing cache and an actionable report' { + Set-Content "$directory/AzL-node.vhdx" 'old-incomplete-image' + Mock Invoke-LocalBoxBaseImageTransfer { throw 'AzCopy failed with exit code 1. HTTP 401.' } + { Sync-LocalBoxBaseImages @arguments } | Should -Throw '*attempt 3/3*HTTP 401*' + Should -Invoke Invoke-LocalBoxBaseImageTransfer -Times 3 -Exactly + Should -Invoke Start-Sleep -Times 2 -Exactly + Get-Content "$directory/AzL-node.vhdx" | Should -Be 'old-incomplete-image' + $report = Get-Content $statusPath -Raw | ConvertFrom-Json + $report.State | Should -Be 'Failed' + $report.Code | Should -Be 'BaseImageDownloadFailed' + @(Get-ChildItem $directory -Directory -Force).Count | Should -Be 0 + } + It 'rejects malformed checksums even when transfers report success' { + Mock Invoke-LocalBoxBaseImageTransfer { Set-Content $Destination 'not-a-checksum' } + { Sync-LocalBoxBaseImages @arguments } | Should -Throw '*malformed*' + Test-Path "$directory/AzL-node.vhdx" | Should -BeFalse + $status.State | Should -Be 'Failed' + } + It 'does not promote a hash-mismatched image' { + Mock Invoke-LocalBoxBaseImageTransfer { + if ($Destination.EndsWith('.sha256')) { Set-Content $Destination ('0' * 64) } + else { Set-Content $Destination 'fixture-image' } + } + { Sync-LocalBoxBaseImages @arguments } | Should -Throw '*does not match*' + Should -Invoke Invoke-LocalBoxBaseImageTransfer -Times 6 -Exactly + Test-Path "$directory/AzL-node.vhdx" | Should -BeFalse + } + It 'does not create another attempt when the remaining budget cannot cover the retry delay' { + $arguments.TimeoutSeconds = 30 + $arguments.RetryDelaySeconds = 60 + Mock Invoke-LocalBoxBaseImageTransfer { throw 'Download failed' } + { Sync-LocalBoxBaseImages @arguments } | Should -Throw + Should -Invoke Start-Sleep -Times 0 + Should -Invoke Invoke-LocalBoxBaseImageTransfer -Times 1 -Exactly + } + +} + +Describe 'Guarded startup replay protection' { + BeforeEach { + $startupRoot = Join-Path $TestDrive ([guid]::NewGuid().ToString()) + New-Item -ItemType Directory -Path "$startupRoot/SovereignStartup" -Force | Out-Null + $stateFile = "$startupRoot/SovereignStartup/state.json" + Mock Set-LocalBoxBaseImageGuard {} + } + It 'does not repeat completed initialization' { + @{ BootstrapId = $id; State = 'Succeeded' } | ConvertTo-Json | Set-Content $stateFile + Invoke-LocalBoxGuardedStartup $id $startupRoot + Should -Invoke Set-LocalBoxBaseImageGuard -Times 0 + } + It 'blocks incomplete or failed initialization without rerunning storage' -TestCases @( + @{ PreviousState = 'Running' }, @{ PreviousState = 'Failed' } + ) { + param($PreviousState) + @{ BootstrapId = $id; State = $PreviousState } | ConvertTo-Json | Set-Content $stateFile + { Invoke-LocalBoxGuardedStartup $id $startupRoot } | Should -Throw '*Automatic replay*disabled*' + Should -Invoke Set-LocalBoxBaseImageGuard -Times 0 + } + It 'fails closed on corrupt previous state' { + Set-Content $stateFile 'not json' + { Invoke-LocalBoxGuardedStartup $id $startupRoot } | Should -Throw + Should -Invoke Set-LocalBoxBaseImageGuard -Times 0 + } + It 'preserves the primary error if recording its report also fails' { + Mock Save-LocalBoxStartupStatus { throw 'fixture-state-write-denied' } + { Invoke-LocalBoxGuardedStartup $id $startupRoot } | Should -Throw '*fixture-state-write-denied*' + Should -Invoke Set-LocalBoxBaseImageGuard -Times 0 + } + It 'records an upstream failure and prevents a second invocation' { + Set-Content "$startupRoot/LocalBoxLogonScript.ps1" 'throw "fixture-upstream-failed"' + { Invoke-LocalBoxGuardedStartup $id $startupRoot } | Should -Throw '*fixture-upstream-failed*' + (Get-Content $stateFile -Raw | ConvertFrom-Json).Code | Should -Be 'StartupFailed' + { Invoke-LocalBoxGuardedStartup $id $startupRoot } | Should -Throw '*Automatic replay*disabled*' + Should -Invoke Set-LocalBoxBaseImageGuard -Times 1 -Exactly + } + It 'records successful script completion without claiming infrastructure readiness' { + Set-Content "$startupRoot/LocalBoxLogonScript.ps1" 'Write-Output "fixture-script-completed"' + Invoke-LocalBoxGuardedStartup $id $startupRoot | Out-Null + $saved = Get-Content $stateFile -Raw | ConvertFrom-Json + $saved.State | Should -Be 'Succeeded' + $saved.Message | Should -Match 'Full health gates are still required' + } + } + + Describe 'Native transfer outcome checks' { + BeforeEach { + $processState = @{ Finished = $true; Exit = 0; Killed = $false; Disposed = $false } + $fakeProcess = [pscustomobject]@{ HasExited = $true; ExitCode = 0 } + $fakeProcess | Add-Member ScriptMethod WaitForExit { param($milliseconds) return $processState.Finished } + $fakeProcess | Add-Member ScriptMethod Kill { param($tree) $processState.Killed = $true; $this.HasExited = $true } + $fakeProcess | Add-Member ScriptMethod Dispose { $processState.Disposed = $true } + Mock Get-Command { @{ Path = 'fixture-azcopy' } } -ParameterFilter { $Name -eq 'azcopy' } + Mock Start-Process { $fakeProcess } + $transferPath = Join-Path $TestDrive ([guid]::NewGuid().ToString()) + New-Item -ItemType Directory -Path $transferPath | Out-Null + $transferArgs = @{ Uri = 'https://azlocalvhds.blob.core.windows.net/image'; Destination = "$transferPath/image"; LogDirectory = $transferPath; TimeoutSeconds = 1 } + $originalLogs = $env:AZCOPY_LOG_LOCATION + $originalPlans = $env:AZCOPY_JOB_PLAN_LOCATION + } + AfterEach { + $env:AZCOPY_LOG_LOCATION | Should -Be $originalLogs + $env:AZCOPY_JOB_PLAN_LOCATION | Should -Be $originalPlans + } + It 'rejects a zero exit without an output file' { + { Invoke-LocalBoxBaseImageTransfer @transferArgs } | Should -Throw '*success without creating*' + $processState.Disposed | Should -BeTrue + } + It 'extracts only HTTP status codes instead of raw native error output' { + $fakeProcess.ExitCode = 1 + Set-Content "$transferPath/error.log" 'RESPONSE 401 secret-response-fixture' + { Invoke-LocalBoxBaseImageTransfer @transferArgs } | Should -Throw '*exit code 1. HTTP 401.*' + try { Invoke-LocalBoxBaseImageTransfer @transferArgs } catch { $_.Exception.Message | Should -Not -Match 'secret-response-fixture' } + } + It 'terminates only its own timed-out process and restores the environment' { + $processState.Finished = $false + $fakeProcess.HasExited = $false + { Invoke-LocalBoxBaseImageTransfer @transferArgs } | Should -Throw + $processState.Killed | Should -BeTrue + $processState.Disposed | Should -BeTrue + } + It 'restores the environment even when process termination fails' { + $processState.Finished = $false + $fakeProcess.HasExited = $false + $fakeProcess | Add-Member ScriptMethod Kill { param($tree) throw 'fixture-kill-denied' } -Force + { Invoke-LocalBoxBaseImageTransfer @transferArgs } | Should -Throw '*fixture-kill-denied*' + $processState.Disposed | Should -BeTrue + } + } + + Describe 'Console startup correlation' { + BeforeEach { + $tracking = @{ Checked = $false; Id = $null } + $monitorState = @{ Vm = @{ tags = @{ 'microhack-startup-version' = '1'; 'microhack-startup-id' = $id }; location = 'spaincentral' } } + $report = @{ BootstrapId = $id; SubscriptionId = $subscription; ResourceGroupName = 'group'; State = 'Failed'; Phase = 'BaseImages'; Code = 'BaseImageDownloadFailed'; Message = 'AzCopy HTTP 401' } + $monitorState.Command = @{ properties = @{ + provisioningState = 'Succeeded' + parameters = @( + @{ name = 'BootstrapId'; value = $id } + @{ name = 'SubscriptionId'; value = $subscription } + @{ name = 'ResourceGroupName'; value = 'group' } + @{ name = 'StartedAfter'; value = '2026-11-05T00:00:00Z' } + ) + instanceView = @{ startTime = '2026-11-05T00:00:01Z'; executionState = 'Failed'; exitCode = 1; output = 'LOCALBOX_STARTUP_REPORT:' + ($report | ConvertTo-Json -Compress) } + } } + Mock Invoke-LocalBoxAutomationRequest { + if ($Path -notmatch '/runCommands/') { return $monitorState.Vm } + return $monitorState.Command + } + } + It 'does not create a startup monitor for a legacy VM' { + $monitorState.Vm.tags = @{} + Get-LocalBoxStartupProgress $tracking $subscription group 100 + Should -Invoke Invoke-LocalBoxAutomationRequest -Times 0 -ParameterFilter { $Method -eq 'PUT' -or $Path -match '/runCommands/' } + } + It 'surfaces the correlated download failure immediately' { + { Get-LocalBoxStartupProgress $tracking $subscription group 100 } | Should -Throw '*BaseImages*BaseImageDownloadFailed*HTTP 401*' + } + It 'never overwrites a foreign command' { + $monitorState.Command.properties.parameters[0].value = 'foreign' + { Get-LocalBoxStartupProgress $tracking $subscription group 100 } | Should -Throw '*not replaced*' + Should -Invoke Invoke-LocalBoxAutomationRequest -Times 0 -ParameterFilter { $Method -eq 'PUT' } + } + It 'does not accept stale execution output' { + $monitorState.Command.properties.instanceView.startTime = '2026-11-04T00:00:01Z' + Get-LocalBoxStartupProgress $tracking $subscription group 100 + } + It 'rejects terminal output without a verified report' { + $monitorState.Command.properties.instanceView.output = 'untrusted raw stdout' + { Get-LocalBoxStartupProgress $tracking $subscription group 100 } | Should -Throw '*without a verified report*' + } + It 'starts one asynchronous scoped monitor when none exists' { + $monitorState.Command.properties.instanceView.executionState = 'Running' + Mock Invoke-LocalBoxAutomationRequest { return $null } -ParameterFilter { $AllowNotFound } + Get-LocalBoxStartupProgress $tracking $subscription group 100 + Get-LocalBoxStartupProgress $tracking $subscription group 90 + Should -Invoke Invoke-LocalBoxAutomationRequest -Times 1 -Exactly -ParameterFilter { + $Method -eq 'PUT' -and $Body.properties.asyncExecution -and $Body.properties.timeoutInSeconds -eq 100 -and + ($Body.properties.parameters | Where-Object name -eq BootstrapId).value -eq $id + } + } + } +Describe 'Startup report validation' { + It 'rejects reports for another bootstrap or subscription' { + $path = "$TestDrive/state.json" + @{ BootstrapId = 'foreign'; SubscriptionId = $subscription; ResourceGroupName = 'group'; State = 'Failed'; Phase = 'BaseImages' } | + ConvertTo-Json | Set-Content $path + { Get-LocalBoxStartupReport $path $id $subscription group } | Should -Throw '*does not match*' + } + It 'rejects corrupt JSON and invalid states' { + $path = "$TestDrive/state.json" + Set-Content $path 'not-json' + { Get-LocalBoxStartupReport $path $id $subscription group } | Should -Throw '*read or parsed*' + @{ BootstrapId = $id; SubscriptionId = $subscription; ResourceGroupName = 'group'; State = 'Unknown'; Phase = 'BaseImages' } | + ConvertTo-Json | Set-Content $path + { Get-LocalBoxStartupReport $path $id $subscription group } | Should -Throw '*invalid state metadata*' + } +} + +Describe 'Startup report monitor thresholds' { + BeforeEach { + $clock = @{ Now = [datetime]'2026-11-05T00:00:00Z' } + Mock Get-Date { $clock.Now } + Mock Start-Sleep { $clock.Now = $clock.Now.AddHours(1) } + Mock Get-LocalBoxStartupReport { $null } + Mock Get-ScheduledTask { @{ State = 'Ready' } } + } + It 'fails at the one-hour missing-report threshold instead of waiting eight hours' { + $report = Wait-LocalBoxStartupReport $id $subscription group 28800 + $report.Code | Should -Be 'StartupNotStarted' + Should -Invoke Start-Sleep -Times 1 -Exactly + ($clock.Now - [datetime]'2026-11-05T00:00:00Z').TotalSeconds | Should -Be 3600 + } + It 'detects a stopped startup task only after the two-minute startup grace' { + Mock Get-LocalBoxStartupReport { @{ State = 'Running' } } + Mock Start-Sleep { $clock.Now = $clock.Now.AddSeconds(120) } + (Wait-LocalBoxStartupReport $id $subscription group 28800).Code | Should -Be 'StartupExited' + Should -Invoke Get-ScheduledTask -Times 1 -Exactly + ($clock.Now - [datetime]'2026-11-05T00:00:00Z').TotalSeconds | Should -Be 120 + } + It 'returns reported failure immediately without waiting or inspecting tasks' { + Mock Get-LocalBoxStartupReport { @{ State = 'Failed'; Code = 'BaseImageDownloadFailed' } } + (Wait-LocalBoxStartupReport $id $subscription group 28800).Code | Should -Be 'BaseImageDownloadFailed' + Should -Invoke Start-Sleep -Times 0 + Should -Invoke Get-ScheduledTask -Times 0 + } + It 'does not treat a monitoring exception as startup success' { + Mock Get-LocalBoxStartupReport { throw 'Invalid metadata' } + (Wait-LocalBoxStartupReport $id $subscription group 28800).State | Should -Be 'Failed' + } +} diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md index ddcc457ee..1d8dd85dc 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md @@ -58,6 +58,18 @@ required to validate regional service availability and effective inherited polic ## Global event LocalBox checks +Run `Invoke-Pester ./localbox-startup.tests.ps1,./localbox-deployment.tests.ps1 -Output Detailed` +for future-only startup protection. Offline checks cover new/existing VM +custom-data handling, immutable identity retention, strict upstream source +recognition, checksum/cache validation, bounded retries, native exit/timeout +handling, cleanup, persistent replay prevention, Console report correlation and +one-hour/two-minute monitoring thresholds. Mocked transfers never download VHDs +or restart a live VM. The modified real upstream Bicep was compiled, and both +upstream script transformations were syntax-checked offline. A new Windows +deployment is still required to verify custom-data delivery, scheduled-task +execution and end-to-end Console reporting together; existing failed instances +are deliberately not retrofitted. + Run `Invoke-Pester ./localbox-failure-summary.tests.ps1,./localbox-automation.tests.ps1 -Output Detailed` for Console failure reporting. Tests cover run/content/group/scope correlation, real NUnit failure counts and messages, missing/malformed reports, prohibited XML From 81cd5c693e4d1696c915407cb3e504837ff1d1a9 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Fri, 6 Nov 2026 07:36:22 +0000 Subject: [PATCH 18/37] Wait for LocalBox prerequisites and preserve startup identity Prevent WinGet/download startup races, resolve installed AzCopy despite a stale PATH, and retain startup identity through upstream progress tag replacement. Reject existing monitored VMs with missing identity rather than replaying bootstrap. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../labautomation/deploy-localbox.ps1 | 15 +++ .../labautomation/localbox-automation.ps1 | 6 +- .../labautomation/localbox-startup-watch.ps1 | 2 +- .../labautomation/localbox-startup.ps1 | 66 +++++++++- .../resources/hosted-events/readme.md | 24 ++++ .../tests/localbox-deployment.tests.ps1 | 19 +++ .../tests/localbox-startup.tests.ps1 | 115 +++++++++++++++++- .../resources/tests/readme.md | 13 ++ 8 files changed, 253 insertions(+), 7 deletions(-) diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/deploy-localbox.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/deploy-localbox.ps1 index 7972f6289..559565c15 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/deploy-localbox.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/deploy-localbox.ps1 @@ -354,6 +354,13 @@ try { Write-Host 'Using correlated startup protection with bounded base-image downloads. Existing VM custom data is never replaced.' } else { + $existingMonitors = @(Invoke-AzJson -Arguments @( + 'vm', 'run-command', 'list', '--subscription', $SubscriptionId, + '--resource-group', $ResourceGroupName, '--vm-name', 'LocalBox-Client' + ) | Where-Object name -eq 'sovereign-localbox-startup-watch') + if ($existingMonitors.Count) { + throw 'The existing Client has a startup monitor but no startup identity tags. Organizer review is required; no bootstrap migration was attempted.' + } Write-Warning 'This existing Client predates startup protection. Its bootstrap is left unchanged; use a new VM to enable guarded startup.' } @@ -390,6 +397,14 @@ try { '--template-file', $templateFile, '--parameters', "@$parametersFile" ) | Out-Null + if ($startupId -ne [guid]::Empty) { + # Upstream progress updates replace VM tags with the resource group's tags. + Invoke-AzJson -Arguments @( + 'group', 'update', '--subscription', $SubscriptionId, '--name', $ResourceGroupName, + '--set', 'tags.microhack-startup-version=1', "tags.microhack-startup-id=$startupId" + ) | Out-Null + } + Write-Host "Starting LocalBox deployment '$deploymentName'..." -ForegroundColor Cyan $deploymentArguments = @( 'deployment', 'group', 'create', '--subscription', $SubscriptionId, diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation.ps1 index f905c6e31..763aa2a08 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation.ps1 @@ -97,7 +97,11 @@ function Get-LocalBoxStartupProgress { if (-not $Tracking.Checked) { $vm = Invoke-LocalBoxAutomationRequest -Path "$vmId`?api-version=2024-07-01" $Tracking.Checked = $true - if (-not $vm.tags.'microhack-startup-version' -and -not $vm.tags.'microhack-startup-id') { return } + if (-not $vm.tags.'microhack-startup-version' -and -not $vm.tags.'microhack-startup-id') { + $existing = Invoke-LocalBoxAutomationRequest -Path "$uri&`$expand=instanceView" -AllowNotFound + if ($existing) { throw 'The Client startup identity tags are missing but a startup monitor exists. Organizer review is required; do not rerun bootstrap or assume this is an unprotected legacy VM.' } + return + } $id = [guid]::Empty if ($vm.tags.'microhack-startup-version' -ne '1' -or -not [guid]::TryParse([string]$vm.tags.'microhack-startup-id', [ref]$id) -or $id -eq [guid]::Empty) { diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-startup-watch.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-startup-watch.ps1 index 5ce55d41e..52b428220 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-startup-watch.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-startup-watch.ps1 @@ -11,7 +11,7 @@ function Get-LocalBoxStartupReport { catch { throw 'The upstream startup report could not be read or parsed.' } if ($status.BootstrapId -ne $Id -or $status.SubscriptionId -ne $Subscription -or $status.ResourceGroupName -ne $Group -or $status.State -notin @('Running', 'Succeeded', 'Failed') -or - $status.Phase -notin @('Startup', 'BaseImages', 'NestedDeployment', 'Complete')) { + $status.Phase -notin @('Prerequisites', 'Startup', 'BaseImages', 'NestedDeployment', 'Complete')) { throw 'The upstream startup report does not match this VM or has invalid state metadata.' } @{ diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-startup.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-startup.ps1 index aad6aa656..cd6ee2e97 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-startup.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-startup.ps1 @@ -101,9 +101,60 @@ function Test-LocalBoxBaseImage { return (Get-FileHash -LiteralPath $Image -Algorithm SHA256 -ErrorAction Stop).Hash -eq $expected } +function Resolve-LocalBoxAzCopy { + param([string]$LocalAppData = $env:LOCALAPPDATA, [string]$ProgramFiles = $env:ProgramFiles) + $client = @(Get-Command azcopy -CommandType Application -ErrorAction SilentlyContinue) + if ($client.Count) { return $client[0].Path } + $links = @( + if ($LocalAppData) { Join-Path $LocalAppData 'Microsoft/WinGet/Links/azcopy.exe' } + if ($ProgramFiles) { + Join-Path $ProgramFiles 'WinGet/Links/azcopy.exe' + Join-Path $ProgramFiles 'AzCopy/azcopy.exe' + } + ) + foreach ($path in $links) { + if (Test-Path -LiteralPath $path -PathType Leaf -ErrorAction Stop) { return $path } + } + if ($LocalAppData) { + $pattern = Join-Path $LocalAppData 'Microsoft/WinGet/Packages/Microsoft.Azure.AZCopy.10_*/azcopy_windows_*/azcopy.exe' + if (Test-Path -Path $pattern -PathType Leaf -ErrorAction Stop) { + $candidates = @(Get-ChildItem -Path $pattern -File -ErrorAction Stop) + if ($candidates.Count -ne 1) { throw 'Multiple unlinked AzCopy installations exist for the startup user; repair the WinGet link before retrying.' } + return $candidates[0].FullName + } + } + throw 'AzCopy is unavailable after prerequisite installation. Inspect the WinGet provisioning log and the startup user WinGet installation; no image transfer was attempted.' +} + +function Wait-LocalBoxStartupPrerequisites { + param([hashtable]$Status, [string]$StatusPath, [ValidateRange(1, 3600)][int]$TimeoutSeconds = 3600) + $ErrorActionPreference = 'Stop' + $deadline = (Get-Date).ToUniversalTime().AddSeconds($TimeoutSeconds) + $Status.Phase = 'Prerequisites' + $Status.Message = 'Waiting for WinGetLogonScript to finish before initializing storage or downloading images.' + Save-LocalBoxStartupStatus $Status $StatusPath + Write-Host $Status.Message + do { + $tasks = @(Get-ScheduledTask -ErrorAction Stop | Where-Object { $_.TaskName -eq 'WinGetLogonScript' -and $_.TaskPath -eq '\' }) + if ($tasks.Count -gt 1) { throw 'Ambiguous WinGet prerequisite task state.' } + if (-not $tasks.Count) { break } + if ($tasks[0].State -eq 'Disabled') { throw 'WinGetLogonScript is disabled; prerequisite installation cannot be verified.' } + if ((Get-Date).ToUniversalTime() -ge $deadline) { + throw 'WinGetLogonScript did not complete within the one-hour prerequisite budget. Inspect its provisioning log; storage initialization was not started.' + } + Start-Sleep -Seconds 15 + } while ($true) + $env:PATH = (@($env:PATH, [Environment]::GetEnvironmentVariable('PATH', 'Machine'), + [Environment]::GetEnvironmentVariable('PATH', 'User')) | Where-Object { $_ } | Select-Object -Unique) -join [IO.Path]::PathSeparator + $null = Resolve-LocalBoxAzCopy + $Status.Phase = 'Startup' + $Status.Message = 'WinGet prerequisites completed and AzCopy resolved; upstream initialization may proceed.' + Save-LocalBoxStartupStatus $Status $StatusPath +} + function Invoke-LocalBoxBaseImageTransfer { param([string]$Uri, [string]$Destination, [string]$LogDirectory, [int]$TimeoutSeconds) - $client = Get-Command azcopy -CommandType Application -ErrorAction Stop + $client = Resolve-LocalBoxAzCopy $stdout = Join-Path $LogDirectory 'stdout.log' $stderr = Join-Path $LogDirectory 'stderr.log' $oldLogs = $env:AZCOPY_LOG_LOCATION @@ -112,7 +163,7 @@ function Invoke-LocalBoxBaseImageTransfer { try { $env:AZCOPY_LOG_LOCATION = $LogDirectory $env:AZCOPY_JOB_PLAN_LOCATION = $LogDirectory - $process = Start-Process -FilePath $client.Path -ArgumentList @('copy', "`"$Uri`"", "`"$Destination`"", + $process = Start-Process -FilePath $client -ArgumentList @('copy', "`"$Uri`"", "`"$Destination`"", '--recursive=true', '--check-length=false', '--log-level=ERROR') -PassThru -NoNewWindow ` -RedirectStandardOutput $stdout -RedirectStandardError $stderr -ErrorAction Stop if (-not $process.WaitForExit($TimeoutSeconds * 1000)) { @@ -248,6 +299,7 @@ function Invoke-LocalBoxGuardedStartup { StartedAt = [datetime]::UtcNow.ToString('o') } Save-LocalBoxStartupStatus $status $statusPath + Wait-LocalBoxStartupPrerequisites -Status $status -StatusPath $statusPath Set-LocalBoxBaseImageGuard -Path "$Root/New-LocalBoxCluster.ps1" -PlanPath "$Root/SovereignStartup/images.json" -Id $Id & "$Root/LocalBoxLogonScript.ps1" if (-not $?) { throw 'Upstream LocalBox startup returned unsuccessfully.' } @@ -265,8 +317,14 @@ function Invoke-LocalBoxGuardedStartup { $latest = Get-Content -LiteralPath $statusPath -Raw | ConvertFrom-Json -AsHashtable -ErrorAction Stop if ($latest.State -ne 'Failed') { $latest.State = 'Failed' - $latest.Code = 'StartupFailed' - $latest.Message = 'Upstream LocalBox startup failed. Inspect LocalBoxLogonScript.log and New-LocalBoxCluster.log; do not blindly rerun storage initialization.' + if ($latest.Phase -eq 'Prerequisites') { + $latest.Code = 'StartupPrerequisitesFailed' + $latest.Message = 'WinGet prerequisite readiness or AzCopy discovery failed before storage initialization. Inspect WinGet-provisioning logs and WinGetLogonScript task state.' + } + else { + $latest.Code = 'StartupFailed' + $latest.Message = 'Upstream LocalBox startup failed. Inspect LocalBoxLogonScript.log and New-LocalBoxCluster.log; do not blindly rerun storage initialization.' + } Save-LocalBoxStartupStatus $latest $statusPath } } diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md index ecfe145bb..e6d8bb8c6 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md @@ -312,6 +312,22 @@ New Client VMs receive a deployment-specific startup guard through immutable VM custom data. Existing unprotected VMs are **not migrated or repaired**; existing guarded VMs keep their original payload and startup identity. +Before entering upstream storage initialization, the guard waits for +`WinGetLogonScript` to finish and unregister itself (at most **one hour**). +The upstream WinGet and LocalBox tasks can both start at logon; starting downloads +while WinGet is still installing AzCopy is not safe. After installation, the +guard refreshes its process PATH and resolves AzCopy from PATH, the current +startup user's WinGet link/package, or known machine installation paths. It does +not search other users' profiles, install a replacement binary, or blindly pick +between multiple unlinked package versions. Prerequisite failure is reported as +`Prerequisites` / `StartupPrerequisitesFailed`, before storage initialization. + +The startup identity is also merged into the resource-group tags before deployment: +upstream progress updates replace VM tags with the resource-group tags. If an +existing Client has a startup monitor but has lost its identity tags, deployment +and monitoring stop for organizer review rather than silently treating it as a +legacy VM. No automatic repair of old guarded instances is performed. + The guard replaces the recognized upstream base-VHD download/checksum block: - Each base image gets at most **three attempts**, with **60 seconds** between @@ -343,6 +359,14 @@ nested-readiness budget. Offline tests and compilation against the real upstream templates/scripts validate this integration; a fresh Windows/Console deployment is still required to verify the complete new-VM path. +TEST10 on traininglab-17 demonstrated the early failure path: Console stopped +after **25m25s**, reporting missing AzCopy. The stored monitor report ended at +**20:59:09 UTC on 5 November 2026**; the WinGet AzCopy link was installed at +**21:00:18 UTC**, and WinGet finished at **21:03:59 UTC**. This exposed the +installation race addressed by the prerequisite gate; it was not another HTTP +401 failure. Read-only inspection also confirmed the upstream tag replacement. +Neither bootstrap nor storage initialization was replayed during diagnosis. + #### Automatic Azure Local validation recovery The Console can perform **one automatic validation retry**, after a **two-minute diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-deployment.tests.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-deployment.tests.ps1 index 083d264ab..77f239863 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-deployment.tests.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-deployment.tests.ps1 @@ -57,6 +57,8 @@ Describe 'LocalBox deployment entry points' { Bastions = @() Networks = @() Clients = @() + Monitors = @() + StartupTagUpdate = @() QueriesFail = $false ValidationFails = $false Submitted = 0 @@ -109,6 +111,12 @@ var settings = { } '^network vnet list' { ConvertTo-Json -InputObject @($deploymentTestState.Networks) -Depth 10; break } '^resource list' { ConvertTo-Json -InputObject @($deploymentTestState.Clients) -Depth 10; break } + '^vm run-command list' { ConvertTo-Json -InputObject @($deploymentTestState.Monitors) -Depth 10; break } + '^group update' { + if ($args -contains 'tags.microhack-startup-version=1') { $deploymentTestState.StartupTagUpdate = @($args) } + '{"name":"rg-localbox-test","location":"spaincentral"}' + break + } '^group (create|show|update)' { '{"name":"rg-localbox-test","location":"spaincentral"}'; break } '^bicep build' { $deploymentTestState.Builds++ @@ -134,6 +142,13 @@ var settings = { & "$fixture/labautomation/deploy-localbox.ps1" @arguments $deploymentTestState.HostSource | Should -Not -Match 'customData|SovereignStartup' $deploymentTestState.Parameters.tags.value.PSObject.Properties.Name | Should -Not -Contain 'microhack-startup-id' + $deploymentTestState.StartupTagUpdate.Count | Should -Be 0 + } + It 'does not redeploy a guarded VM whose progress update removed identity tags' { + $deploymentTestState.Clients = @(@{ name = 'LocalBox-Client'; tags = @{} }) + $deploymentTestState.Monitors = @(@{ name = 'sovereign-localbox-startup-watch' }) + { & "$fixture/labautomation/deploy-localbox.ps1" @arguments } | Should -Throw '*startup monitor but no startup identity tags*' + $deploymentTestState.Submitted | Should -Be 0 } It 'retains the immutable startup identity on a guarded existing VM' { $deploymentTestState.Clients = @(@{ name = 'LocalBox-Client'; tags = @{ @@ -159,6 +174,10 @@ var settings = { $deploymentTestState.Parameters.tags.value.'microhack-startup-version' | Should -Be '1' $deploymentTestState.Parameters.tags.value.'microhack-startup-id' | Should -Match '^[a-f0-9-]{36}$' $deploymentTestState.HostSource | Should -Match 'customData:' + $deploymentTestState.StartupTagUpdate | Should -Contain 'tags.microhack-startup-version=1' + $deploymentTestState.StartupTagUpdate | Should -Contain "tags.microhack-startup-id=$($deploymentTestState.Parameters.tags.value.'microhack-startup-id')" + $deploymentTestState.StartupTagUpdate | Should -Contain '--set' + $deploymentTestState.StartupTagUpdate | Should -Not -Contain '--tags' $deploymentTestState.NetworkSource | Should -Be (Get-Content "$root/labautomation/localbox-network.bicep" -Raw) $deploymentTestState.Downloads.Count | Should -Be 5 @($deploymentTestState.Downloads | Where-Object { $_ -like '*/network/network.bicep' }).Count | Should -Be 0 diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-startup.tests.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-startup.tests.ps1 index 3f5f8b0b7..f343ec09b 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-startup.tests.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-startup.tests.ps1 @@ -194,6 +194,7 @@ Describe 'Guarded startup replay protection' { New-Item -ItemType Directory -Path "$startupRoot/SovereignStartup" -Force | Out-Null $stateFile = "$startupRoot/SovereignStartup/state.json" Mock Set-LocalBoxBaseImageGuard {} + Mock Wait-LocalBoxStartupPrerequisites {} } It 'does not repeat completed initialization' { @{ BootstrapId = $id; State = 'Succeeded' } | ConvertTo-Json | Set-Content $stateFile @@ -225,6 +226,16 @@ Describe 'Guarded startup replay protection' { { Invoke-LocalBoxGuardedStartup $id $startupRoot } | Should -Throw '*Automatic replay*disabled*' Should -Invoke Set-LocalBoxBaseImageGuard -Times 1 -Exactly } + It 'does not enter storage initialization when prerequisites fail' { + Mock Wait-LocalBoxStartupPrerequisites { + $Status.Phase = 'Prerequisites' + Save-LocalBoxStartupStatus $Status $StatusPath + throw 'fixture-prerequisites-incomplete' + } + { Invoke-LocalBoxGuardedStartup $id $startupRoot } | Should -Throw '*fixture-prerequisites-incomplete*' + (Get-Content $stateFile -Raw | ConvertFrom-Json).Code | Should -Be 'StartupPrerequisitesFailed' + Should -Invoke Set-LocalBoxBaseImageGuard -Times 0 + } It 'records successful script completion without claiming infrastructure readiness' { Set-Content "$startupRoot/LocalBoxLogonScript.ps1" 'Write-Output "fixture-script-completed"' Invoke-LocalBoxGuardedStartup $id $startupRoot | Out-Null @@ -301,8 +312,14 @@ Describe 'Guarded startup replay protection' { } It 'does not create a startup monitor for a legacy VM' { $monitorState.Vm.tags = @{} + $monitorState.Command = $null Get-LocalBoxStartupProgress $tracking $subscription group 100 - Should -Invoke Invoke-LocalBoxAutomationRequest -Times 0 -ParameterFilter { $Method -eq 'PUT' -or $Path -match '/runCommands/' } + Should -Invoke Invoke-LocalBoxAutomationRequest -Times 0 -ParameterFilter { $Method -eq 'PUT' } + } + It 'fails explicitly if startup identity was lost after a monitor was created' { + $monitorState.Vm.tags = @{} + { Get-LocalBoxStartupProgress $tracking $subscription group 100 } | Should -Throw '*identity tags are missing*' + Should -Invoke Invoke-LocalBoxAutomationRequest -Times 0 -ParameterFilter { $Method -eq 'PUT' } } It 'surfaces the correlated download failure immediately' { { Get-LocalBoxStartupProgress $tracking $subscription group 100 } | Should -Throw '*BaseImages*BaseImageDownloadFailed*HTTP 401*' @@ -332,12 +349,108 @@ Describe 'Guarded startup replay protection' { } } Describe 'Startup report validation' { + It 'accepts the prerequisite phase while WinGet is still installing' { + $path = "$TestDrive/state.json" + @{ BootstrapId = $id; SubscriptionId = $subscription; ResourceGroupName = 'group'; State = 'Running'; Phase = 'Prerequisites'; Code = ''; Message = 'Waiting for WinGet' } | + ConvertTo-Json | Set-Content $path + (Get-LocalBoxStartupReport $path $id $subscription group).Phase | Should -Be 'Prerequisites' + } It 'rejects reports for another bootstrap or subscription' { $path = "$TestDrive/state.json" @{ BootstrapId = 'foreign'; SubscriptionId = $subscription; ResourceGroupName = 'group'; State = 'Failed'; Phase = 'BaseImages' } | ConvertTo-Json | Set-Content $path { Get-LocalBoxStartupReport $path $id $subscription group } | Should -Throw '*does not match*' } + + Describe 'WinGet startup prerequisite ordering' { + BeforeEach { + $prerequisiteClock = @{ Now = [datetime]'2026-11-06T00:00:00Z'; Sleeps = 0 } + $originalPath = $env:PATH + $prerequisiteStatus = @{ BootstrapId = $id; State = 'Running'; Phase = 'Startup' } + $prerequisitePath = "$TestDrive/prerequisites.json" + Mock Get-Date { $prerequisiteClock.Now } + Mock Get-ScheduledTask { @() } + Mock Start-Sleep { + $prerequisiteClock.Sleeps++ + $prerequisiteClock.Now = $prerequisiteClock.Now.AddSeconds(15) + } + Mock Resolve-LocalBoxAzCopy { 'fixture-azcopy.exe' } + } + AfterEach { $env:PATH = $originalPath } + It 'waits for installer completion before resolving tools or initializing storage' { + Mock Get-ScheduledTask { + if ($prerequisiteClock.Sleeps -lt 2) { @{ TaskName = 'WinGetLogonScript'; TaskPath = '\'; State = 'Running' } } + } + Mock Resolve-LocalBoxAzCopy { + $prerequisiteClock.Sleeps | Should -Be 2 + 'fixture-azcopy.exe' + } + Wait-LocalBoxStartupPrerequisites $prerequisiteStatus $prerequisitePath + Should -Invoke Start-Sleep -Times 2 -Exactly + Should -Invoke Resolve-LocalBoxAzCopy -Times 1 -Exactly + $prerequisiteStatus.Phase | Should -Be 'Startup' + } + It 'also waits for a registered task that has not started yet' { + Mock Get-ScheduledTask { + if ($prerequisiteClock.Sleeps -eq 0) { @{ TaskName = 'WinGetLogonScript'; TaskPath = '\'; State = 'Ready' } } + } + Wait-LocalBoxStartupPrerequisites $prerequisiteStatus $prerequisitePath + Should -Invoke Start-Sleep -Times 1 -Exactly + } + It 'stops at the one-hour installation budget without attempting a download' { + Mock Get-ScheduledTask { @{ TaskName = 'WinGetLogonScript'; TaskPath = '\'; State = 'Running' } } + Mock Start-Sleep { $prerequisiteClock.Now = $prerequisiteClock.Now.AddHours(1) } + { Wait-LocalBoxStartupPrerequisites $prerequisiteStatus $prerequisitePath } | Should -Throw '*one-hour prerequisite budget*' + ($prerequisiteClock.Now - [datetime]'2026-11-06T00:00:00Z').TotalSeconds | Should -Be 3600 + Should -Invoke Resolve-LocalBoxAzCopy -Times 0 + } + It 'does not treat an installation/discovery failure as ready' { + Mock Resolve-LocalBoxAzCopy { throw 'fixture-missing-azcopy' } + { Wait-LocalBoxStartupPrerequisites $prerequisiteStatus $prerequisitePath } | Should -Throw '*fixture-missing-azcopy*' + $prerequisiteStatus.Phase | Should -Be 'Prerequisites' + } + It 'surfaces task query errors rather than assuming WinGet completed' { + Mock Get-ScheduledTask { throw 'fixture-scheduler-unavailable' } + { Wait-LocalBoxStartupPrerequisites $prerequisiteStatus $prerequisitePath } | Should -Throw '*fixture-scheduler-unavailable*' + Should -Invoke Resolve-LocalBoxAzCopy -Times 0 + } + } + + Describe 'AzCopy resolution with a stale process PATH' { + BeforeEach { + Mock Get-Command { @() } -ParameterFilter { $Name -eq 'azcopy' } + $localData = Join-Path $TestDrive ([guid]::NewGuid().ToString()) + $programRoot = Join-Path $TestDrive ([guid]::NewGuid().ToString()) + $resolveArguments = @{ LocalAppData = $localData; ProgramFiles = $programRoot } + } + It 'preserves an existing native PATH resolution' { + Mock Get-Command { @{ Path = 'native-azcopy.exe' } } -ParameterFilter { $Name -eq 'azcopy' } + Resolve-LocalBoxAzCopy @resolveArguments | Should -Be 'native-azcopy.exe' + } + It 'finds the current startup user WinGet link without searching other profiles' { + $directory = Join-Path $localData 'Microsoft/WinGet/Links' + New-Item -ItemType Directory -Path $directory -Force | Out-Null + Set-Content "$directory/azcopy.exe" 'fixture' + Resolve-LocalBoxAzCopy @resolveArguments | Should -Be "$directory/azcopy.exe" + } + It 'finds the exact installed package when its WinGet link is absent' { + $directory = Join-Path $localData 'Microsoft/WinGet/Packages/Microsoft.Azure.AZCopy.10_fixture/azcopy_windows_amd64_10.34.1' + New-Item -ItemType Directory -Path $directory -Force | Out-Null + Set-Content "$directory/azcopy.exe" 'fixture' + Resolve-LocalBoxAzCopy @resolveArguments | Should -Be "$directory/azcopy.exe" + } + It 'fails explicitly instead of choosing between multiple unlinked versions' { + foreach ($version in @('10.34.0','10.34.1')) { + $directory = Join-Path $localData "Microsoft/WinGet/Packages/Microsoft.Azure.AZCopy.10_fixture/azcopy_windows_amd64_$version" + New-Item -ItemType Directory -Path $directory -Force | Out-Null + Set-Content "$directory/azcopy.exe" 'fixture' + } + { Resolve-LocalBoxAzCopy @resolveArguments } | Should -Throw '*Multiple unlinked AzCopy*' + } + It 'reports an actionable error when installation did not provide AzCopy' { + { Resolve-LocalBoxAzCopy @resolveArguments } | Should -Throw '*Inspect the WinGet provisioning log*' + } + } It 'rejects corrupt JSON and invalid states' { $path = "$TestDrive/state.json" Set-Content $path 'not-json' diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md index 1d8dd85dc..214393a6d 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md @@ -70,6 +70,19 @@ deployment is still required to verify custom-data delivery, scheduled-task execution and end-to-end Console reporting together; existing failed instances are deliberately not retrofitted. +The startup suite also covers WinGet-before-storage ordering, the one-hour +prerequisite timeout, stale-PATH AzCopy resolution using only the current user's +known WinGet paths, ambiguous/missing installations, and reporting the new +prerequisite phase. Deployment checks verify that startup identity tags are +merged into the resource group so upstream progress updates preserve them. +An existing monitor with missing VM identity tags fails explicitly without +restarting or overwriting anything. +On traininglab-17, a read-only Windows probe ran the actual resolver with AzCopy +absent from the process PATH. It resolved the startup user's existing WinGet +link and successfully ran `azcopy --version`; no image was downloaded and no +deployment was resumed. The prerequisite race and tag-retention fixes still +require a fresh end-to-end Console deployment. + Run `Invoke-Pester ./localbox-failure-summary.tests.ps1,./localbox-automation.tests.ps1 -Output Detailed` for Console failure reporting. Tests cover run/content/group/scope correlation, real NUnit failure counts and messages, missing/malformed reports, prohibited XML From 73241c151b9b5d772aa32d3f6ebc5bdc44954564 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Fri, 6 Nov 2026 14:36:17 +0000 Subject: [PATCH 19/37] Preserve upstream LocalBox error semantics at startup boundary Keep guard operations strict without promoting upstream non-terminating lookups to fatal failures. Embed the shared diagnostic formatter so startup reports include bounded redacted underlying errors. Verify with regression tests and read-only PowerShell 7 checks on both nested nodes. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../labautomation/localbox-startup.ps1 | 26 ++++++++++++- .../resources/hosted-events/readme.md | 22 +++++++++++ .../tests/localbox-deployment.tests.ps1 | 1 + .../tests/localbox-startup.tests.ps1 | 38 +++++++++++++++++++ .../resources/tests/readme.md | 15 ++++++++ 5 files changed, 100 insertions(+), 2 deletions(-) diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-startup.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-startup.ps1 index cd6ee2e97..7cbfb203e 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-startup.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-startup.ps1 @@ -21,6 +21,20 @@ function Set-LocalBoxStartupTemplate { if ([regex]::Matches($text, 'osProfile: \{').Count -ne 1) { throw 'Cannot uniquely locate the upstream VM osProfile.' } $source = (Get-Content -LiteralPath $StartupSource -Raw -ErrorAction Stop).Replace('__LOCALBOX_BOOTSTRAP_ID__', $Id.ToString()) if ([Text.Encoding]::UTF8.GetByteCount($source) -gt 65535) { throw 'LocalBox startup protection exceeds the VM custom-data limit.' } + $diagnosticsPath = Join-Path $PSScriptRoot 'localbox-automation.ps1' + $parseErrors = $null + $diagnostics = [Management.Automation.Language.Parser]::ParseFile($diagnosticsPath, [ref]$null, [ref]$parseErrors) + if ($parseErrors.Count) { throw 'Cannot parse the shared LocalBox diagnostic formatter for startup packaging.' } + $formatter = $diagnostics.FindAll({ + param($node) + $node -is [Management.Automation.Language.FunctionDefinitionAst] -and $node.Name -eq 'ConvertTo-LocalBoxDiagnosticText' + }, $true) + $sourceAst = [Management.Automation.Language.Parser]::ParseInput($source, [ref]$null, [ref]$parseErrors) + if ($parseErrors.Count -or $formatter.Count -ne 1 -or -not $sourceAst.ParamBlock) { + throw 'Cannot embed the shared diagnostic formatter in the startup payload.' + } + $source = $source.Insert($sourceAst.ParamBlock.Extent.EndOffset, "`n`n" + $formatter[0].Extent.Text) + if ([Text.Encoding]::UTF8.GetByteCount($source) -gt 65535) { throw 'LocalBox startup protection exceeds the VM custom-data limit.' } $destination = Join-Path (Split-Path -Parent $HostTemplate) 'localbox-startup.ps1' Set-Content -LiteralPath $destination -Value $source -Encoding utf8NoBOM -NoNewline -ErrorAction Stop $text = $text.Replace('osProfile: {', "osProfile: {`n customData: base64(loadTextContent('localbox-startup.ps1'))") @@ -274,6 +288,14 @@ function Sync-LocalBoxBaseImages { Save-LocalBoxStartupStatus $Status $StatusPath } +function Invoke-LocalBoxUpstreamStartup { + param([string]$Path) + # Preserve upstream non-terminating errors; the guard's own operations remain strict. + $ErrorActionPreference = 'Continue' + & $Path + if (-not $?) { throw 'Upstream LocalBox startup returned unsuccessfully.' } +} + function Invoke-LocalBoxGuardedStartup { param([string]$Id, [string]$Root = 'C:\LocalBox') $ErrorActionPreference = 'Stop' @@ -301,8 +323,7 @@ function Invoke-LocalBoxGuardedStartup { Save-LocalBoxStartupStatus $status $statusPath Wait-LocalBoxStartupPrerequisites -Status $status -StatusPath $statusPath Set-LocalBoxBaseImageGuard -Path "$Root/New-LocalBoxCluster.ps1" -PlanPath "$Root/SovereignStartup/images.json" -Id $Id - & "$Root/LocalBoxLogonScript.ps1" - if (-not $?) { throw 'Upstream LocalBox startup returned unsuccessfully.' } + Invoke-LocalBoxUpstreamStartup -Path "$Root/LocalBoxLogonScript.ps1" $status = Get-Content -LiteralPath $statusPath -Raw | ConvertFrom-Json -AsHashtable -ErrorAction Stop if ($status.State -eq 'Failed') { throw 'Upstream startup returned after a reported base-image failure.' } $status.State = 'Succeeded' @@ -325,6 +346,7 @@ function Invoke-LocalBoxGuardedStartup { $latest.Code = 'StartupFailed' $latest.Message = 'Upstream LocalBox startup failed. Inspect LocalBoxLogonScript.log and New-LocalBoxCluster.log; do not blindly rerun storage initialization.' } + $latest.Message = "$(ConvertTo-LocalBoxDiagnosticText $failure.Exception.Message 600) $($latest.Message)" Save-LocalBoxStartupStatus $latest $statusPath } } diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md index e6d8bb8c6..ede3da15f 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md @@ -367,6 +367,28 @@ installation race addressed by the prerequisite gate; it was not another HTTP 401 failure. Read-only inspection also confirmed the upstream tag replacement. Neither bootstrap nor storage initialization was replayed during diagnosis. +TEST11 on traininglab-05 progressed through WinGet, verified base-image downloads +and creation of all three nested VMs, then stopped at cluster-build step 9 after +approximately **1h18m**. Upstream looked up `ImageCustomizationScheduledTask`, +which was absent on both nodes, although both module directories checked by +`Wait-AzureEdgeBootstrap` were already present. The guard's inherited +`ErrorActionPreference=Stop` incorrectly promoted that upstream non-terminating +lookup error into a fatal exception. + +The guard now preserves upstream's `Continue` error semantics **only while +invoking the upstream logon script**. Non-terminating errors remain visible; +explicit throws, explicit `-ErrorAction Stop` and upstream readiness timeouts +still fail. Guard operations, prerequisite checks and image transfers remain +strict, and all final deployment/health gates are unchanged. This does not bypass +node readiness or create a replacement customization task. + +New startup payloads embed the existing shared diagnostic formatter. A terminating +startup failure now includes its bounded, credential-redacted underlying message +in the correlated report instead of only a generic instruction to inspect logs. +Existing VMs retain their original immutable payload; a fresh deployment is +required to exercise these changes. TEST11 was inspected without replaying +storage initialization or resuming cluster deployment. + #### Automatic Azure Local validation recovery The Console can perform **one automatic validation retry**, after a **two-minute diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-deployment.tests.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-deployment.tests.ps1 index 77f239863..f34959bf3 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-deployment.tests.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-deployment.tests.ps1 @@ -10,6 +10,7 @@ BeforeAll { New-Item -ItemType Directory -Path "$fixture/labautomation", "$fixture/resources/manual-setup/localbox" -Force | Out-Null Copy-Item "$root/labautomation/deploy-localbox.ps1" "$fixture/labautomation" Copy-Item "$root/labautomation/localbox-startup.ps1" "$fixture/labautomation" + Copy-Item "$root/labautomation/localbox-automation.ps1" "$fixture/labautomation" Copy-Item "$root/resources/manual-setup/localbox/deploy-localbox.ps1" "$fixture/resources/manual-setup/localbox" function az {} } diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-startup.tests.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-startup.tests.ps1 index f343ec09b..143c7874d 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-startup.tests.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-startup.tests.ps1 @@ -43,6 +43,12 @@ var command = { $text | Should -Match '&& powershell.exe.*-Install && powershell.exe.*-adminPassword opaque' Get-Content "$TestDrive/localbox-startup.ps1" -Raw | Should -Match $id Get-Content "$TestDrive/localbox-startup.ps1" -Raw | Should -Not -Match '__LOCALBOX_BOOTSTRAP_ID__' + $embeddedAst = [Management.Automation.Language.Parser]::ParseFile("$TestDrive/localbox-startup.ps1", [ref]$null, [ref]$null) + $embeddedFormatter = $embeddedAst.Find({ + param($node) + $node -is [Management.Automation.Language.FunctionDefinitionAst] -and $node.Name -eq 'ConvertTo-LocalBoxDiagnosticText' + }, $true) + $embeddedFormatter.Body.Extent.Text | Should -Be ('{' + (Get-Command ConvertTo-LocalBoxDiagnosticText).Definition + '}') } It 'does not replace immutable custom data for an already guarded VM' { Set-LocalBoxStartupTemplate $hostPath "$root/labautomation/localbox-startup.ps1" $id @@ -223,6 +229,7 @@ Describe 'Guarded startup replay protection' { Set-Content "$startupRoot/LocalBoxLogonScript.ps1" 'throw "fixture-upstream-failed"' { Invoke-LocalBoxGuardedStartup $id $startupRoot } | Should -Throw '*fixture-upstream-failed*' (Get-Content $stateFile -Raw | ConvertFrom-Json).Code | Should -Be 'StartupFailed' + (Get-Content $stateFile -Raw | ConvertFrom-Json).Message | Should -Match 'fixture-upstream-failed' { Invoke-LocalBoxGuardedStartup $id $startupRoot } | Should -Throw '*Automatic replay*disabled*' Should -Invoke Set-LocalBoxBaseImageGuard -Times 1 -Exactly } @@ -243,6 +250,37 @@ Describe 'Guarded startup replay protection' { $saved.State | Should -Be 'Succeeded' $saved.Message | Should -Match 'Full health gates are still required' } + It 'preserves a non-terminating upstream lookup error and still runs its next readiness gate' { + @' +Write-Error "No MSFT_ScheduledTask objects found for ImageCustomizationScheduledTask" +Set-Content "$PSScriptRoot/readiness-checked" 'yes' +'@ | Set-Content "$startupRoot/LocalBoxLogonScript.ps1" + $result = @(Invoke-LocalBoxGuardedStartup $id $startupRoot 2>&1) + ($result | Out-String) | Should -Match 'ImageCustomizationScheduledTask' + Get-Content "$startupRoot/readiness-checked" | Should -Be 'yes' + (Get-Content $stateFile -Raw | ConvertFrom-Json).State | Should -Be 'Succeeded' + } + It 'still fails when upstream readiness throws after a non-terminating lookup' { + @' +Write-Error "ImageCustomizationScheduledTask is absent" +throw 'Timeout waiting for required bootstrap paths' +'@ | Set-Content "$startupRoot/LocalBoxLogonScript.ps1" + { Invoke-LocalBoxGuardedStartup $id $startupRoot 2>$null } | Should -Throw '*required bootstrap paths*' + (Get-Content $stateFile -Raw | ConvertFrom-Json).State | Should -Be 'Failed' + } + It 'keeps an explicit upstream ErrorAction Stop terminating' { + Set-Content "$startupRoot/LocalBoxLogonScript.ps1" 'Write-Error "fixture-required-operation-failed" -ErrorAction Stop' + { Invoke-LocalBoxGuardedStartup $id $startupRoot } | Should -Throw '*fixture-required-operation-failed*' + (Get-Content $stateFile -Raw | ConvertFrom-Json).State | Should -Be 'Failed' + } + It 'records a bounded redacted error rather than only a generic failure' { + Set-Content "$startupRoot/LocalBoxLogonScript.ps1" 'throw (''Missing task; password=fixture-sensitive token=fixture-token '' + (''x'' * 1000))' + { Invoke-LocalBoxGuardedStartup $id $startupRoot } | Should -Throw + $saved = Get-Content $stateFile -Raw | ConvertFrom-Json + $saved.Message | Should -Match 'Missing task' + $saved.Message | Should -Not -Match 'fixture-sensitive|fixture-token' + $saved.Message.Length | Should -BeLessOrEqual 1000 + } } Describe 'Native transfer outcome checks' { diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md index 214393a6d..27693da48 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md @@ -83,6 +83,21 @@ link and successfully ran `azcopy --version`; no image was downloaded and no deployment was resumed. The prerequisite race and tag-retention fixes still require a fresh end-to-end Console deployment. +Error-boundary regressions cover a non-terminating upstream task lookup followed +by its readiness gate, a subsequent terminating readiness failure, explicit +`-ErrorAction Stop`, and bounded/redacted underlying startup errors. Packaging +checks verify that the embedded diagnostic formatter is identical to the shared +host formatter, rather than a separate redaction implementation. The guard's own +strict download/checksum failures and the final readiness gates remain covered. +Read-only TEST11 diagnostics confirmed the missing customization task and both +required bootstrap-module directories on each node; the full resumed deployment +was not executed. +A PowerShell 7 probe on the same Client also reproduced the lookup failure under +the old strict boundary, then used the actual corrected boundary to reach and +pass those checks on both nodes. Both lookup errors remained visible and the +parent preference stayed strict. The probe only queried tasks and paths; it did +not start tasks, replay initialization or resume deployment. + Run `Invoke-Pester ./localbox-failure-summary.tests.ps1,./localbox-automation.tests.ps1 -Output Detailed` for Console failure reporting. Tests cover run/content/group/scope correlation, real NUnit failure counts and messages, missing/malformed reports, prohibited XML From b7db4afebe674746c2b3fb8eb2eed442e42bf051 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Sat, 7 Nov 2026 05:47:35 +0000 Subject: [PATCH 20/37] Harden LocalBox prerequisites and validation recovery Guard WinGet repair and native configuration results, remove the retired Azure Data Studio prerequisite, preserve module error semantics, and refresh the validation retry PATH. Include bounded retry diagnostics, regression tests, and verified live recovery evidence without claiming Full readiness. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../labautomation/localbox-automation.ps1 | 2 +- .../labautomation/localbox-startup.ps1 | 126 ++++++++++++- .../localbox-validation-retry.ps1 | 10 ++ .../resources/hosted-events/readme.md | 93 +++++++++- .../tests/localbox-startup.tests.ps1 | 46 +++++ .../tests/localbox-validation-retry.tests.ps1 | 34 ++++ .../resources/tests/localbox-winget.tests.ps1 | 166 ++++++++++++++++++ .../resources/tests/readme.md | 52 +++++- 8 files changed, 515 insertions(+), 14 deletions(-) create mode 100644 03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-winget.tests.ps1 diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation.ps1 index 763aa2a08..0f65356d7 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation.ps1 @@ -80,7 +80,7 @@ function Get-LocalBoxValidationRetryState { Detail = 'Waiting for validation retry 1/1 to start; ignoring stale instanceView.' } } if ($state.executionState -in @('Failed', 'Canceled', 'TimedOut')) { - throw 'Automatic validation retry failed or timed out. No further automatic retry will be submitted; inspect sovereign-localbox-validation-retry and localcluster-validate.' + throw "Automatic validation retry failed or timed out. $(ConvertTo-LocalBoxDiagnosticText $state.error 800) No further automatic retry will be submitted; inspect sovereign-localbox-validation-retry and localcluster-validate." } if ($state.executionState -eq 'Succeeded') { if ($null -eq $state.exitCode -or $state.exitCode -ne 0) { throw 'Validation recovery did not exit successfully.' } diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-startup.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-startup.ps1 index 7cbfb203e..665bbb28e 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-startup.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-startup.ps1 @@ -54,11 +54,120 @@ $guardRoot = Join-Path $LocalBoxPath 'SovereignStartup' New-Item -ItemType Directory -Path $guardRoot -Force -ErrorAction Stop | Out-Null Set-Acl -LiteralPath $guardRoot -AclObject (Get-Acl -LiteralPath 'C:\AzureData' -ErrorAction Stop) -ErrorAction Stop Copy-Item -LiteralPath 'C:\AzureData\CustomData.bin' -Destination "$guardRoot\startup.ps1" -Force -ErrorAction Stop +. "$guardRoot\startup.ps1" +Set-LocalBoxWinGetGuard -Path "$LocalBoxPath\WinGet.ps1" +Set-LocalBoxPackageConfiguration -Path "$LocalBoxPath\DSC\packages.dsc.yml" $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument "$guardRoot\startup.ps1" '@ Set-Content -LiteralPath $Path -Value $text.Replace($action, $replacement) -Encoding utf8 -ErrorAction Stop } +function Set-LocalBoxWinGetGuard { + param([string]$Path) + $text = Get-Content -LiteralPath $Path -Raw -ErrorAction Stop + $preference = '$ErrorActionPreference = $env:ErrorActionPreference' + $repair = 'Repair-WinGetPackageManager -AllUsers -Force -Latest -Verbose' + $packageCommand = '& $winget configure --file "$($Env:LocalBoxDir)\DSC\packages.dsc.yml" --accept-configuration-agreements --disable-interactivity' + $hostCommand = '& $winget configure --file "$($Env:LocalBoxDir)\DSC\hyper-v.dsc.yml" --accept-configuration-agreements --disable-interactivity' + foreach ($line in @($preference, $packageCommand, $hostCommand)) { + if ([regex]::Matches($text, [regex]::Escape($line)).Count -ne 1) { throw 'Unsupported upstream WinGet script; prerequisite protection was not installed.' } + } + if ([regex]::Matches($text, [regex]::Escape($repair)).Count -ne 2) { throw 'Unsupported upstream WinGet repair sequence.' } + $text = $text.Replace($preference, '$ErrorActionPreference = ''Stop''') + $first = $text.IndexOf($repair) + $text = $text.Remove($first, $repair.Length).Insert($first, '. "$Env:LocalBoxDir\SovereignStartup\startup.ps1"' + "`n" + 'Initialize-LocalBoxWinGet') + $text = $text.Replace($repair, '') + $text = $text.Replace($packageCommand, 'Invoke-LocalBoxWinGetConfiguration -Path "$Env:LocalBoxDir\DSC\packages.dsc.yml"') + $text = $text.Replace($hostCommand, 'Invoke-LocalBoxWinGetConfiguration -Path "$Env:LocalBoxDir\DSC\hyper-v.dsc.yml"') + Set-Content -LiteralPath $Path -Value $text -Encoding utf8 -ErrorAction Stop +} + +function Set-LocalBoxPackageConfiguration { + param([string]$Path) + $text = (Get-Content -LiteralPath $Path -Raw -ErrorAction Stop).Replace("`r`n", "`n") + $block = @' + - resource: Microsoft.WinGet.DSC/WinGetPackage + id: Microsoft.AzureDataStudio + directives: + description: Install Microsoft Azure Data Studio + settings: + id: Microsoft.AzureDataStudio + source: winget +'@ + $pattern = '(?m)^' + [regex]::Escape($block) + '[ \t]*(?:\n|$)' + $matches = [regex]::Matches($text, $pattern) + if ($matches.Count -eq 0 -and $text -notmatch 'AzureDataStudio') { return } + if ($matches.Count -ne 1 -or [regex]::Replace($text, $pattern, '') -match 'AzureDataStudio') { + throw 'Unrecognized retired Azure Data Studio package configuration; no package entries were changed.' + } + Write-Host 'Removing the retired Azure Data Studio package from LocalBox prerequisites; all remaining package results must still succeed.' + Set-Content -LiteralPath $Path -Value ([regex]::Replace($text, $pattern, '')) -Encoding utf8 -ErrorAction Stop +} + +function Initialize-LocalBoxWinGet { + param([ValidateRange(1, 3)][int]$Attempts = 3) + $ErrorActionPreference = 'Stop' + $module = Get-Module -ListAvailable Microsoft.WinGet.Client | Sort-Object Version -Descending | Select-Object -First 1 + if (-not $module) { throw 'Microsoft.WinGet.Client is not installed; package configuration was not started.' } + $version = $module.Version.ToString() + Import-Module Microsoft.WinGet.Client -RequiredVersion $version -ErrorAction Stop + for ($attempt = 1; $attempt -le $Attempts; $attempt++) { + try { + Repair-WinGetPackageManager -Version $version -AllUsers -Force -ErrorAction Stop + $actual = [string](Get-WinGetVersion -ErrorAction Stop) + if ($actual.TrimStart('v') -ne $version) { throw "WinGet CLI '$actual' does not match installed client module '$version'." } + $null = @(Get-WinGetPackage -ErrorAction Stop) + Write-Host "WinGet CLI/client $version compatibility and package enumeration verified." + return + } + catch { + if ($attempt -eq $Attempts) { throw } + Write-Warning "WinGet manager repair attempt $attempt/$Attempts failed: $(ConvertTo-LocalBoxDiagnosticText $_.Exception.Message 400). Retrying in 30 seconds." + Start-Sleep -Seconds 30 + } + } +} + +function Invoke-LocalBoxWinGetConfiguration { + param([string]$Path, [string]$LogDirectory = 'C:\LocalBox\SovereignStartup', + [ValidateRange(1, 3)][int]$Attempts = 3, [ValidateRange(1, 900)][int]$AttemptTimeoutSeconds = 900) + $ErrorActionPreference = 'Stop' + if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) { throw 'The WinGet configuration file is missing.' } + $winget = Join-Path $env:LOCALAPPDATA 'Microsoft/WindowsApps/winget.exe' + if (-not (Test-Path -LiteralPath $winget -PathType Leaf)) { throw 'WinGet is unavailable in the startup user profile.' } + for ($attempt = 1; $attempt -le $Attempts; $attempt++) { + $process = $null + $log = Join-Path $LogDirectory ("winget-{0}-{1}-{2}" -f [IO.Path]::GetFileNameWithoutExtension($Path), $PID, $attempt) + try { + $process = Start-Process -FilePath $winget -ArgumentList @('configure', '--file', "`"$Path`"", + '--accept-configuration-agreements', '--disable-interactivity') -NoNewWindow -PassThru ` + -RedirectStandardOutput "$log.stdout.log" -RedirectStandardError "$log.stderr.log" -ErrorAction Stop + if (-not $process.WaitForExit($AttemptTimeoutSeconds * 1000)) { throw "WinGet configuration exceeded its $AttemptTimeoutSeconds-second attempt budget." } + $process.WaitForExit() + if ($null -eq $process.ExitCode -or $process.ExitCode -ne 0) { + throw "WinGet configuration failed with exit code '$($process.ExitCode)'. Inspect the private $log logs." + } + return + } + catch { + if ($attempt -eq $Attempts) { throw } + Write-Warning "WinGet configuration attempt $attempt/$Attempts failed: $(ConvertTo-LocalBoxDiagnosticText $_.Exception.Message 400). Retrying in 30 seconds." + } + finally { + if ($process) { + try { + if (-not $process.HasExited) { + $process.Kill($true) + if (-not $process.WaitForExit(10000)) { throw 'WinGet process did not stop after its timeout.' } + } + } + finally { $process.Dispose() } + } + } + Start-Sleep -Seconds 30 + } +} + function Save-LocalBoxStartupStatus { param([hashtable]$Status, [string]$Path) $Status.UpdatedAt = [datetime]::UtcNow.ToString('o') @@ -153,6 +262,12 @@ function Wait-LocalBoxStartupPrerequisites { if ($tasks.Count -gt 1) { throw 'Ambiguous WinGet prerequisite task state.' } if (-not $tasks.Count) { break } if ($tasks[0].State -eq 'Disabled') { throw 'WinGetLogonScript is disabled; prerequisite installation cannot be verified.' } + if ($tasks[0].State -eq 'Ready') { + $info = $tasks[0] | Get-ScheduledTaskInfo -ErrorAction Stop + if ($info.LastRunTime -gt [datetime]'2000-01-01' -and $info.LastTaskResult -ne 0) { + throw "WinGetLogonScript stopped with task result $($info.LastTaskResult). Inspect its provisioning and private WinGet logs; storage initialization was not started." + } + } if ((Get-Date).ToUniversalTime() -ge $deadline) { throw 'WinGetLogonScript did not complete within the one-hour prerequisite budget. Inspect its provisioning log; storage initialization was not started.' } @@ -290,10 +405,15 @@ function Sync-LocalBoxBaseImages { function Invoke-LocalBoxUpstreamStartup { param([string]$Path) - # Preserve upstream non-terminating errors; the guard's own operations remain strict. + # Modules resolve preferences through global scope, not the calling function. $ErrorActionPreference = 'Continue' - & $Path - if (-not $?) { throw 'Upstream LocalBox startup returned unsuccessfully.' } + $previousGlobalPreference = $global:ErrorActionPreference + try { + $global:ErrorActionPreference = 'Continue' + & $Path + if (-not $?) { throw 'Upstream LocalBox startup returned unsuccessfully.' } + } + finally { $global:ErrorActionPreference = $previousGlobalPreference } } function Invoke-LocalBoxGuardedStartup { diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-validation-retry.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-validation-retry.ps1 index de0d8b2ae..43acb33cd 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-validation-retry.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-validation-retry.ps1 @@ -6,6 +6,15 @@ param( [string]$RecoveryVersion ) +function Update-LocalBoxValidationRetryPath { + param( + [string]$MachinePath = [Environment]::GetEnvironmentVariable('PATH', 'Machine'), + [string]$UserPath = [Environment]::GetEnvironmentVariable('PATH', 'User') + ) + $env:PATH = (@($env:PATH, $MachinePath, $UserPath) | + Where-Object { $_ } | Select-Object -Unique) -join [IO.Path]::PathSeparator +} + function Invoke-LocalBoxValidationRetryAz { param([string[]]$Arguments, [switch]$NoOutput) $PSNativeCommandUseErrorActionPreference = $false @@ -38,6 +47,7 @@ function Invoke-LocalBoxValidationRetry { foreach ($path in @($template, $parameters)) { if (-not (Test-Path -LiteralPath $path -PathType Leaf)) { throw "Required existing LocalBox deployment file is missing: $path" } } + Update-LocalBoxValidationRetryPath Get-Command az -ErrorAction Stop | Out-Null Write-Host "[LocalBox] Automatic validation retry 1/1: waiting $RetryDelaySeconds seconds before rechecking the failed deployment." Start-Sleep -Seconds $RetryDelaySeconds diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md index ede3da15f..59a8d989c 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md @@ -322,6 +322,36 @@ not search other users' profiles, install a replacement binary, or blindly pick between multiple unlinked package versions. Prerequisite failure is reported as `Prerequisites` / `StartupPrerequisitesFailed`, before storage initialization. +WinGet provisioning is also guarded. Its CLI is repaired to the installed +`Microsoft.WinGet.Client` module version, with at most three attempts and +30-second retry delays; version matching and a real `Get-WinGetPackage` COM +enumeration must succeed before package configuration. Both configuration calls +check their native exit codes, retain private stdout/stderr logs and allow at +most three attempts, each capped at 15 minutes. A failed WinGet task is reported +without waiting for the entire prerequisite grace period. + +The recognized upstream `Microsoft.AzureDataStudio` configuration entry is +removed explicitly: [Azure Data Studio retired in February 2026](https://learn.microsoft.com/en-us/sql/tools/whats-happening-azure-data-studio) +and its WinGet package is no longer available. This workshop does not depend on +it. VS Code, SSMS and all other configured packages remain required; their errors +are not ignored. Unrecognized configuration changes stop deployment for review. + +TEST12 exposed why checking only whether the WinGet task disappeared was +insufficient. Both upstream manager repairs received HTTP 504, leaving CLI +1.11.510 with Client/DSC modules 1.29.380. Subsequent DSC calls failed with +`0x800706BA` (RPC server unavailable); see +[microsoft/winget-cli#6560](https://github.com/microsoft/winget-cli/issues/6560). +Live recovery on the existing traininglab-17 VM repaired the CLI to 1.29.380, +verified COM enumeration, and found the additional retired-package failure. +After removing only that entry, package configuration returned zero and AzCopy +passed its executable version check. Startup was resumed only after verifying +that no storage pools or nested VMs existed and archiving the original failed +startup report. The resumed startup began at **18:10:38 UTC on 6 November 2026** +and verified both base-image SHA256 checksums at **18:15:09 UTC** (4m31s after +resume), then entered nested deployment with a healthy storage pool. This is +authorized recovery evidence, not a claim that the complete resumed deployment +or a new cold deployment has passed. + The startup identity is also merged into the resource-group tags before deployment: upstream progress updates replace VM tags with the resource-group tags. If an existing Client has a startup monitor but has lost its identity tags, deployment @@ -376,12 +406,36 @@ which was absent on both nodes, although both module directories checked by lookup error into a fatal exception. The guard now preserves upstream's `Continue` error semantics **only while -invoking the upstream logon script**. Non-terminating errors remain visible; +invoking the upstream logon script**, including the global preference used by +imported modules, and restores the previous global preference in `finally`. +Non-terminating errors remain visible; explicit throws, explicit `-ErrorAction Stop` and upstream readiness timeouts still fail. Guard operations, prerequisite checks and image transfers remain strict, and all final deployment/health gates are unchanged. This does not bypass node readiness or create a replacement customization task. +TEST12's authorized resume exposed an additional scope issue at **18:55:57 UTC +on 6 November 2026**, 45m19s after resume: setting `Continue` only in the calling +function did not affect the imported LocalBox module, which still resolved the +global `Stop` preference. The earlier script-level probe did not exercise that +module boundary. Regression tests now import a real fixture module, covering +non-terminating errors, explicit failures and preference restoration. A read-only +probe on the current Client reproduced the old module-scoped failure, then +verified both nodes with the corrected boundary at **19:03:55 UTC**; both lookup +errors remained visible and no tasks were started. Authorized recovery resumed +at **19:10:24 UTC** from cluster-build step 9 only, preserving the healthy storage +pool and all three running nested VMs. It uses a separate protected helper; +the original read-only startup payload remains unchanged. Full end-to-end +readiness remains unverified. + +By **19:20:50 UTC**, the continuation had reached step 10, reported successful +Arc bootstrap on AzLHOST2, and verified all mandatory resource providers as +registered. The upstream ten-minute registration-propagation wait still applies +before validation submission. An operator-monitor token expired at 19:19:07; +the guest task kept running. Monitoring was resumed without replaying any guest +stage, using refresh deadlines derived from the actual token expirations. +Reaching step 10 is not a successful validation or Full health result. + New startup payloads embed the existing shared diagnostic formatter. A terminating startup failure now includes its bounded, credential-redacted underlying message in the correlated report instead of only a generic instruction to inspect logs. @@ -401,7 +455,15 @@ failures, unreadable diagnostics and ambiguous state still stop the job. The host submits the asynchronous Managed Run Command `sovereign-localbox-validation-retry` on LocalBox-Client. Keep [localbox-validation-retry.ps1](../../labautomation/localbox-validation-retry.ps1) -in the imported `labautomation` directory. The recovery: +in the imported `labautomation` directory. + +Before using Azure CLI, the standalone Run Command refreshes its process PATH +from the persisted machine/user paths, retaining existing entries. The VM agent +can otherwise retain a PATH from before WinGet installed Azure CLI. Missing CLI +still fails before any validation is submitted. Terminal retry errors include a +bounded, redacted underlying diagnostic in Console. + +The recovery: 1. Rechecks the original failed validation correlation after the delay. 2. Uses the Client's managed identity and existing `C:\LocalBox\azlocal.json` and @@ -430,6 +492,33 @@ Do not manually re-trigger validation/deployment while automatic recovery is running. Cancellation or forced guest termination may prevent CLI-profile cleanup; inspect the Client before resetting recovery state. +TEST12's first validation failed with the recognized content-download timeout at +**21:40:51 UTC on 6 November 2026**. Its automatic retry stopped before the delay +or any validation submission because the agent's stale PATH could not resolve +`az`. A live SYSTEM-context probe reproduced the missing command, then verified +the actual PATH-refresh/CLI helpers, Azure CLI 2.92.0, managed-identity ARM access +and matching existing template/parameter contracts without submitting validation. +The authorized repair replaced only that failed preflight execution; retry 1/1 +started at **21:47:59 UTC**, retaining the original remaining timeout and failed +validation correlation. ARM confirmed a new validation execution in `Running` +state at **21:50:11 UTC**, after the required backoff. Validation succeeded at +**22:25:07 UTC** (34m56s after submission), and recovery submitted cluster +deployment and exited successfully at **22:25:32 UTC**. `localcluster-deploy` +succeeded at **00:48:09 UTC on 7 November 2026**; the nested connectivity gate +subsequently passed. No bootstrap, storage or VM stage was replayed. A later +operator-only token-resource lookup failure interrupted the handoff to guest +preparation, not the completed Azure Local deployment; monitoring/preparation +was resumed without infrastructure replay. AKS/image preparation, the reference +VM and Full health verification remain required. + +The operator's Contributor role also prevented the normal container-scoped Blob +role assignments. After explicit authorization, the Client managed identity +created and verified only the two required **Storage Blob Data Contributor** +grants on the private automation container at **05:43:10 UTC on 7 November**. +Operator Blob data access was verified before continuing guest preparation. +No broader roles were granted. These observed times include operator repair and +authorization pauses; they are not an uninterrupted cold-deployment benchmark. + If an older run failed with `Could not find a part of the path '/content/.../resources/prepare-localbox.ps1'`, it reached ARM completion but could not package guest automation. Re-import the corrected branch (including diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-startup.tests.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-startup.tests.ps1 index 143c7874d..631656648 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-startup.tests.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-startup.tests.ps1 @@ -7,6 +7,7 @@ BeforeAll { $subscription = '20000000-0000-0000-0000-000000000001' function Update-MhhToken {} function Get-ScheduledTask { param($TaskName) } + function Get-ScheduledTaskInfo { param([Parameter(ValueFromPipeline)]$InputObject) } $imageScript = @' Write-Host 'before' # Download LocalBox VHDs @@ -273,6 +274,43 @@ throw 'Timeout waiting for required bootstrap paths' { Invoke-LocalBoxGuardedStartup $id $startupRoot } | Should -Throw '*fixture-required-operation-failed*' (Get-Content $stateFile -Raw | ConvertFrom-Json).State | Should -Be 'Failed' } + It 'preserves module error semantics and restores global preference for ' -TestCases @( + @{ Outcome = 'Continue'; Ending = 'Set-Content "$PSScriptRoot/module-readiness-checked" yes' } + @{ Outcome = 'Throw'; Ending = 'throw "fixture-module-readiness-failed"' } + @{ Outcome = 'ExplicitStop'; Ending = 'Write-Error "fixture-module-required-operation-failed" -ErrorAction Stop' } + ) { + param($Outcome, $Ending) + $moduleName = 'FixtureLocalBoxBootstrap' + @' +function Invoke-FixtureBootstrap { + Write-Error 'ImageCustomizationScheduledTask is absent' + __ENDING__ +} +'@.Replace('__ENDING__', $Ending) | Set-Content "$startupRoot/$moduleName.psm1" + @' +Import-Module "$PSScriptRoot/FixtureLocalBoxBootstrap.psm1" -Force +Invoke-FixtureBootstrap +'@ | Set-Content "$startupRoot/LocalBoxLogonScript.ps1" + $previousGlobalPreference = $global:ErrorActionPreference + try { + $global:ErrorActionPreference = 'Stop' + if ($Outcome -eq 'Continue') { + $result = @(Invoke-LocalBoxGuardedStartup $id $startupRoot 2>&1) + ($result | Out-String) | Should -Match 'ImageCustomizationScheduledTask' + Get-Content "$startupRoot/module-readiness-checked" | Should -Be 'yes' + (Get-Content $stateFile -Raw | ConvertFrom-Json).State | Should -Be 'Succeeded' + } + else { + { Invoke-LocalBoxGuardedStartup $id $startupRoot 2>$null } | Should -Throw '*fixture-module-*' + (Get-Content $stateFile -Raw | ConvertFrom-Json).State | Should -Be 'Failed' + } + $global:ErrorActionPreference | Should -Be 'Stop' + } + finally { + $global:ErrorActionPreference = $previousGlobalPreference + Remove-Module $moduleName -ErrorAction Stop + } + } It 'records a bounded redacted error rather than only a generic failure' { Set-Content "$startupRoot/LocalBoxLogonScript.ps1" 'throw (''Missing task; password=fixture-sensitive token=fixture-token '' + (''x'' * 1000))' { Invoke-LocalBoxGuardedStartup $id $startupRoot } | Should -Throw @@ -408,6 +446,7 @@ Describe 'Startup report validation' { $prerequisitePath = "$TestDrive/prerequisites.json" Mock Get-Date { $prerequisiteClock.Now } Mock Get-ScheduledTask { @() } + Mock Get-ScheduledTaskInfo { @{ LastRunTime = [datetime]'1999-01-01'; LastTaskResult = 267011 } } Mock Start-Sleep { $prerequisiteClock.Sleeps++ $prerequisiteClock.Now = $prerequisiteClock.Now.AddSeconds(15) @@ -452,6 +491,13 @@ Describe 'Startup report validation' { { Wait-LocalBoxStartupPrerequisites $prerequisiteStatus $prerequisitePath } | Should -Throw '*fixture-scheduler-unavailable*' Should -Invoke Resolve-LocalBoxAzCopy -Times 0 } + It 'fails promptly when WinGet exited unsuccessfully instead of waiting an hour' { + Mock Get-ScheduledTask { @{ TaskName = 'WinGetLogonScript'; TaskPath = '\'; State = 'Ready' } } + Mock Get-ScheduledTaskInfo { @{ LastRunTime = [datetime]'2026-11-06T00:00:00Z'; LastTaskResult = 1 } } + { Wait-LocalBoxStartupPrerequisites $prerequisiteStatus $prerequisitePath } | Should -Throw '*task result 1*' + Should -Invoke Start-Sleep -Times 0 + Should -Invoke Resolve-LocalBoxAzCopy -Times 0 + } } Describe 'AzCopy resolution with a stale process PATH' { diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-validation-retry.tests.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-validation-retry.tests.ps1 index d8a926c7e..b5cbe0a0a 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-validation-retry.tests.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-validation-retry.tests.ps1 @@ -14,6 +14,21 @@ BeforeAll { } } +Describe 'Run Command Azure CLI environment' { + BeforeEach { $previousPath = $env:PATH } + AfterEach { $env:PATH = $previousPath } + It 'keeps the existing PATH and adds persisted machine and user installations' { + $env:PATH = 'existing-bin' + Update-LocalBoxValidationRetryPath -MachinePath 'machine-cli-bin' -UserPath 'user-bin' + $env:PATH | Should -Be (@('existing-bin', 'machine-cli-bin', 'user-bin') -join [IO.Path]::PathSeparator) + } + It 'ignores absent persisted paths and repeated identical values' { + $env:PATH = 'existing-bin' + Update-LocalBoxValidationRetryPath -MachinePath '' -UserPath 'existing-bin' + $env:PATH | Should -Be 'existing-bin' + } +} + Describe 'Validation failure classification' { It 'recognizes the reported content-download timeout, including nested ARM errors' { Test-LocalBoxValidationDownloadTimeout $downloadError | Should -BeTrue @@ -147,6 +162,16 @@ Describe 'Bounded Console validation recovery' { { Wait-LocalBoxNestedDeployment -SubscriptionId $testSubscription -ResourceGroupName $testGroup } | Should -Throw '*Automatic validation retry failed*' $recoveryTest.Submitted | Should -Be 0 } + It 'reports bounded redacted recovery errors without allowing another retry' { + $command = @{ properties = @{ provisioningState = 'Succeeded'; parameters = $retryParameters + instanceView = @{ executionState = 'Failed'; exitCode = 1; startTime = [datetime]::UtcNow.ToString('o') + error = "Azure CLI unavailable; token=fixture-secret " + ('x' * 2000) } } } + $failure = { Get-LocalBoxValidationRetryState $command $testSubscription $testGroup } | + Should -Throw '*Azure CLI unavailable*' -PassThru + $failure.Exception.Message | Should -Not -Match 'fixture-secret' + $failure.Exception.Message | Should -Match 'No further automatic retry' + $failure.Exception.Message.Length | Should -BeLessThan 1100 + } It 'rejects a second failed validation after successful recovery execution rather than looping' { $recoveryTest.Existing = @{ properties = @{ provisioningState = 'Succeeded'; parameters = $retryParameters } } Mock Invoke-LocalBoxAutomationAz { @@ -185,6 +210,7 @@ Describe 'Validation recovery on LocalBox-Client' { @{ parameters = @{ deploymentMode = @{ allowedValues = @('Validate', 'Deploy') } } } | ConvertTo-Json -Depth 10 | Set-Content "$templateDir/azlocal.json" Mock Start-Sleep {} + Mock Update-LocalBoxValidationRetryPath {} Mock Invoke-LocalBoxValidationRetryAz { $guestRetryTest.Profile = $env:AZURE_CONFIG_DIR switch ($Arguments[2]) { @@ -218,6 +244,7 @@ Describe 'Validation recovery on LocalBox-Client' { $before = (Get-FileHash "$templateDir/azlocal.parameters.json").Hash $previousProfile = $env:AZURE_CONFIG_DIR Invoke-LocalBoxValidationRetry @guestArguments + Should -Invoke Update-LocalBoxValidationRetryPath -Times 1 -Exactly Should -Invoke Start-Sleep -Times 1 -Exactly -ParameterFilter { $Seconds -eq 120 } $guestRetryTest.Creates.Count | Should -Be 2 $guestRetryTest.Creates[0] | Should -Contain 'deploymentMode=Validate' @@ -274,6 +301,13 @@ Describe 'Validation recovery on LocalBox-Client' { { Invoke-LocalBoxValidationRetry @guestArguments } | Should -Throw '*file is missing*' Should -Invoke Invoke-LocalBoxValidationRetryAz -Times 0 } + It 'fails before waiting or submitting anything when CLI remains unavailable' { + Mock Get-Command { throw 'Azure CLI is unavailable after PATH refresh' } -ParameterFilter { $Name -eq 'az' } + { Invoke-LocalBoxValidationRetry @guestArguments } | Should -Throw '*CLI is unavailable*' + Should -Invoke Update-LocalBoxValidationRetryPath -Times 1 -Exactly + Should -Invoke Start-Sleep -Times 0 + Should -Invoke Invoke-LocalBoxValidationRetryAz -Times 0 + } } Describe 'Recovery CLI failure handling' { diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-winget.tests.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-winget.tests.ps1 new file mode 100644 index 000000000..94b3eb663 --- /dev/null +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-winget.tests.ps1 @@ -0,0 +1,166 @@ +BeforeAll { + . "$PSScriptRoot/../../labautomation/localbox-startup.ps1" + . "$PSScriptRoot/../../labautomation/localbox-automation.ps1" + function Repair-WinGetPackageManager { param($Version, [switch]$AllUsers, [switch]$Force) } + function Get-WinGetVersion {} + function Get-WinGetPackage {} +} + +Describe 'WinGet repair and compatibility gate' { + BeforeEach { + Mock Get-Module { @{ Version = [version]'1.29.380' } } -ParameterFilter { $Name -eq 'Microsoft.WinGet.Client' } + Mock Import-Module {} -ParameterFilter { $Name -eq 'Microsoft.WinGet.Client' } + Mock Repair-WinGetPackageManager {} + Mock Get-WinGetVersion { 'v1.29.380' } + Mock Get-WinGetPackage { @{ Id = 'fixture' } } + Mock Start-Sleep {} + } + It 'repairs the CLI to match the installed client and verifies COM enumeration' { + Initialize-LocalBoxWinGet + Should -Invoke Repair-WinGetPackageManager -Times 1 -Exactly -ParameterFilter { $Version -eq '1.29.380' -and $AllUsers -and $Force } + Should -Invoke Get-WinGetPackage -Times 1 -Exactly + } + It 'retries the observed HTTP 504 instead of continuing with an incompatible CLI' { + $repairCounter = @{ Value = 0 } + Mock Repair-WinGetPackageManager { + $repairCounter.Value++ + if ($repairCounter.Value -eq 1) { throw '504 Gateway Time-out' } + } + Initialize-LocalBoxWinGet + Should -Invoke Repair-WinGetPackageManager -Times 2 -Exactly + Should -Invoke Start-Sleep -Times 1 -Exactly -ParameterFilter { $Seconds -eq 30 } + Should -Invoke Get-WinGetPackage -Times 1 -Exactly + } + It 'stops after three failed repairs and never proceeds to package enumeration' { + Mock Repair-WinGetPackageManager { throw '504 Gateway Time-out' } + { Initialize-LocalBoxWinGet } | Should -Throw '*504*' + Should -Invoke Repair-WinGetPackageManager -Times 3 -Exactly + Should -Invoke Get-WinGetPackage -Times 0 + } + It 'rejects a stale CLI version even when repair reports success' { + Mock Get-WinGetVersion { 'v1.11.510' } + { Initialize-LocalBoxWinGet } | Should -Throw '*does not match*' + Should -Invoke Get-WinGetPackage -Times 0 + } + It 'rejects the exact RPC activation failure despite matching version strings' { + Mock Get-WinGetPackage { throw 'Failed to create instance: -2147023174' } + { Initialize-LocalBoxWinGet } | Should -Throw '*-2147023174*' + Should -Invoke Get-WinGetPackage -Times 3 -Exactly + } +} + +Describe 'Native WinGet configuration outcomes' { + BeforeEach { + $oldLocalData = $env:LOCALAPPDATA + $env:LOCALAPPDATA = $TestDrive + New-Item -ItemType Directory -Path "$TestDrive/Microsoft/WindowsApps" -Force | Out-Null + Set-Content "$TestDrive/Microsoft/WindowsApps/winget.exe" 'fixture' + $configPath = "$TestDrive/packages.dsc.yml" + Set-Content $configPath 'fixture' + $wingetRun = @{ Complete = $true; Killed = $false; Disposed = $false } + $native = [pscustomobject]@{ ExitCode = 0; HasExited = $true } + $native | Add-Member ScriptMethod WaitForExit { param($milliseconds) return $wingetRun.Complete -or $wingetRun.Killed } + $native | Add-Member ScriptMethod Kill { param($tree) $wingetRun.Killed = $true; $this.HasExited = $true } + $native | Add-Member ScriptMethod Dispose { $wingetRun.Disposed = $true } + Mock Start-Process { $native } + Mock Start-Sleep {} + } + AfterEach { $env:LOCALAPPDATA = $oldLocalData } + It 'accepts only a native zero exit and retains private logs' { + Invoke-LocalBoxWinGetConfiguration -Path $configPath -LogDirectory $TestDrive + Should -Invoke Start-Process -Times 1 -Exactly -ParameterFilter { $RedirectStandardOutput -like "$TestDrive/*stdout.log" -and $RedirectStandardError -like "$TestDrive/*stderr.log" } + $wingetRun.Disposed | Should -BeTrue + } + It 'does not treat the observed DSC failure exit as success' { + $native.ExitCode = -1978286075 + { Invoke-LocalBoxWinGetConfiguration -Path $configPath -LogDirectory $TestDrive } | Should -Throw '*-1978286075*' + Should -Invoke Start-Process -Times 3 -Exactly + Should -Invoke Start-Sleep -Times 2 -Exactly + } + It 'rejects a missing native exit code' { + $native.ExitCode = $null + { Invoke-LocalBoxWinGetConfiguration -Path $configPath -LogDirectory $TestDrive -Attempts 1 } | Should -Throw '*exit code*' + } + It 'bounds the native wait and terminates only its owned process' { + $wingetRun.Complete = $false + $native.HasExited = $false + { Invoke-LocalBoxWinGetConfiguration -Path $configPath -LogDirectory $TestDrive -Attempts 1 } | Should -Throw '*900-second*' + $wingetRun.Killed | Should -BeTrue + $wingetRun.Disposed | Should -BeTrue + } +} + +Describe 'Upstream WinGet hardening transform' { + BeforeEach { + $path = "$TestDrive/WinGet.ps1" + @' +$ErrorActionPreference = $env:ErrorActionPreference +Repair-WinGetPackageManager -AllUsers -Force -Latest -Verbose +Repair-WinGetPackageManager -AllUsers -Force -Latest -Verbose +& $winget configure --file "$($Env:LocalBoxDir)\DSC\packages.dsc.yml" --accept-configuration-agreements --disable-interactivity +& $winget configure --file "$($Env:LocalBoxDir)\DSC\hyper-v.dsc.yml" --accept-configuration-agreements --disable-interactivity +Get-ScheduledTask *LogonScript* | Start-ScheduledTask +Unregister-ScheduledTask -TaskName 'WinGetLogonScript' -Confirm:$false +'@ | Set-Content $path + } + + Describe 'Retired package removal' { + BeforeEach { + $path = "$TestDrive/packages.dsc.yml" + $supported = @' +properties: + resources: + - resource: Microsoft.WinGet.DSC/WinGetPackage + id: azcopy + settings: + id: Microsoft.Azure.AZCopy.10 + source: winget +'@ + $retired = @' + - resource: Microsoft.WinGet.DSC/WinGetPackage + id: Microsoft.AzureDataStudio + directives: + description: Install Microsoft Azure Data Studio + settings: + id: Microsoft.AzureDataStudio + source: winget +'@ + "$supported`n$retired" | Set-Content $path + } + It 'removes only the exact retired resource and preserves supported tools' { + Set-LocalBoxPackageConfiguration $path + $text = Get-Content $path -Raw + $text | Should -Not -Match 'AzureDataStudio' + $text.TrimEnd() | Should -Be $supported.TrimEnd() + Set-LocalBoxPackageConfiguration $path + (Get-Content $path -Raw).TrimEnd() | Should -Be $supported.TrimEnd() + } + It 'handles Windows CRLF configurations' { + ((Get-Content $path -Raw).Replace("`n", "`r`n")) | Set-Content $path + Set-LocalBoxPackageConfiguration $path + Get-Content $path -Raw | Should -Not -Match 'AzureDataStudio' + } + It 'rejects unexpected changes to the retired package block without modifying it' { + (Get-Content $path -Raw).Replace('Install Microsoft Azure Data Studio', 'Unknown changed block') | Set-Content $path + $before = Get-Content $path -Raw + { Set-LocalBoxPackageConfiguration $path } | Should -Throw '*Unrecognized retired*' + Get-Content $path -Raw | Should -Be $before + } + } + It 'protects repair and both native configuration calls before upstream task startup' { + Set-LocalBoxWinGetGuard $path + $text = Get-Content $path -Raw + $text | Should -Match '\$ErrorActionPreference = ''Stop''' + $text | Should -Match 'Initialize-LocalBoxWinGet' + $text | Should -Not -Match 'Repair-WinGetPackageManager|& \$winget configure' + [regex]::Matches($text, 'Invoke-LocalBoxWinGetConfiguration').Count | Should -Be 2 + $text.IndexOf('Invoke-LocalBoxWinGetConfiguration') | Should -BeLessThan $text.IndexOf('Start-ScheduledTask') + $parseErrors = $null + [void][Management.Automation.Language.Parser]::ParseFile($path, [ref]$null, [ref]$parseErrors) + $parseErrors.Count | Should -Be 0 + } + It 'rejects upstream drift rather than running unprotected configuration' { + (Get-Content $path -Raw).Replace('hyper-v.dsc.yml', 'different.yml') | Set-Content $path + { Set-LocalBoxWinGetGuard $path } | Should -Throw '*Unsupported upstream WinGet*' + } +} diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md index 27693da48..8885cad45 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md @@ -58,6 +58,26 @@ required to validate regional service availability and effective inherited polic ## Global event LocalBox checks +Run `Invoke-Pester ./localbox-winget.tests.ps1,./localbox-startup.tests.ps1,./localbox-deployment.tests.ps1 -Output Detailed` +for WinGet provisioning failures. These checks cover bounded HTTP 504 retries, +CLI/module version mismatch, COM activation failure, nonzero/missing native exit +codes, process timeout cleanup, prerequisite-task failure propagation, exact +upstream transformations and removal of only the retired Azure Data Studio +resource. Real upstream Bootstrap/WinGet scripts and the package configuration +are also transformed and syntax/compilation checked offline. + +The combined WinGet, startup, deployment, automation, validation-retry, global +LocalBox and failure-summary suites passed **267 tests** after these changes. +The real upstream Bicep compiled with the 30,679-byte guarded payload, and +existing-VM custom-data preservation was verified. + +Live TEST12 repair verified matching CLI/Client 1.29.380, successful COM +enumeration, zero-exit package configuration after the retired-entry removal, +and executable AzCopy. After safe resume, both base-image SHA256 checksums +passed at 18:15:09 UTC on 6 November 2026. The current instance is being used for the remaining +nested deployment and Full health gates; these prerequisite checks alone do not +establish end-to-end readiness. + Run `Invoke-Pester ./localbox-startup.tests.ps1,./localbox-deployment.tests.ps1 -Output Detailed` for future-only startup protection. Offline checks cover new/existing VM custom-data handling, immutable identity retention, strict upstream source @@ -92,11 +112,18 @@ strict download/checksum failures and the final readiness gates remain covered. Read-only TEST11 diagnostics confirmed the missing customization task and both required bootstrap-module directories on each node; the full resumed deployment was not executed. -A PowerShell 7 probe on the same Client also reproduced the lookup failure under -the old strict boundary, then used the actual corrected boundary to reach and -pass those checks on both nodes. Both lookup errors remained visible and the -parent preference stayed strict. The probe only queried tasks and paths; it did -not start tasks, replay initialization or resume deployment. +A PowerShell 7 probe on that Client reproduced the lookup failure under the old +strict boundary and passed both node checks with function-local `Continue`. +However, it did not model imported-module preference resolution: TEST12 still +failed because the module resolved global `Stop`. The corrected boundary now +temporarily sets global `Continue` too and restores it in `finally`. Three +regressions exercise a real imported module, including explicit `Stop` and +terminating readiness failures. A new read-only TEST12 probe reproduced the +module-scoped failure and verified the corrected boundary against both nodes and +the installed module's readiness helper. Both lookup errors remained visible and +the global preference was restored. Neither probe started tasks or replayed +initialization; the separate authorized step-9 continuation is still subject to +all deployment and Full health gates. Run `Invoke-Pester ./localbox-failure-summary.tests.ps1,./localbox-automation.tests.ps1 -Output Detailed` for Console failure reporting. Tests cover run/content/group/scope correlation, @@ -114,12 +141,21 @@ the later passing MetalLB repair reports were not used as the failure fixture. Run `Invoke-Pester ./localbox-validation-retry.tests.ps1 -Output Detailed` for the bounded Azure Local validation recovery. These mocked tests cover the exact content-download timeout classifier, the unchanged nested wait budget, one -asynchronous recovery, Console restart/resume, stale execution state, error +asynchronous recovery, Console restart/resume, stale execution state, persisted +PATH refresh, missing-CLI failure before submission, bounded/redacted error propagation, existing local-file/correlation checks, managed-identity CLI-profile cleanup, and validation-before-deploy ordering. They never submit Azure deployments. The reported traininglab-58 failure could not be read with the -available operator permissions; real retry and subsequent cluster deployment -remain a fresh-Console integration check. +available operator permissions. TEST12 later exercised the real timeout path, +but its retry initially failed before submission because the VM agent retained +a PATH from before Azure CLI installation. A read-only live SYSTEM probe +verified the actual repaired helpers, native CLI, managed-identity access and +matching existing template/parameter contract. The repaired bounded retry +successfully validated that instance at 22:25:07 UTC on 6 November 2026 and +submitted deployment. ARM deployment succeeded at 00:48:09 UTC on 7 November, +and the nested connectivity gate passed. Guest preparation and Full health +verification remain pending; this is resumed-instance evidence, not proof of a +successful fresh Console deployment. Run `Invoke-Pester ./localbox-deployment.tests.ps1 -Output Detailed` for the LocalBox Developer Bastion deployment contract. These tests require Azure CLI From 32397bff86bd7547294bc3613c189549a3d040e3 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Sat, 7 Nov 2026 06:03:18 +0000 Subject: [PATCH 21/37] Reconcile accepted AKS deployments after no-wait visibility errors Require new scoped ARM correlation before accepting the observed aksarc post-submission 404. Resume existing accepted deployments without recreating AKS and retain image overlap and final readiness gates. Add regression tests and live evidence. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../resources/hosted-events/readme.md | 18 +++- .../resources/prepare-localbox.ps1 | 52 ++++++++- .../tests/prepare-localbox.tests.ps1 | 102 ++++++++++++++++++ .../resources/tests/readme.md | 9 ++ 4 files changed, 179 insertions(+), 2 deletions(-) diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md index 59a8d989c..710556057 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md @@ -175,7 +175,15 @@ The [orchestrator](../../labautomation/localbox-automation.ps1) implements: proxy access and MetalLB before creating the reference VM. The final Full readiness gate is unchanged. - Existing AKS deployments retain verification-before-image ordering on retries. + Completed or legacy existing AKS deployments retain verification-before-image + ordering on retries. A matching accepted/running `pciDeployment-` + is instead reused while the image imports; AKS creation is not repeated. + `aksarc` 2.7.1 can return `ResourceNotFound` after submitting this deployment + because it immediately reads resources that ARM has not created yet. Only + that cluster-specific error can be reconciled, and only against a new + correlation with matching cluster, custom location and region in an + accepted/running/succeeded ARM deployment. Missing, stale, failed, conflicting + or unreadable evidence still fails; this confirms submission, not readiness. Manual preparation supports `-SequentialProvisioning` to restore serial provisioning if contention is suspected. `-SkipAks` and `-WhatIf` retain their existing behavior. If preparation fails after asynchronous submission, AKS may @@ -183,6 +191,14 @@ The [orchestrator](../../labautomation/localbox-automation.ps1) implements: back. Inspect both resource states before retrying rather than deleting the environment. The parent guest timeout still applies. Compare equivalent cold deployments before treating the potential ~71-minute saving as measured. + + TEST12 exercised this race on 7 November 2026: `aksarc` reported a missing + cluster at **05:51:31 UTC**, while its ARM deployment was already running + and created the connected-cluster parent at **05:51:34 UTC**. The child + provisioning operation continued in Azure. A read-only live probe verified + the new submission-reconciliation helper against that same running deployment + at **06:01:19 UTC**, without issuing another create. Full health remained + unverified; the failed guest report was not reclassified as successful. 5. Wait for the reference VM to be Running, Arc Connected and MDE installation Succeeded. Request an Update Manager assessment and require a successful, timestamped result. This assesses updates; it does **not** install patches. diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/prepare-localbox.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/prepare-localbox.ps1 index 8cae15df9..3514e6d7b 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/prepare-localbox.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/prepare-localbox.ps1 @@ -973,6 +973,30 @@ function Initialize-LocalBoxStorage { } } +function Get-LocalBoxAksDeployment { + param([string]$SubscriptionId, [string]$ResourceGroupName, [string]$ClusterName, + [string]$CustomLocationId, [string]$Location) + $scope = @('--subscription', $SubscriptionId, '--resource-group', $ResourceGroupName) + $name = "pciDeployment-$ClusterName" + $existing = @((Invoke-LocalBoxAz (@('deployment', 'group', 'list') + $scope)) | Where-Object name -eq $name) + if (-not $existing.Count) { return } + if ($existing.Count -ne 1) { throw 'Ambiguous AKS ARM deployment records.' } + $deployment = Invoke-LocalBoxAz (@('deployment', 'group', 'show') + $scope + @('--name', $name)) + $parameters = $deployment.properties.parameters + $clusterId = "/subscriptions/$SubscriptionId/resourceGroups/$ResourceGroupName/providers/Microsoft.Kubernetes/connectedClusters/$ClusterName" + $correlation = [guid]::Empty + if ($deployment.name -ne $name -or + $parameters.clusterId.value -ine $clusterId -or $parameters.customLocation.value -ine $CustomLocationId -or + $parameters.provisionedClustersName.value -ne $ClusterName -or $parameters.location.value -ine $Location -or + -not [guid]::TryParse([string]$deployment.properties.correlationId, [ref]$correlation) -or $correlation -eq [guid]::Empty) { + throw 'AKS ARM deployment does not match the requested cluster, location or correlation contract.' + } + if ($deployment.properties.provisioningState -notin @('Accepted', 'Running', 'Succeeded')) { + throw "AKS ARM deployment $name is not accepted or successful: $($deployment.properties.provisioningState). Inspect its deployment operations." + } + return $deployment +} + function Complete-LocalBoxAksPreparation { [CmdletBinding(SupportsShouldProcess)] param( @@ -1119,7 +1143,18 @@ function Invoke-LocalBoxPreparation { agentPoolProfiles = @(@{ count = $Settings.NodeCount; vmSize = $Settings.NodeVmSize }) } } if ($Settings.KubernetesVersion) { $aksExpected.properties.kubernetesVersion = $Settings.KubernetesVersion } + $aksDeploymentArguments = @{ + SubscriptionId = $Settings.SubscriptionId; ResourceGroupName = $Settings.ResourceGroupName + ClusterName = $Settings.AksClusterName; CustomLocationId = $custom.id; Location = $location + } $existingAks = @((Invoke-LocalBoxAz @('resource', 'list', '--resource-group', $Settings.ResourceGroupName)) | Where-Object id -ieq $aksId) + if (-not $Settings.SkipAks -and $existingAks.Count -and -not $Settings.SequentialProvisioning -and -not $WhatIfPreference) { + $deployment = Get-LocalBoxAksDeployment @aksDeploymentArguments + if ($deployment -and $deployment.properties.provisioningState -in @('Accepted', 'Running')) { + $aksVerificationPending = $true + Write-Host "Resuming accepted AKS deployment $($deployment.name); VM image import can overlap it. No AKS create will be repeated." + } + } if (-not $Settings.SkipAks -and -not $existingAks.Count) { $arguments = @('aksarc', 'create') + $common + @('--name', $Settings.AksClusterName, '--vnet-ids', $networks[1].Id, '--aad-admin-group-object-ids', $groupId, '--generate-ssh-keys', '--control-plane-ip', $config.AKSControlPlaneIP, @@ -1132,7 +1167,22 @@ function Invoke-LocalBoxPreparation { if (-not $Settings.SequentialProvisioning) { # Azure provisions asynchronously; local CLI commands still run serially. $aksVerificationPending = $true - Invoke-LocalBoxAz ($arguments + @('--no-wait')) -TimeoutSeconds $timeout -NoOutput + $previousDeployment = Get-LocalBoxAksDeployment @aksDeploymentArguments + try { + Invoke-LocalBoxAz ($arguments + @('--no-wait')) -TimeoutSeconds $timeout -NoOutput + } + catch { + $failure = $_ + if ($failure.Exception.Message -notmatch '\(ResourceNotFound\)' -or + $failure.Exception.Message -notmatch [regex]::Escape("Microsoft.Kubernetes/connectedClusters/$($Settings.AksClusterName)")) { + throw + } + # aksarc 2.7.1 reads the child immediately after submitting a no-wait ARM deployment. + $submitted = Get-LocalBoxAksDeployment @aksDeploymentArguments + if (-not $submitted -or ($previousDeployment -and + $submitted.properties.correlationId -eq $previousDeployment.properties.correlationId)) { throw $failure } + Write-Warning "aksarc reported ResourceNotFound after submission, but a new matching ARM deployment $($submitted.name) is $($submitted.properties.provisioningState). No create is retried; full AKS readiness remains required." + } Write-Host "AKS creation submitted at $([datetime]::UtcNow.ToString('o')); provisioning will overlap VM image import. AKS readiness is not yet verified." } else { diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/prepare-localbox.tests.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/prepare-localbox.tests.ps1 index aa273e2dc..eef41bbcd 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/prepare-localbox.tests.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/prepare-localbox.tests.ps1 @@ -2032,6 +2032,61 @@ Describe 'LocalBox failed image recovery' { } } +Describe 'AKS ARM submission evidence' { + BeforeEach { + $arguments = @{ SubscriptionId = 'test'; ResourceGroupName = 'localbox'; ClusterName = 'localbox-aks' + CustomLocationId = 'custom'; Location = 'australiaeast' } + $deployment = @{ name = 'pciDeployment-localbox-aks'; properties = @{ + provisioningState = 'Running'; correlationId = '10000000-0000-0000-0000-000000000001' + parameters = @{ + clusterId = @{ value = '/subscriptions/test/resourceGroups/localbox/providers/Microsoft.Kubernetes/connectedClusters/localbox-aks' } + customLocation = @{ value = 'custom' }; provisionedClustersName = @{ value = 'localbox-aks' } + location = @{ value = 'australiaeast' } + } + } } + Mock Invoke-LocalBoxAz { $deployment } + } + It 'accepts only scoped correlated ARM submission states' -TestCases @( + @{ State = 'Accepted' }, @{ State = 'Running' }, @{ State = 'Succeeded' } + ) { + param($State) + $deployment.properties.provisioningState = $State + (Get-LocalBoxAksDeployment @arguments).properties.correlationId | Should -Be $deployment.properties.correlationId + Should -Invoke Invoke-LocalBoxAz -Times 1 -Exactly -ParameterFilter { + $Arguments[2] -eq 'show' -and $Arguments -contains '--subscription' -and $Arguments -contains 'test' + } + } + It 'does not invent submission evidence when no deployment exists' { + Mock Invoke-LocalBoxAz {} + Get-LocalBoxAksDeployment @arguments | Should -BeNullOrEmpty + Should -Invoke Invoke-LocalBoxAz -Times 0 -ParameterFilter { $Arguments[2] -eq 'show' } + } + It 'rejects failed canceled and unknown states' -TestCases @( + @{ State = 'Failed' }, @{ State = 'Canceled' }, @{ State = '' } + ) { + param($State) + $deployment.properties.provisioningState = $State + { Get-LocalBoxAksDeployment @arguments } | Should -Throw '*not accepted or successful*' + } + It 'rejects mismatched deployment parameters' -TestCases @( + @{ Field = 'clusterId' }, @{ Field = 'customLocation' }, @{ Field = 'provisionedClustersName' }, @{ Field = 'location' } + ) { + param($Field) + $deployment.properties.parameters[$Field].value = 'another-target' + { Get-LocalBoxAksDeployment @arguments } | Should -Throw '*does not match*' + } + It 'rejects missing correlation and ambiguous records' { + $deployment.properties.correlationId = '' + { Get-LocalBoxAksDeployment @arguments } | Should -Throw '*correlation contract*' + Mock Invoke-LocalBoxAz { $deployment; $deployment } -ParameterFilter { $Arguments[2] -eq 'list' } + { Get-LocalBoxAksDeployment @arguments } | Should -Throw '*Ambiguous*' + } + It 'propagates unreadable deployment evidence' { + Mock Invoke-LocalBoxAz { throw 'AuthorizationFailed' } + { Get-LocalBoxAksDeployment @arguments } | Should -Throw '*AuthorizationFailed*' + } +} + Describe 'LocalBox preparation dependency order' { BeforeAll { $source = ${function:Invoke-LocalBoxPreparation}.ToString() @@ -2106,6 +2161,53 @@ Describe 'LocalBox preparation dependency order' { Should -Invoke Sync-LocalBoxImage -Times 0 Should -Invoke Sync-LocalBoxReferenceVm -Times 0 } + It 'reconciles the observed no-wait 404 only against a new accepted deployment' { + $receipt = @{ Reads = 0 } + Mock Get-LocalBoxAksDeployment { + $receipt.Reads++ + if ($receipt.Reads -gt 1) { + @{ name = 'pciDeployment-localbox-aks'; properties = @{ provisioningState = 'Running'; correlationId = 'new' } } + } + } + Mock Invoke-LocalBoxAz { + $events.Add('aks-create') + throw "(ResourceNotFound) The Resource 'Microsoft.Kubernetes/connectedClusters/localbox-aks' was not found." + } -ParameterFilter { $Arguments -contains '--no-wait' } + & $prepareWorkloads -Confirm:$false + $events -join ',' | Should -Be 'network:localbox-vm-lnet-vlan200,network:localbox-aks-lnet-vlan110,aks-validate,aks-create,image,proxy,metallb,reference' + Should -Invoke Invoke-LocalBoxAz -Times 1 -Exactly -ParameterFilter { $Arguments -contains '--no-wait' } + Should -Invoke Get-LocalBoxAksDeployment -Times 2 -Exactly + } + It 'does not treat missing or stale deployment evidence as an accepted submission' -TestCases @( + @{ Existing = $false }, @{ Existing = $true } + ) { + param($Existing) + Mock Get-LocalBoxAksDeployment { + if ($Existing) { @{ properties = @{ correlationId = 'unchanged' } } } + } + Mock Invoke-LocalBoxAz { + throw "(ResourceNotFound) The Resource 'Microsoft.Kubernetes/connectedClusters/localbox-aks' was not found." + } -ParameterFilter { $Arguments -contains '--no-wait' } + { & $prepareWorkloads -Confirm:$false } | Should -Throw '*ResourceNotFound*' + Should -Invoke Sync-LocalBoxImage -Times 0 + Should -Invoke Invoke-LocalBoxAz -Times 1 -Exactly -ParameterFilter { $Arguments -contains '--no-wait' } + } + It 'does not reconcile a 404 for an unrelated resource' { + Mock Get-LocalBoxAksDeployment {} + Mock Invoke-LocalBoxAz { throw '(ResourceNotFound) Microsoft.AzureStackHCI/logicalNetworks/missing' } -ParameterFilter { $Arguments -contains '--no-wait' } + { & $prepareWorkloads -Confirm:$false } | Should -Throw '*logicalNetworks/missing*' + Should -Invoke Get-LocalBoxAksDeployment -Times 1 -Exactly + Should -Invoke Sync-LocalBoxImage -Times 0 + } + It 'overlaps image import with an existing accepted AKS deployment without recreating it' { + Mock Invoke-LocalBoxAz { + @{ id = "$scope/providers/Microsoft.Kubernetes/connectedClusters/localbox-aks" } + } -ParameterFilter { $Arguments[0] -eq 'resource' } + Mock Get-LocalBoxAksDeployment { @{ name = 'pciDeployment-localbox-aks'; properties = @{ provisioningState = 'Running' } } } + & $prepareWorkloads -Confirm:$false + $events -join ',' | Should -Be 'network:localbox-vm-lnet-vlan200,network:localbox-aks-lnet-vlan110,image,proxy,metallb,reference' + Should -Invoke Invoke-LocalBoxAz -Times 0 -ParameterFilter { $Arguments[0] -eq 'aksarc' } + } It 'does not import the image or submit AKS after failed AKS validation' { Mock Invoke-LocalBoxAz { throw 'AKS validation failed' } -ParameterFilter { $Arguments -contains '--validate' } { & $prepareWorkloads -Confirm:$false } | Should -Throw '*AKS validation failed*' diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md index 8885cad45..5187817cc 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md @@ -289,6 +289,15 @@ guards. A cold live deployment is needed to measure actual overlap and contentio mocked ordering is not a performance benchmark. Read-only verification on traininglab-02 confirmed that the installed `aksarc` extension advertises `create --no-wait`; no provisioning was invoked by that check. +TEST12 subsequently exposed an actual `aksarc` 2.7.1 race: the command submitted +ARM deployment, then immediately read the not-yet-created cluster and returned +`ResourceNotFound`. New tests require a new, scoped ARM correlation before +reconciling this specific submission error; stale/missing receipts, mismatched +parameters, failed/canceled states, unrelated 404s and unreadable evidence fail. +Existing accepted deployments can overlap image import without another create, +while completed/legacy deployments retain their prior ordering. The actual +receipt helper was verified read-only against the still-running TEST12 +deployment; this does not establish final AKS or Full health readiness. Storage tests verify no repeated disk/partition mutations on a completed rerun, resumption of an interrupted partition expansion, and detection of unallocated From 5d785aba8f2db620b11c5a4fd27d3ee35b860a89 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Sat, 7 Nov 2026 06:05:25 +0000 Subject: [PATCH 22/37] Pin verified AKS submission reconciliation guest sources Pin the complete guest source set to 32397bff86bd7547294bc3613c189549a3d040e3 and update the preparation hash. Record 501 passing preparation and automation tests. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../labautomation/localbox-automation-source.json | 4 ++-- .../01_Sovereign_Cloud/resources/tests/readme.md | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation-source.json b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation-source.json index 6f72cf677..0b448847d 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation-source.json +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation-source.json @@ -1,8 +1,8 @@ { "repository": "janegilring/MicroHack", - "commit": "2c7fad3e6e56818869bb33c07f08aa0cafd64071", + "commit": "32397bff86bd7547294bc3613c189549a3d040e3", "files": { - "prepare-localbox.ps1": "8d965246fefc72ad89652054261b0f343cb1dea1bc1d68038a8abb20cba513fd", + "prepare-localbox.ps1": "f3aa12804fb29a56ee4b087797664c9d2b22e9aa9b2e914242af3c56c21f0073", "invoke-localbox-automation.ps1": "cbcb8dbee2523497a3d6b8c0586154c01e579bce3183de9aeb10d4e3ccb7c425", "test-sovereign-cloud.ps1": "e23583ac0a3747261d57469a7ae3e31438c0cb70beb9bf1c751a6244e4ea5117", "tests/localbox.health.tests.ps1": "92f922dff75b1bcecc7c3e9a523c613930bd1a538ef8bb0fd50dd2ee1267b4d9", diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md index 5187817cc..12f29c7f6 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md @@ -298,6 +298,8 @@ Existing accepted deployments can overlap image import without another create, while completed/legacy deployments retain their prior ordering. The actual receipt helper was verified read-only against the still-running TEST12 deployment; this does not establish final AKS or Full health readiness. +The focused preparation and automation suites passed **501 tests** after the +complete five-file guest source pin was synchronized with this change. Storage tests verify no repeated disk/partition mutations on a completed rerun, resumption of an interrupted partition expansion, and detection of unallocated From 4fdcfe25d5713b459e88c5765fff6ac66156cd79 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Sat, 7 Nov 2026 06:08:09 +0000 Subject: [PATCH 23/37] Correct live aksarc version evidence and record image handoff Use the version reported by az extension show rather than a bundled Python dependency's metadata. Keep the implementation comment version-independent and record the resumed image-import stage. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../resources/hosted-events/readme.md | 10 +++++++--- .../01_Sovereign_Cloud/resources/prepare-localbox.ps1 | 2 +- .../01_Sovereign_Cloud/resources/tests/readme.md | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md index 710556057..8b778508b 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md @@ -178,7 +178,7 @@ The [orchestrator](../../labautomation/localbox-automation.ps1) implements: Completed or legacy existing AKS deployments retain verification-before-image ordering on retries. A matching accepted/running `pciDeployment-` is instead reused while the image imports; AKS creation is not repeated. - `aksarc` 2.7.1 can return `ResourceNotFound` after submitting this deployment + `aksarc` 1.5.117 can return `ResourceNotFound` after submitting this deployment because it immediately reads resources that ARM has not created yet. Only that cluster-specific error can be reconciled, and only against a new correlation with matching cluster, custom location and region in an @@ -197,8 +197,12 @@ The [orchestrator](../../labautomation/localbox-automation.ps1) implements: and created the connected-cluster parent at **05:51:34 UTC**. The child provisioning operation continued in Azure. A read-only live probe verified the new submission-reconciliation helper against that same running deployment - at **06:01:19 UTC**, without issuing another create. Full health remained - unverified; the failed guest report was not reclassified as successful. + at **06:01:19 UTC**, without issuing another create. The replacement preparation + run reused that deployment and reached image import at **06:06:03 UTC**. + The installed extension version comes from `az extension show`; recursively + inspecting the first Python `METADATA` file can instead report a dependency's + version. Full health remained unverified; the failed guest report was not + reclassified as successful. 5. Wait for the reference VM to be Running, Arc Connected and MDE installation Succeeded. Request an Update Manager assessment and require a successful, timestamped result. This assesses updates; it does **not** install patches. diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/prepare-localbox.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/prepare-localbox.ps1 index 3514e6d7b..c2d4b768d 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/prepare-localbox.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/prepare-localbox.ps1 @@ -1177,7 +1177,7 @@ function Invoke-LocalBoxPreparation { $failure.Exception.Message -notmatch [regex]::Escape("Microsoft.Kubernetes/connectedClusters/$($Settings.AksClusterName)")) { throw } - # aksarc 2.7.1 reads the child immediately after submitting a no-wait ARM deployment. + # aksarc can read the child before its no-wait ARM deployment creates it. $submitted = Get-LocalBoxAksDeployment @aksDeploymentArguments if (-not $submitted -or ($previousDeployment -and $submitted.properties.correlationId -eq $previousDeployment.properties.correlationId)) { throw $failure } diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md index 12f29c7f6..4e660038b 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md @@ -289,7 +289,7 @@ guards. A cold live deployment is needed to measure actual overlap and contentio mocked ordering is not a performance benchmark. Read-only verification on traininglab-02 confirmed that the installed `aksarc` extension advertises `create --no-wait`; no provisioning was invoked by that check. -TEST12 subsequently exposed an actual `aksarc` 2.7.1 race: the command submitted +TEST12 subsequently exposed an actual `aksarc` 1.5.117 race: the command submitted ARM deployment, then immediately read the not-yet-created cluster and returned `ResourceNotFound`. New tests require a new, scoped ARM correlation before reconciling this specific submission error; stale/missing receipts, mismatched From cf30f14e60d337c5c35c22c54c4a3cf945bf5e2b Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Sat, 7 Nov 2026 06:08:38 +0000 Subject: [PATCH 24/37] Synchronize guest pin after aksarc evidence correction Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../labautomation/localbox-automation-source.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation-source.json b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation-source.json index 0b448847d..7672e1b2f 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation-source.json +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation-source.json @@ -1,8 +1,8 @@ { "repository": "janegilring/MicroHack", - "commit": "32397bff86bd7547294bc3613c189549a3d040e3", + "commit": "4fdcfe25d5713b459e88c5765fff6ac66156cd79", "files": { - "prepare-localbox.ps1": "f3aa12804fb29a56ee4b087797664c9d2b22e9aa9b2e914242af3c56c21f0073", + "prepare-localbox.ps1": "62934e83e421df476a133e127c7ef3285dd62da8ef0fe04921800f89e5e03c69", "invoke-localbox-automation.ps1": "cbcb8dbee2523497a3d6b8c0586154c01e579bce3183de9aeb10d4e3ccb7c425", "test-sovereign-cloud.ps1": "e23583ac0a3747261d57469a7ae3e31438c0cb70beb9bf1c751a6244e4ea5117", "tests/localbox.health.tests.ps1": "92f922dff75b1bcecc7c3e9a523c613930bd1a538ef8bb0fd50dd2ee1267b4d9", From ceed77b0bb6e707507abff271b2f1c4a3562729e Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Sat, 7 Nov 2026 19:20:48 +0000 Subject: [PATCH 25/37] Record TEST12 teardown and remaining Full validation gap Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../resources/hosted-events/readme.md | 10 ++++++++++ .../01_Sovereign_Cloud/resources/tests/readme.md | 6 ++++++ 2 files changed, 16 insertions(+) diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md index 8b778508b..206f99165 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md @@ -539,6 +539,16 @@ Operator Blob data access was verified before continuing guest preparation. No broader roles were granted. These observed times include operator repair and authorization pauses; they are not an uninterrupted cold-deployment benchmark. +**TEST12 close-out:** the organizer subsequently confirmed that the original +resources were torn down. Monitoring lost access at 19:00 UTC on 7 November; +the teardown's exact time was not established. Azure Local validation, deployment +and nested connectivity were verified, but no successful Full health report was +verified before teardown. The last direct workload checks showed the existing +AKS deployment running and the VM image downloading concurrently. Recovery of +this instance has stopped; replacement resources are not assumed to be the same +deployment. Published fixes and offline tests are not proof of a successful +fresh end-to-end deployment. + If an older run failed with `Could not find a part of the path '/content/.../resources/prepare-localbox.ps1'`, it reached ARM completion but could not package guest automation. Re-import the corrected branch (including diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md index 4e660038b..f6e7d51d2 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md @@ -157,6 +157,12 @@ and the nested connectivity gate passed. Guest preparation and Full health verification remain pending; this is resumed-instance evidence, not proof of a successful fresh Console deployment. +TEST12 was subsequently torn down, as confirmed by the organizer. Recovery has +stopped. No successful Full health report was verified before teardown; the +verified intermediate stages and offline test results must not be interpreted +as an end-to-end pass. A separately authorized deployment is required to finish +that integration verification. + Run `Invoke-Pester ./localbox-deployment.tests.ps1 -Output Detailed` for the LocalBox Developer Bastion deployment contract. These tests require Azure CLI with Bicep available: they compile the network module locally, then mock Azure From 238925b6c989de13f1f957ba7aacbca5b9536039 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Sat, 7 Nov 2026 19:30:26 +0000 Subject: [PATCH 26/37] Gate LocalBox preparation on Blob write readiness Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../resources/hosted-events/readme.md | 8 +++ .../resources/invoke-localbox-automation.ps1 | 6 +- .../tests/localbox-automation.tests.ps1 | 64 ++++++++++++++++++- .../resources/tests/readme.md | 7 ++ 4 files changed, 81 insertions(+), 4 deletions(-) diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md index 206f99165..513ff8623 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md @@ -563,6 +563,14 @@ staging storage account. The Console identity and Client identity receive keys or participant data-plane grants are created. Storage firewall rules are not changed. The Console logs the authenticated result URL and run ID. +The guest's existing ten-minute Blob readiness gate verifies both listing and +the initial `status.json` upload before starting workload preparation. TEST13 +on 7 November 2026 passed listing after retries but failed that first upload +at 11:06 UTC, despite the correct container-scoped role assignment. Read access +alone is therefore insufficient evidence of write readiness during propagation. +The upload now shares the bounded retry gate; persistent failure still blocks +preparation and Full readiness. No additional permissions are granted. + Each run has a separate prefix containing `status.json`, `automation.log`, `upstream.xml`, `health.xml` and `health.json` when those phases produced them. Status is updated at phase boundaries. Failures retain available diagnostics and diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/invoke-localbox-automation.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/invoke-localbox-automation.ps1 index b5f1f5a41..89992ea24 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/invoke-localbox-automation.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/invoke-localbox-automation.ps1 @@ -333,13 +333,13 @@ function Invoke-LocalBoxGuestAutomation { Invoke-LocalBoxAz @('login', '--identity') | Out-Null Invoke-LocalBoxAz @('account', 'set', '--subscription', $Configuration.SubscriptionId) | Out-Null $azureReady = $true - # Blob role assignments may still be propagating when the guest starts. + # Listing can succeed before uploads while Blob role assignments propagate. + $status.Phase = 'Preparation' Wait-SovereignCheck -TimeoutSeconds 600 -Check { Invoke-LocalBoxAz @('storage', 'blob', 'list', '--account-name', $Configuration.StorageAccountName, '--container-name', $Configuration.Container, '--auth-mode', 'login', '--num-results', '1') | Out-Null + Write-LocalBoxAutomationStatus $Configuration $Directory $status } - $status.Phase = 'Preparation' - Write-LocalBoxAutomationStatus $Configuration $Directory $status & "$ResourceRoot/prepare-localbox.ps1" -SubscriptionId $Configuration.SubscriptionId -ResourceGroupName $Configuration.ResourceGroupName ` -AksAdminGroupObjectId $group.ToString() -ManifestPath $ManifestPath -Confirm:$false if (-not $?) { throw 'LocalBox preparation failed.' } diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-automation.tests.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-automation.tests.ps1 index adc9884f4..8fd0e5ab2 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-automation.tests.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-automation.tests.ps1 @@ -704,7 +704,9 @@ Describe 'Temporary AKS API routing for hosted health checks' { Describe 'Guest default preparation and health orchestration' { BeforeAll { function Invoke-LocalBoxAz { param($Arguments, $TimeoutSeconds, [switch]$NoOutput) } - function Wait-SovereignCheck { param($TimeoutSeconds, $Check) } + . "$resourceRoot/test-sovereign-cloud.ps1" + $realWaitSovereignCheck = ${function:Wait-SovereignCheck} + $realWriteAutomationStatus = ${function:Write-LocalBoxAutomationStatus} function Get-LocalBoxReferenceVmReadiness { param($MachineId) } function Invoke-TestPreparation { param($SubscriptionId, $ResourceGroupName, $AksAdminGroupObjectId) } $guestFixture = Join-Path $TestDrive 'guest' @@ -783,6 +785,66 @@ Invoke-TestPreparation -SubscriptionId $SubscriptionId -ResourceGroupName $Resou $status.DefenderCloudReporting | Should -Be 'NotVerified' $env:AZURE_CONFIG_DIR | Should -Be 'caller-profile' } + Context 'Blob read and write readiness' { + BeforeEach { + $script:blobWriteAttempts = 0 + $script:blobRetrySleeps = 0 + Mock Start-Sleep { + $script:blobRetrySleeps++ + if ($script:blobRetrySleeps -gt 3) { throw 'Unexpected additional Blob readiness retries in test.' } + } + Mock Wait-SovereignCheck { & $realWaitSovereignCheck -TimeoutSeconds $TimeoutSeconds -Check $Check } + Mock Write-LocalBoxAutomationStatus { & $realWriteAutomationStatus $Configuration $Directory $Status } + } + It 'waits for the initial status upload even after listing succeeds' { + Mock Invoke-LocalBoxAz { + $script:blobWriteAttempts++ + if ($script:blobWriteAttempts -le 3) { + Should -Invoke Invoke-TestPreparation -Times 0 + } + if ($script:blobWriteAttempts -le 2) { + throw 'Azure CLI storage blob failed: You do not have the required permissions needed to perform this operation.' + } + } -ParameterFilter { $Arguments[0..2] -join ' ' -eq 'storage blob upload' } + + Invoke-LocalBoxGuestAutomation $guestConfig $guestDirectory $guestFixture 'input.json' -ManifestPath $manifestFile + + Should -Invoke Invoke-TestPreparation -Times 1 -Exactly + Should -Invoke Start-Sleep -Times 2 -Exactly -ParameterFilter { $Seconds -eq 15 } + Should -Invoke Wait-SovereignCheck -Times 1 -Exactly -ParameterFilter { $TimeoutSeconds -eq 600 } + Should -Invoke Invoke-LocalBoxAz -Times 3 -Exactly -ParameterFilter { + $Arguments[0..2] -join ' ' -eq 'storage blob list' + } + (Get-Content "$guestDirectory/status.json" -Raw | ConvertFrom-Json).State | Should -Be 'Succeeded' + } + It 'does not provision or run health tests when Blob writes never become ready' { + Mock Wait-SovereignCheck { & $realWaitSovereignCheck -TimeoutSeconds 0 -Check $Check } + Mock Invoke-LocalBoxAz { throw 'Blob upload permissions are not ready' } -ParameterFilter { + $Arguments[0..2] -join ' ' -eq 'storage blob upload' + } + + { Invoke-LocalBoxGuestAutomation $guestConfig $guestDirectory $guestFixture 'input.json' -ManifestPath $manifestFile } | + Should -Throw '*Blob upload permissions are not ready*' + + Should -Invoke Wait-SovereignCheck -Times 1 -Exactly -ParameterFilter { $TimeoutSeconds -eq 600 } + Should -Invoke Invoke-TestPreparation -Times 0 + Should -Invoke Invoke-LocalBoxAutomationHealthProcess -Times 0 + $status = Get-Content "$guestDirectory/status.json" -Raw | ConvertFrom-Json + $status.State | Should -Be 'Failed' + $status.Error | Should -Be 'Blob upload permissions are not ready' + } + It 'retains fail-fast handling for explicit authorization errors' { + Mock Invoke-LocalBoxAz { throw 'AuthorizationFailed: access denied' } -ParameterFilter { + $Arguments[0..2] -join ' ' -eq 'storage blob upload' + } + + { Invoke-LocalBoxGuestAutomation $guestConfig $guestDirectory $guestFixture 'input.json' -ManifestPath $manifestFile } | + Should -Throw '*AuthorizationFailed*' + + Should -Invoke Start-Sleep -Times 0 + Should -Invoke Invoke-TestPreparation -Times 0 + } + } It 'captures prerequisite failures locally even before Azure authentication' { Mock Initialize-LocalBoxAutomationEnvironment { throw 'Get-VM unavailable' } { Invoke-LocalBoxGuestAutomation $guestConfig $guestDirectory $guestFixture 'input.json' -ManifestPath $manifestFile } | Should -Throw '*Get-VM unavailable*' diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md index f6e7d51d2..685b84617 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md @@ -181,6 +181,13 @@ update-assessment failure, credential-file cleanup and diagnostic-upload failure They do not exercise Windows SYSTEM, real managed Run Command, Blob RBAC propagation, AKS admin credentials, or a multi-hour Console token rotation; those are live gates. +Blob readiness regression cases use the real retry loop and status writer with +mocked Azure CLI calls: listing succeeds while the first two uploads fail, +preparation starts only after a successful upload, exhausted write readiness +blocks preparation and health tests, and explicit authorization errors remain +fatal. These cover the TEST13 read-before-write permission-propagation failure +without treating mocked retries as proof of live Blob access. + Packaging tests also emulate the Console's `labautomation`-only mount, with no sibling `resources` directory. They verify immutable source pins and SHA256 checks, reject missing/mutable/corrupt sources, prevent mixing partial local From c4e935c65c961e88fcc2a8b5cb965b882be65a65 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Sat, 7 Nov 2026 19:33:05 +0000 Subject: [PATCH 27/37] Pin the Blob write-readiness guest fix Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../labautomation/localbox-automation-source.json | 4 ++-- .../01_Sovereign_Cloud/resources/tests/readme.md | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation-source.json b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation-source.json index 7672e1b2f..a2e0ad30a 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation-source.json +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation-source.json @@ -1,9 +1,9 @@ { "repository": "janegilring/MicroHack", - "commit": "4fdcfe25d5713b459e88c5765fff6ac66156cd79", + "commit": "238925b6c989de13f1f957ba7aacbca5b9536039", "files": { "prepare-localbox.ps1": "62934e83e421df476a133e127c7ef3285dd62da8ef0fe04921800f89e5e03c69", - "invoke-localbox-automation.ps1": "cbcb8dbee2523497a3d6b8c0586154c01e579bce3183de9aeb10d4e3ccb7c425", + "invoke-localbox-automation.ps1": "e2aad00c6750f1b18c85aa8ff2d40d1f77b9645f50a9eb3c1273e08abe762bc0", "test-sovereign-cloud.ps1": "e23583ac0a3747261d57469a7ae3e31438c0cb70beb9bf1c751a6244e4ea5117", "tests/localbox.health.tests.ps1": "92f922dff75b1bcecc7c3e9a523c613930bd1a538ef8bb0fd50dd2ee1267b4d9", "tests/sovereign-lab.health.tests.ps1": "2916dd41ea0e53b4599bfb72a8f21ac6d152a476aff72593bd33e075fdde7ada" diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md index 685b84617..9422b5327 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md @@ -187,6 +187,8 @@ preparation starts only after a successful upload, exhausted write readiness blocks preparation and health tests, and explicit authorization errors remain fatal. These cover the TEST13 read-before-write permission-propagation failure without treating mocked retries as proof of live Blob access. +The TEST13 fix passed all **529 tests** across the automation, preparation and +global LocalBox suites, including synchronized guest-source checksum checks. Packaging tests also emulate the Console's `labautomation`-only mount, with no sibling `resources` directory. They verify immutable source pins and SHA256 From ca255062782afd7b73ca10215b47edbf9187448c Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Sat, 7 Nov 2026 19:35:38 +0000 Subject: [PATCH 28/37] Record TEST13 passing Blob readiness after recovery Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../01_Sovereign_Cloud/resources/hosted-events/readme.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md index 513ff8623..80c466b79 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md @@ -571,6 +571,13 @@ alone is therefore insufficient evidence of write readiness during propagation. The upload now shares the bounded retry gate; persistent failure still blocks preparation and Full readiness. No additional permissions are granted. +The authorized TEST13 recovery started a new guest run at **19:33 UTC** using +the tested package. It passed the initial Blob upload and reached CLI prerequisite +installation. Azure Local was not rebuilt, startup was not replayed, and role +assignments were unchanged. The published five-file download was independently +checksum-verified against the resumed package. Full health remains unverified; +this guest-only recovery does not resume the failed Console's downstream stages. + Each run has a separate prefix containing `status.json`, `automation.log`, `upstream.xml`, `health.xml` and `health.json` when those phases produced them. Status is updated at phase boundaries. Failures retain available diagnostics and From 46fa053f366f55556a1f49e2a82c6fca2a98424e Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Sat, 7 Nov 2026 23:52:57 +0000 Subject: [PATCH 29/37] Record verified TEST13 Full health and MetalLB recovery Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../resources/hosted-events/readme.md | 16 ++++++++++++++++ .../01_Sovereign_Cloud/resources/tests/readme.md | 8 ++++++++ 2 files changed, 24 insertions(+) diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md index 80c466b79..b838176b1 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md @@ -578,6 +578,22 @@ assignments were unchanged. The published five-file download was independently checksum-verified against the resumed package. Full health remains unverified; this guest-only recovery does not resume the failed Console's downstream stages. +**TEST13 completion:** the resumed guest finished at **23:48:40 UTC on +7 November 2026**, after **4h 15m 5s**. A separate read-only guest probe verified +the run/package/group/Azure-scope identity, report timestamps, Full health JSON, +and both NUnit XML reports: **12/12 upstream and 14/14 Sovereign tests passed**, +with zero failed, skipped or unrun tests. The reference VM update assessment +succeeded at 23:41:22 UTC. These results cover AKS runtime health, the imported +image, reference VM, Arc connectivity and MDE installation. + +The bounded MetalLB recovery was exercised successfully: attempt **1/1** +reapplied the unchanged Azure-managed ARP pool at 23:45:55 UTC and verified +the Kubernetes pool and L2 advertisement at 23:46:14 UTC, without deletion. +The Full suite subsequently passed. This supersedes the pending LocalBox health +checkpoint above, but is a **successful resumed LocalBox run**, not proof of an +uninterrupted fresh Console event. Console downstream stages, real participant +sign-in and Defender cloud reporting remain outside this verified result. + Each run has a separate prefix containing `status.json`, `automation.log`, `upstream.xml`, `health.xml` and `health.json` when those phases produced them. Status is updated at phase boundaries. Failures retain available diagnostics and diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md index 9422b5327..7bdf7fa18 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md @@ -189,6 +189,14 @@ fatal. These cover the TEST13 read-before-write permission-propagation failure without treating mocked retries as proof of live Blob access. The TEST13 fix passed all **529 tests** across the automation, preparation and global LocalBox suites, including synchronized guest-source checksum checks. +The authorized live resume subsequently completed at 23:48:40 UTC on +7 November 2026. A separate read-only probe used the repository's result/XML +validators to verify the exact run, package, group and Azure scope, both XML +reports, matching Full JSON counts and report timestamps: **12/12 upstream and +14/14 Sovereign tests passed**, with none failed, skipped or unrun. The run also +exercised successful bounded MetalLB recovery (one unchanged ARP-pool reapply). +This establishes resumed shared LocalBox Full health, not the skipped Console +downstream stages, participant sign-in or Defender cloud reporting. Packaging tests also emulate the Console's `labautomation`-only mount, with no sibling `resources` directory. They verify immutable source pins and SHA256 From 8f284c0b0fabcf204f1d912f03cf73d0ba048a9b Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Sun, 8 Nov 2026 07:20:18 +0000 Subject: [PATCH 30/37] Fix BGInfo discovery and process-aware startup monitoring Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../labautomation/localbox-automation.ps1 | 11 +- .../labautomation/localbox-startup-watch.ps1 | 28 ++- .../labautomation/localbox-startup.ps1 | 82 ++++++-- .../resources/hosted-events/readme.md | 22 +- .../tests/localbox-startup.tests.ps1 | 196 +++++++++++++++++- .../resources/tests/readme.md | 16 ++ 6 files changed, 330 insertions(+), 25 deletions(-) diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation.ps1 index 0f65356d7..bf7ae83d0 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-automation.ps1 @@ -90,6 +90,15 @@ function Get-LocalBoxValidationRetryState { Detail = "Automatic validation retry 1/1: $($state.executionState). Deployment remains gated on successful validation." } } +function New-LocalBoxStartupWatchScript { + $source = Get-Content (Join-Path $PSScriptRoot 'localbox-startup-watch.ps1') -Raw -ErrorAction Stop + $errors = $null + $ast = [Management.Automation.Language.Parser]::ParseInput($source, [ref]$null, [ref]$errors) + if ($errors.Count -or -not $ast.ParamBlock) { throw 'Cannot parse the LocalBox startup monitor for packaging.' } + $formatter = "function ConvertTo-LocalBoxDiagnosticText {`n$((Get-Command ConvertTo-LocalBoxDiagnosticText -CommandType Function).Definition)`n}" + $source.Insert($ast.ParamBlock.Extent.EndOffset, "`n`n$formatter") +} + function Get-LocalBoxStartupProgress { param([hashtable]$Tracking, [string]$SubscriptionId, [string]$ResourceGroupName, [int]$TimeoutSeconds) $vmId = "/subscriptions/$SubscriptionId/resourceGroups/$ResourceGroupName/providers/Microsoft.Compute/virtualMachines/LocalBox-Client" @@ -115,7 +124,7 @@ function Get-LocalBoxStartupProgress { Invoke-LocalBoxAutomationRequest -Path $uri -Method PUT -Body @{ location = $vm.location properties = @{ - source = @{ script = Get-Content (Join-Path $PSScriptRoot 'localbox-startup-watch.ps1') -Raw -ErrorAction Stop } + source = @{ script = New-LocalBoxStartupWatchScript } asyncExecution = $true; timeoutInSeconds = $TimeoutSeconds parameters = @( @{ name = 'BootstrapId'; value = $Tracking.Id } diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-startup-watch.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-startup-watch.ps1 index 52b428220..943439181 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-startup-watch.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-startup-watch.ps1 @@ -21,6 +21,25 @@ function Get-LocalBoxStartupReport { } } +function Test-LocalBoxStartupActive { + param([string]$Name = 'Global\MicroHackLocalBoxStartup') + $mutex = $null + if (-not [Threading.Mutex]::TryOpenExisting($Name, [ref]$mutex)) { return $false } + $acquired = $false + try { + try { $acquired = $mutex.WaitOne(0) } + catch [Threading.AbandonedMutexException] { + # WaitOne grants ownership when the previous process died. + $acquired = $true + } + return -not $acquired + } + finally { + if ($acquired) { $mutex.ReleaseMutex() } + $mutex.Dispose() + } +} + function Wait-LocalBoxStartupReport { param([string]$Id, [string]$Subscription, [string]$Group, [int]$TimeoutSeconds) $ErrorActionPreference = 'Stop' @@ -41,10 +60,11 @@ function Wait-LocalBoxStartupReport { return $report } if ($status -and (Get-Date).ToUniversalTime() -ge $started.AddMinutes(2)) { - $task = Get-ScheduledTask -TaskName LocalBoxLogonScript -ErrorAction Stop - if ($task.State -ne 'Running') { + if (-not (Test-LocalBoxStartupActive)) { + $status = Get-LocalBoxStartupReport -Path 'C:\LocalBox\SovereignStartup\state.json' -Id $Id -Subscription $Subscription -Group $Group + if ($status -and $status.State -in @('Succeeded', 'Failed')) { return $status } $report.Code = 'StartupExited' - $report.Message = 'The startup task stopped without a successful report. Inspect LocalBox logs; do not repeat storage initialization blindly.' + $report.Message = 'The guarded startup process is no longer active without a terminal report. Inspect LocalBox logs; do not repeat storage initialization blindly.' return $report } } @@ -57,7 +77,7 @@ function Wait-LocalBoxStartupReport { } while ($true) } catch { - $report.Message = 'Startup monitoring failed to verify the local report or scheduled task. Inspect sovereign-localbox-startup-watch and LocalBox logs.' + $report.Message = "Startup monitoring failed: $(ConvertTo-LocalBoxDiagnosticText $_.Exception.Message 600) Inspect sovereign-localbox-startup-watch and LocalBox logs." return $report } } diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-startup.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-startup.ps1 index 665bbb28e..235323b0c 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-startup.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-startup.ps1 @@ -57,6 +57,7 @@ Copy-Item -LiteralPath 'C:\AzureData\CustomData.bin' -Destination "$guardRoot\st . "$guardRoot\startup.ps1" Set-LocalBoxWinGetGuard -Path "$LocalBoxPath\WinGet.ps1" Set-LocalBoxPackageConfiguration -Path "$LocalBoxPath\DSC\packages.dsc.yml" +Set-LocalBoxBgInfoGuard -Path "$($LocalBoxConfig.Paths.TestsDir)\Invoke-Test.ps1" $Action = New-ScheduledTaskAction -Execute $ScheduledTaskExecutable -Argument "$guardRoot\startup.ps1" '@ Set-Content -LiteralPath $Path -Value $text.Replace($action, $replacement) -Encoding utf8 -ErrorAction Stop @@ -224,29 +225,81 @@ function Test-LocalBoxBaseImage { return (Get-FileHash -LiteralPath $Image -Algorithm SHA256 -ErrorAction Stop).Hash -eq $expected } -function Resolve-LocalBoxAzCopy { - param([string]$LocalAppData = $env:LOCALAPPDATA, [string]$ProgramFiles = $env:ProgramFiles) - $client = @(Get-Command azcopy -CommandType Application -ErrorAction SilentlyContinue) +function Set-LocalBoxBgInfoGuard { + param([string]$Path) + $text = Get-Content -LiteralPath $Path -Raw -ErrorAction Stop + $command = 'bginfo64.exe $Env:LocalBoxTestsDir\localbox-bginfo.bgi /timer:0 /NOLICPROMPT' + $pattern = '(?m)^' + [regex]::Escape($command) + '\r?$' + if ([regex]::Matches($text, $pattern).Count -ne 1) { + throw 'Unsupported upstream BGInfo invocation; test-runner protection was not installed.' + } + $replacement = @' +. "$Env:LocalBoxDir\SovereignStartup\startup.ps1" +Invoke-LocalBoxBgInfo -ConfigurationPath "$Env:LocalBoxTestsDir\localbox-bginfo.bgi" +'@ + Set-Content -LiteralPath $Path -Value $text.Replace($command, $replacement) -Encoding utf8 -ErrorAction Stop +} + +function Resolve-LocalBoxStartupTool { + param( + [ValidateSet('AzCopy', 'BGInfo')][string]$Tool, + [string]$LocalAppData, [string]$ProgramFiles + ) + $command = if ($Tool -eq 'AzCopy') { 'azcopy' } else { 'bginfo64.exe' } + $executable = if ($Tool -eq 'AzCopy') { 'azcopy.exe' } else { 'Bginfo64.exe' } + $client = @(Get-Command $command -CommandType Application -ErrorAction SilentlyContinue) if ($client.Count) { return $client[0].Path } $links = @( - if ($LocalAppData) { Join-Path $LocalAppData 'Microsoft/WinGet/Links/azcopy.exe' } + if ($LocalAppData) { Join-Path $LocalAppData "Microsoft/WinGet/Links/$executable" } if ($ProgramFiles) { - Join-Path $ProgramFiles 'WinGet/Links/azcopy.exe' - Join-Path $ProgramFiles 'AzCopy/azcopy.exe' + Join-Path $ProgramFiles "WinGet/Links/$executable" + if ($Tool -eq 'AzCopy') { Join-Path $ProgramFiles 'AzCopy/azcopy.exe' } } ) foreach ($path in $links) { if (Test-Path -LiteralPath $path -PathType Leaf -ErrorAction Stop) { return $path } } - if ($LocalAppData) { - $pattern = Join-Path $LocalAppData 'Microsoft/WinGet/Packages/Microsoft.Azure.AZCopy.10_*/azcopy_windows_*/azcopy.exe' + $patterns = @( + if ($LocalAppData) { + if ($Tool -eq 'AzCopy') { + Join-Path $LocalAppData 'Microsoft/WinGet/Packages/Microsoft.Azure.AZCopy.10_*/azcopy_windows_*/azcopy.exe' + } + else { Join-Path $LocalAppData 'Microsoft/WinGet/Packages/Microsoft.Sysinternals.BGInfo_*/Bginfo64.exe' } + } + if ($Tool -eq 'BGInfo' -and $ProgramFiles) { + Join-Path $ProgramFiles 'WinGet/Packages/Microsoft.Sysinternals.BGInfo_*/Bginfo64.exe' + } + ) + $candidates = @(foreach ($pattern in $patterns) { if (Test-Path -Path $pattern -PathType Leaf -ErrorAction Stop) { - $candidates = @(Get-ChildItem -Path $pattern -File -ErrorAction Stop) - if ($candidates.Count -ne 1) { throw 'Multiple unlinked AzCopy installations exist for the startup user; repair the WinGet link before retrying.' } - return $candidates[0].FullName + Get-ChildItem -Path $pattern -File -ErrorAction Stop } + }) + if ($candidates.Count -gt 1) { throw "Multiple unlinked $Tool installations exist; repair the WinGet link before retrying." } + if ($candidates.Count -eq 1) { return $candidates[0].FullName } + throw "$Tool is unavailable after prerequisite installation. Inspect the WinGet provisioning log and the startup user WinGet installation." +} + +function Resolve-LocalBoxAzCopy { + param([string]$LocalAppData = $env:LOCALAPPDATA, [string]$ProgramFiles = $env:ProgramFiles) + Resolve-LocalBoxStartupTool -Tool AzCopy -LocalAppData $LocalAppData -ProgramFiles $ProgramFiles +} + +function Resolve-LocalBoxBgInfo { + param([string]$LocalAppData = $env:LOCALAPPDATA, [string]$ProgramFiles = $env:ProgramFiles) + Resolve-LocalBoxStartupTool -Tool BGInfo -LocalAppData $LocalAppData -ProgramFiles $ProgramFiles +} + +function Invoke-LocalBoxBgInfo { + param([string]$ConfigurationPath) + $ErrorActionPreference = 'Stop' + if (-not (Test-Path -LiteralPath $ConfigurationPath -PathType Leaf)) { + throw 'The LocalBox BGInfo configuration is missing.' } - throw 'AzCopy is unavailable after prerequisite installation. Inspect the WinGet provisioning log and the startup user WinGet installation; no image transfer was attempted.' + $client = Resolve-LocalBoxBgInfo + $PSNativeCommandUseErrorActionPreference = $false + & $client $ConfigurationPath /timer:0 /NOLICPROMPT + if ($LASTEXITCODE -ne 0) { throw "BGInfo failed with exit code $LASTEXITCODE." } } function Wait-LocalBoxStartupPrerequisites { @@ -276,8 +329,9 @@ function Wait-LocalBoxStartupPrerequisites { $env:PATH = (@($env:PATH, [Environment]::GetEnvironmentVariable('PATH', 'Machine'), [Environment]::GetEnvironmentVariable('PATH', 'User')) | Where-Object { $_ } | Select-Object -Unique) -join [IO.Path]::PathSeparator $null = Resolve-LocalBoxAzCopy + $null = Resolve-LocalBoxBgInfo $Status.Phase = 'Startup' - $Status.Message = 'WinGet prerequisites completed and AzCopy resolved; upstream initialization may proceed.' + $Status.Message = 'WinGet prerequisites completed; AzCopy and BGInfo resolved. Upstream initialization may proceed.' Save-LocalBoxStartupStatus $Status $StatusPath } @@ -460,7 +514,7 @@ function Invoke-LocalBoxGuardedStartup { $latest.State = 'Failed' if ($latest.Phase -eq 'Prerequisites') { $latest.Code = 'StartupPrerequisitesFailed' - $latest.Message = 'WinGet prerequisite readiness or AzCopy discovery failed before storage initialization. Inspect WinGet-provisioning logs and WinGetLogonScript task state.' + $latest.Message = 'WinGet prerequisite readiness or AzCopy/BGInfo discovery failed before storage initialization. Inspect WinGet-provisioning logs and WinGetLogonScript task state.' } else { $latest.Code = 'StartupFailed' diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md index b838176b1..9478469d5 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md @@ -336,12 +336,22 @@ Before entering upstream storage initialization, the guard waits for `WinGetLogonScript` to finish and unregister itself (at most **one hour**). The upstream WinGet and LocalBox tasks can both start at logon; starting downloads while WinGet is still installing AzCopy is not safe. After installation, the -guard refreshes its process PATH and resolves AzCopy from PATH, the current +guard refreshes its process PATH and resolves AzCopy and BGInfo from PATH, the current startup user's WinGet link/package, or known machine installation paths. It does not search other users' profiles, install a replacement binary, or blindly pick between multiple unlinked package versions. Prerequisite failure is reported as `Prerequisites` / `StartupPrerequisitesFailed`, before storage initialization. +TEST14 exposed an installed BGInfo executable that was not discoverable as +`bginfo64.exe`. The upstream test runner failed at its desktop-status update, +after running its tests but before registering its future-logon test task. +For new deployments, the guard patches that exact +invocation to resolve the installed binary explicitly and check its exit code. +The retained runner uses the same helper on subsequent logons, not just during +initial startup. Missing or ambiguous installations and upstream script drift +still fail explicitly; the fix neither downloads another binary nor suppresses +test failures. Existing immutable startup payloads are not changed. + WinGet provisioning is also guarded. Its CLI is repaired to the installed `Microsoft.WinGet.Client` module version, with at most three attempts and 30-second retry delays; version matching and a real `Get-WinGetPackage` COM @@ -399,8 +409,14 @@ It verifies the startup identity and scope, then reports the failure phase, code and bounded diagnostic message (for example, `BaseImages`, `BaseImageDownloadFailed`, and `HTTP 401`) instead of waiting eight hours for a deployment that was never started. No startup report within **one hour** fails -the gate; a Running report with a stopped task fails after a **two-minute -monitoring grace period**. Inspect the protected +the gate; after a **two-minute monitoring grace period**, a Running report must +have a live owner of the guarded startup mutex. The upstream script deliberately +unregisters its scheduled task before desktop setup and final tests, so task +registration is not used as process-liveness evidence. After the lock is released, +the monitor rereads the report to handle completion between observations; absence +of a terminal report remains a failure. Lock ownership alone never establishes +startup success. Monitor failures include bounded, redacted underlying diagnostics +using the same formatter as Console. Inspect the protected `C:\LocalBox\SovereignStartup\state.json` and the LocalBox startup logs. Startup success is **not infrastructure readiness**. All existing ARM, cluster, diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-startup.tests.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-startup.tests.ps1 index 631656648..a90307783 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-startup.tests.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-startup.tests.ps1 @@ -70,6 +70,7 @@ var command = { $text = Get-Content "$TestDrive/Bootstrap.ps1" -Raw $text | Should -Match 'Set-Acl' $text | Should -Match '\$guardRoot\\startup.ps1' + $text | Should -Match 'Set-LocalBoxBgInfoGuard' { Install-LocalBoxStartupGuard "$TestDrive/Bootstrap.ps1" } | Should -Throw '*Unsupported*' } } @@ -452,6 +453,7 @@ Describe 'Startup report validation' { $prerequisiteClock.Now = $prerequisiteClock.Now.AddSeconds(15) } Mock Resolve-LocalBoxAzCopy { 'fixture-azcopy.exe' } + Mock Resolve-LocalBoxBgInfo { 'fixture-bginfo.exe' } } AfterEach { $env:PATH = $originalPath } It 'waits for installer completion before resolving tools or initializing storage' { @@ -465,6 +467,7 @@ Describe 'Startup report validation' { Wait-LocalBoxStartupPrerequisites $prerequisiteStatus $prerequisitePath Should -Invoke Start-Sleep -Times 2 -Exactly Should -Invoke Resolve-LocalBoxAzCopy -Times 1 -Exactly + Should -Invoke Resolve-LocalBoxBgInfo -Times 1 -Exactly $prerequisiteStatus.Phase | Should -Be 'Startup' } It 'also waits for a registered task that has not started yet' { @@ -486,6 +489,11 @@ Describe 'Startup report validation' { { Wait-LocalBoxStartupPrerequisites $prerequisiteStatus $prerequisitePath } | Should -Throw '*fixture-missing-azcopy*' $prerequisiteStatus.Phase | Should -Be 'Prerequisites' } + It 'requires BGInfo before storage initialization even when AzCopy is available' { + Mock Resolve-LocalBoxBgInfo { throw 'fixture-missing-bginfo' } + { Wait-LocalBoxStartupPrerequisites $prerequisiteStatus $prerequisitePath } | Should -Throw '*fixture-missing-bginfo*' + $prerequisiteStatus.Phase | Should -Be 'Prerequisites' + } It 'surfaces task query errors rather than assuming WinGet completed' { Mock Get-ScheduledTask { throw 'fixture-scheduler-unavailable' } { Wait-LocalBoxStartupPrerequisites $prerequisiteStatus $prerequisitePath } | Should -Throw '*fixture-scheduler-unavailable*' @@ -552,6 +560,7 @@ Describe 'Startup report monitor thresholds' { Mock Start-Sleep { $clock.Now = $clock.Now.AddHours(1) } Mock Get-LocalBoxStartupReport { $null } Mock Get-ScheduledTask { @{ State = 'Ready' } } + Mock Test-LocalBoxStartupActive { $false } } It 'fails at the one-hour missing-report threshold instead of waiting eight hours' { $report = Wait-LocalBoxStartupReport $id $subscription group 28800 @@ -559,11 +568,12 @@ Describe 'Startup report monitor thresholds' { Should -Invoke Start-Sleep -Times 1 -Exactly ($clock.Now - [datetime]'2026-11-05T00:00:00Z').TotalSeconds | Should -Be 3600 } - It 'detects a stopped startup task only after the two-minute startup grace' { + It 'detects an exited guarded process only after the two-minute startup grace' { Mock Get-LocalBoxStartupReport { @{ State = 'Running' } } Mock Start-Sleep { $clock.Now = $clock.Now.AddSeconds(120) } (Wait-LocalBoxStartupReport $id $subscription group 28800).Code | Should -Be 'StartupExited' - Should -Invoke Get-ScheduledTask -Times 1 -Exactly + Should -Invoke Test-LocalBoxStartupActive -Times 1 -Exactly + Should -Invoke Get-ScheduledTask -Times 0 ($clock.Now - [datetime]'2026-11-05T00:00:00Z').TotalSeconds | Should -Be 120 } It 'returns reported failure immediately without waiting or inspecting tasks' { @@ -571,9 +581,189 @@ Describe 'Startup report monitor thresholds' { (Wait-LocalBoxStartupReport $id $subscription group 28800).Code | Should -Be 'BaseImageDownloadFailed' Should -Invoke Start-Sleep -Times 0 Should -Invoke Get-ScheduledTask -Times 0 + Should -Invoke Test-LocalBoxStartupActive -Times 0 } It 'does not treat a monitoring exception as startup success' { Mock Get-LocalBoxStartupReport { throw 'Invalid metadata' } - (Wait-LocalBoxStartupReport $id $subscription group 28800).State | Should -Be 'Failed' + $report = Wait-LocalBoxStartupReport $id $subscription group 28800 + $report.State | Should -Be 'Failed' + $report.Message | Should -Match 'Invalid metadata' + } + It 'continues observing a live startup after it unregisters its scheduled task' { + Mock Get-ScheduledTask { throw 'No scheduled task exists' } + Mock Start-Sleep { $clock.Now = $clock.Now.AddSeconds(120) } + Mock Test-LocalBoxStartupActive { $true } + Mock Get-LocalBoxStartupReport { + if ($clock.Now -ge [datetime]'2026-11-05T00:04:00Z') { return @{ State = 'Succeeded'; Phase = 'Complete' } } + @{ State = 'Running'; Phase = 'NestedDeployment' } + } + (Wait-LocalBoxStartupReport $id $subscription group 28800).State | Should -Be 'Succeeded' + Should -Invoke Test-LocalBoxStartupActive -Times 1 -Exactly + Should -Invoke Get-ScheduledTask -Times 0 + } + It 'rereads the terminal report when the process releases its lock between observations' { + $reads = @{ Count = 0 } + Mock Start-Sleep { $clock.Now = $clock.Now.AddSeconds(120) } + Mock Get-LocalBoxStartupReport { + $reads.Count++ + if ($reads.Count -eq 3) { return @{ State = 'Succeeded'; Phase = 'Complete' } } + @{ State = 'Running' } + } + (Wait-LocalBoxStartupReport $id $subscription group 28800).State | Should -Be 'Succeeded' + $reads.Count | Should -Be 3 + } + It 'does not treat a held lock as success when the report remains incomplete' { + Mock Get-LocalBoxStartupReport { @{ State = 'Running' } } + Mock Test-LocalBoxStartupActive { $true } + $report = Wait-LocalBoxStartupReport $id $subscription group 3600 + $report.State | Should -Be 'Failed' + $report.Code | Should -Be 'StartupTimedOut' + } + It 'preserves bounded redacted lock-inspection errors without hiding the failure' { + Mock Get-LocalBoxStartupReport { @{ State = 'Running' } } + Mock Test-LocalBoxStartupActive { throw ('Mutex access denied; password=fixture-password token=fixture-token ' + ('x' * 1000)) } + $report = Wait-LocalBoxStartupReport $id $subscription group 28800 + $report.Code | Should -Be 'StartupMonitorFailed' + $report.State | Should -Be 'Failed' + $report.Message | Should -Match 'Mutex access denied' + $report.Message | Should -Not -Match 'fixture-password|fixture-token' + $report.Message.Length | Should -BeLessOrEqual 800 + } +} + +Describe 'BGInfo resolution and persistent test-runner protection' { + BeforeEach { + Mock Get-Command { @() } -ParameterFilter { $Name -eq 'bginfo64.exe' } + $localData = Join-Path $TestDrive ([guid]::NewGuid().ToString()) + $programRoot = Join-Path $TestDrive ([guid]::NewGuid().ToString()) + $resolveArguments = @{ LocalAppData = $localData; ProgramFiles = $programRoot } + $runner = "$TestDrive/Invoke-Test.ps1" + $bginfoLine = 'bginfo64.exe $Env:LocalBoxTestsDir\localbox-bginfo.bgi /timer:0 /NOLICPROMPT' + } + It 'preserves a native command already available on PATH' { + Mock Get-Command { @{ Path = 'native-bginfo.exe' } } -ParameterFilter { $Name -eq 'bginfo64.exe' } + Resolve-LocalBoxBgInfo @resolveArguments | Should -Be 'native-bginfo.exe' + } + It 'resolves a WinGet link or exact package without relying on process PATH: ' -TestCases @( + @{ Location = 'UserLink'; Relative = 'Microsoft/WinGet/Links' } + @{ Location = 'MachineLink'; Relative = 'WinGet/Links' } + @{ Location = 'UserPackage'; Relative = 'Microsoft/WinGet/Packages/Microsoft.Sysinternals.BGInfo_fixture' } + @{ Location = 'MachinePackage'; Relative = 'WinGet/Packages/Microsoft.Sysinternals.BGInfo_fixture' } + ) { + param($Location, $Relative) + $base = if ($Location -like 'User*') { $localData } else { $programRoot } + $directory = Join-Path $base $Relative + New-Item -ItemType Directory -Path $directory -Force | Out-Null + Set-Content "$directory/Bginfo64.exe" fixture + Resolve-LocalBoxBgInfo @resolveArguments | Should -Be "$directory/Bginfo64.exe" + } + It 'rejects ambiguous unlinked packages instead of choosing a version' { + foreach ($version in @('one', 'two')) { + $directory = Join-Path $localData "Microsoft/WinGet/Packages/Microsoft.Sysinternals.BGInfo_$version" + New-Item -ItemType Directory -Path $directory -Force | Out-Null + Set-Content "$directory/Bginfo64.exe" fixture + } + { Resolve-LocalBoxBgInfo @resolveArguments } | Should -Throw '*Multiple unlinked BGInfo*' + } + It 'reports a missing installation rather than falling back to another user profile' { + { Resolve-LocalBoxBgInfo @resolveArguments } | Should -Throw '*BGInfo is unavailable*' + } + It 'patches the retained runner for startup and future logons: ' -TestCases @( + @{ Newline = "`n" }; @{ Newline = "`r`n" } + ) { + param($Newline) + "Write-Output before${Newline}$bginfoLine${Newline}Write-Output after" | Set-Content $runner + Set-LocalBoxBgInfoGuard $runner + $text = Get-Content $runner -Raw + $text | Should -Match '\. "\$Env:LocalBoxDir\\SovereignStartup\\startup.ps1"' + $text | Should -Match 'Invoke-LocalBoxBgInfo -ConfigurationPath "\$Env:LocalBoxTestsDir\\localbox-bginfo.bgi"' + $text | Should -Match 'Write-Output before' + $text | Should -Match 'Write-Output after' + $text | Should -Not -Match '(?m)^bginfo64.exe' + } + It 'rejects missing or duplicate upstream commands without modifying the runner' -TestCases @( + @{ Count = 0 }; @{ Count = 2 } + ) { + param($Count) + (@($bginfoLine) * $Count -join "`n") | Set-Content $runner + $before = Get-Content $runner -Raw + { Set-LocalBoxBgInfoGuard $runner } | Should -Throw '*Unsupported upstream BGInfo*' + Get-Content $runner -Raw | Should -Be $before + } +} + +Describe 'Checked BGInfo invocation' { + BeforeEach { + $configuration = "$TestDrive/configuration with spaces.bgi" + Set-Content $configuration fixture + $executable = "$TestDrive/bginfo fixture.ps1" + Mock Resolve-LocalBoxBgInfo { $executable } + $previousExitCode = $global:LASTEXITCODE + } + AfterEach { $global:LASTEXITCODE = $previousExitCode } + It 'passes the exact configuration and noninteractive flags to the resolved executable' { + @' +$args | ConvertTo-Json | Set-Content "$PSScriptRoot/bginfo-arguments.json" +$global:LASTEXITCODE = 0 +'@ | Set-Content $executable + Invoke-LocalBoxBgInfo $configuration + $arguments = Get-Content "$TestDrive/bginfo-arguments.json" -Raw | ConvertFrom-Json + $arguments.Count | Should -Be 3 + $arguments[0] | Should -Be $configuration + $arguments[1] | Should -Be '/timer:0' + $arguments[2] | Should -Be '/NOLICPROMPT' + } + It 'fails on a nonzero exit code' { + '$global:LASTEXITCODE = 5' | Set-Content $executable + { Invoke-LocalBoxBgInfo $configuration } | Should -Throw '*BGInfo failed with exit code 5*' + } + It 'does not launch BGInfo without its configuration' { + { Invoke-LocalBoxBgInfo "$TestDrive/missing.bgi" } | Should -Throw '*configuration is missing*' + Should -Invoke Resolve-LocalBoxBgInfo -Times 0 + } +} + +Describe 'Startup lock and standalone monitor packaging' { + It 'reports a nonexistent or unowned lock as inactive and releases acquired handles' { + $name = "MicroHackTest-$([guid]::NewGuid())" + Test-LocalBoxStartupActive $name | Should -BeFalse + $mutex = [Threading.Mutex]::new($false, $name) + try { + Test-LocalBoxStartupActive $name | Should -BeFalse + $mutex.WaitOne(0) | Should -BeTrue + $mutex.ReleaseMutex() + } + finally { $mutex.Dispose() } + } + It 'observes a lock held by another process without acquiring or releasing its ownership' { + $name = "MicroHackTest-$([guid]::NewGuid())" + $job = Start-Job -ArgumentList $name -ScriptBlock { + param($Name) + $mutex = [Threading.Mutex]::new($false, $Name) + if (-not $mutex.WaitOne(0)) { throw 'Fixture could not acquire its startup lock.' } + try { Write-Output 'held'; Start-Sleep -Seconds 30 } + finally { $mutex.ReleaseMutex(); $mutex.Dispose() } + } + try { + $deadline = [datetime]::UtcNow.AddSeconds(15) + while (-not $job.ChildJobs[0].Output.Count -and [datetime]::UtcNow -lt $deadline) { Start-Sleep -Milliseconds 50 } + Receive-Job $job -Keep -ErrorAction Stop | Should -Be 'held' + Test-LocalBoxStartupActive $name | Should -BeTrue + Test-LocalBoxStartupActive $name | Should -BeTrue + } + finally { Stop-Job $job; Remove-Job $job -Force } + } + It 'embeds the shared diagnostic formatter in the submitted standalone monitor' { + $source = New-LocalBoxStartupWatchScript + $errors = $null + $ast = [Management.Automation.Language.Parser]::ParseInput($source, [ref]$null, [ref]$errors) + $errors.Count | Should -Be 0 + $formatter = $ast.FindAll({ + param($node) + $node -is [Management.Automation.Language.FunctionDefinitionAst] -and $node.Name -eq 'ConvertTo-LocalBoxDiagnosticText' + }, $true) + $formatter.Count | Should -Be 1 + $formatter[0].Body.Extent.Text | Should -Be ('{' + "`n" + (Get-Command ConvertTo-LocalBoxDiagnosticText).Definition + "`n}") + $ast.ParamBlock.Parameters.Name.VariablePath.UserPath | Should -Contain 'BootstrapId' } } diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md index 7bdf7fa18..209fd9534 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md @@ -66,6 +66,22 @@ upstream transformations and removal of only the retired Azure Data Studio resource. Real upstream Bootstrap/WinGet scripts and the package configuration are also transformed and syntax/compilation checked offline. +TEST14 startup regressions cover native/user/machine BGInfo discovery, ambiguous +and missing installations, checked invocation with a spaced configuration path, +and exact persistent upstream test-runner patching. Monitor tests cover scheduled +task removal while the guarded process remains active, missing/unowned and +cross-process owned mutexes, terminal-report races, unchanged deadlines, and +bounded redacted failures. The submitted standalone monitor embeds the shared +formatter; no current deployment's immutable startup payload is rewritten. +The coupled startup, WinGet, deployment, automation, validation-retry, global +LocalBox and failure-summary suites passed **292 tests**. Current upstream Bicep +and script transformations compiled with a **33,073-byte** startup payload. +A read-only Windows PowerShell 5.1 probe resolved the actual installed BGInfo +binary, verified cross-process mutex ownership, parsed the packaged standalone +monitor and verified redacted failure details. It did not execute BGInfo, start +startup tasks or change the current test runner. A fresh deployment must still +verify the complete startup and subsequent-logon behavior. + The combined WinGet, startup, deployment, automation, validation-retry, global LocalBox and failure-summary suites passed **267 tests** after these changes. The real upstream Bicep compiled with the 30,679-byte guarded payload, and From f8e09ee200e03dd1fe8d079bf8534bb9332516a7 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Sun, 8 Nov 2026 07:37:46 +0000 Subject: [PATCH 31/37] Document TEST14 Full health, section timings and recovered warnings Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../resources/hosted-events/readme.md | 5 + .../hosted-events/test-06-timing-analysis.md | 7 +- .../hosted-events/test-14-timing-analysis.md | 154 ++++++++++++++++++ 3 files changed, 165 insertions(+), 1 deletion(-) create mode 100644 03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/test-14-timing-analysis.md diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md index 9478469d5..8b011e8ea 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md @@ -314,6 +314,11 @@ allow time for both preparation and later assessments. For a full cold-run breakdown, see the [test-06 timing and optimization assessment](test-06-timing-analysis.md). +The [TEST14 section comparison and final warning review](test-14-timing-analysis.md) +records the 8 November Full pass: 3h46m10s of guest execution, approximately one +hour of actual AKS/image overlap, and a failed validation attempt that offset +the guest-stage savings. It distinguishes LocalBox completion from the +unverified downstream Console outcome. The 4 November 2026 run took **10 hours 30 minutes 44 seconds** from Console start to finish, including **5 hours 24 minutes 9 seconds** of managed guest execution. All 26 guest tests passed, but Console result retrieval failed and diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/test-06-timing-analysis.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/test-06-timing-analysis.md index 38957fa7b..e49e8ea2a 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/test-06-timing-analysis.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/test-06-timing-analysis.md @@ -127,7 +127,7 @@ the interval can include download, storage processing, registration and provider/controller waits. It does not by itself prove a bandwidth bottleneck. Repeated image-write events during progress updates are not separate imports. -The current [preparation script](../prepare-localbox.ps1) runs AKS creation, +At the time of this run, the [preparation script](../prepare-localbox.ps1) ran AKS creation, proxy access/MetalLB, image import, then reference VM creation **serially**. That ordering is the clearest code-level optimization opportunity. @@ -178,6 +178,11 @@ concurrent local CLI sessions. Existing-AKS retries remain serial, and manual preparation offers `-SequentialProvisioning` as a fallback. The estimates below remain historical estimates, not measured savings from this implementation. +**2026-11-08 measured update:** [TEST14](test-14-timing-analysis.md) verified +1h00m26s of actual AKS/image overlap and a 3h46m10s successful guest run. +Its validation retry nevertheless made the comparable initial-ARM-to-guest +interval longer than TEST06; the new report separates those effects. + | Priority | Candidate | Potential benefit from this run | Constraints / verification | |---|---|---|---| | 1 | Start image import alongside AKS after shared storage/network prerequisites are ready | With unchanged operation durations, overlapping the 71-minute AKS deployment with the 179-minute image import could remove roughly **71 minutes** of serial waiting | Both use the same nested environment. Storage, CPU or network contention could reduce or eliminate the saving. Use controlled concurrency, isolated CLI state, explicit error handling, and wait for both branches before full readiness | diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/test-14-timing-analysis.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/test-14-timing-analysis.md new file mode 100644 index 000000000..acb520bbf --- /dev/null +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/test-14-timing-analysis.md @@ -0,0 +1,154 @@ +# LocalBox TEST14: section timings and recovered warnings + +## Verified result and scope + +The `sov-cloud-full-automation-test-14` LocalBox guest run on `traininglab-44` +finished successfully at **07:26:19 UTC on 8 November 2026**. An independent +read-only probe verified the exact run, package, event group and Azure scope, +matching Full health JSON and both NUnit XML reports, and report timestamps. +**12/12 upstream and 14/14 Sovereign tests passed**, with none failed, skipped +or unrun. Update assessment succeeded at 07:18:55.558 UTC. + +- **Guest preparation and verification:** 03:40:08.758 to 07:26:19.066, + **3h 46m 10s**. +- **Initial LocalBox ARM start to guest completion:** 20:50:40.749 on + 7 November to 07:26:19.066 on 8 November, **10h 35m 38s**. +- This second interval excludes Console setup before ARM and all subsequent + shared/participant stages. It is **not the total Console job duration**. +- The participant resource group had no deployments or resources at the + post-completion checks. Console's final status and downstream completion + were not established by this investigation. +- Defender cloud reporting and real participant sign-in remain manual checks. + +The guest completed without operator resubmission or infrastructure recreation. +Automatic validation retry, Blob propagation handling, AKS submission +reconciliation and MetalLB recovery were all exercised. The historical startup +report still records the BGInfo failure described below; it was not rewritten +to disguise that failure. + +## Comparison with previous observations + +These are individual observations, not an SLA or a controlled benchmark. +TEST06 was a serial cold guest run; TEST13 was an explicitly repaired/resumed +instance with substantial earlier operator pauses. Only equivalent guest work +is compared with TEST13. + +| Section | TEST06, 4 November | TEST13 resumed, 7 November | TEST14, 7-8 November | +|---|---:|---:|---:| +| Initial LocalBox ARM deployment | 19m40s | Not compared | 11m55s | +| Post-ARM bootstrap to first validation start | 1h14m30s | Not compared | 1h18m48s | +| Validation window, including retry/handoff where applicable | 46m24s | Not compared | **2h48m18s** | +| Azure Local cluster deployment | 2h34m25s | Not compared | 2h28m10s | +| **Guest preparation through verified tests/finalization** | **5h24m09s** | **4h15m05s** | **3h46m10s** | +| AKS ARM deployment, overlapping image import in TEST13/14 | 1h10m59s | 47m09s | 1h01m05s | +| Image import window | 2h58m44s | 2h55m23s | 2h45m26s | +| Reference VM ARM deployment | 8m30s | Not separately retained | 8m51s | +| Reference VM ARM completion to MDE installation success | 47m44s | Not separately retained | 31m50s | +| Initial ARM start to guest completion, excluding Console setup/downstream | 10h22m23s | Not comparable | **10h35m38s** | + +TEST14's guest phase was **28m55s faster than TEST13** and **1h37m59s faster +than TEST06**. However, the comparable ARM-to-guest interval was **13m15s +longer than TEST06**: the failed validation download offset the guest-stage +improvements. + +Image durations for TEST13/14 use explicit guest timestamps immediately before +image preparation and after its readiness wait. TEST06 uses the image ARM +write interval. These are similar end-to-end import windows, but not identical +instrumentation or pure network-transfer measurements. Bootstrap intervals +are bounded between milestones, not exclusive timers for each internal step. + +## Additive TEST14 timeline + +All times are UTC; midnight separates 7 and 8 November. Durations below are +differences between independently rounded second-resolution boundaries, so +they add to exactly **10:35:38**. A duration rounded directly from fractional +timestamps may differ by one second. + +**M** means an operation/report interval; **B** means a bounded interval between +milestones, potentially containing several activities. Overlapping AKS/image +work appears once in this additive table. + +| Start | End | Duration | Section | Evidence | +|---|---|---|---|---| +| 20:50:41 | 21:02:36 | 00:11:55 | Initial LocalBox ARM deployment | M | +| 21:02:36 | 22:21:24 | 01:18:48 | Bootstrap and nested preparation before validation | B | +| 22:21:24 | 00:31:16 | 02:09:52 | First validation deployment; failed content download | M | +| 00:31:16 | 00:34:03 | 00:02:47 | Failure detection, retry launch and bounded delay | B | +| 00:34:03 | 01:09:42 | 00:35:39 | Successful validation retry | M | +| 01:09:42 | 01:09:56 | 00:00:14 | Validation-to-deployment transition | B | +| 01:09:56 | 03:38:06 | 02:28:10 | Azure Local cluster deployment | M | +| 03:38:06 | 03:40:09 | 00:02:03 | Connectivity/readiness detection and guest launch | B | +| 03:40:09 | 03:47:39 | 00:07:30 | Guest prerequisites, storage/networks and AKS preflight | B | +| 03:47:39 | 06:33:33 | 02:45:54 | AKS provisioning and image import, overlapping | B | +| 06:33:33 | 06:36:05 | 00:02:32 | AKS join, Arc/MetalLB checks and reference VM handoff | B | +| 06:36:05 | 06:44:55 | 00:08:50 | Reference VM ARM deployment | M | +| 06:44:55 | 07:09:23 | 00:24:28 | Reference VM/Arc readiness and MDE scheduling | B | +| 07:09:23 | 07:16:45 | 00:07:22 | MDE extension submission to success | M | +| 07:16:45 | 07:18:56 | 00:02:11 | Onboarding detection and successful update assessment | B | +| 07:18:56 | 07:20:29 | 00:01:33 | Assessment return, tooling and test startup | B | +| 07:20:29 | 07:20:47 | 00:00:18 | Upstream tests through XML write | M | +| 07:20:47 | 07:23:32 | 00:02:45 | Runtime-test setup and initial MetalLB reconciliation wait | B | +| 07:23:32 | 07:23:52 | 00:00:20 | Bounded MetalLB recovery, attempt 1/1 | M | +| 07:23:52 | 07:23:54 | 00:00:02 | Full test startup after recovery | B | +| 07:23:54 | 07:25:59 | 00:02:05 | Sovereign Full tests through XML write | M | +| 07:25:59 | 07:26:19 | 00:00:20 | Final status/report upload and command completion | B | +| | **Total** | **10:35:38** | Initial ARM start to guest completion | | + +The first validation's deployment-settings operation ran from 22:29:35.165 +to 00:31:03.732, within the longer parent deployment interval. Its reported +7200-second content-download timeout triggered exactly one automatic retry. +The successful retry's original-correlation check and remaining overall budget +were preserved. The retry command itself ran from 00:31:47.617 to 01:09:58.753, +including its 120-second delay, validation and deployment submission. + +## Actual AKS/image overlap + +- AKS ARM deployment: **03:47:39.478 to 04:48:44.628**. +- Guest accepted-submission log: **03:48:05.015**. +- Image preparation began: **03:48:06.910**. +- Image resource created: **03:48:18.998**. +- Image readiness observed: **06:33:33.348**; final provider state Succeeded, + progress 100%, reported download size 16,654 MB. + +From image creation until AKS completion, both operations were active for +**1h00m26s**. Thus the image did not wait for AKS to finish. Do not add the +61-minute AKS duration to the image duration when calculating the critical +path. This is measured overlap, not proof that all of the guest improvement +was caused by parallelization: image timing and onboarding also differed. + +## Final warning/error assessment + +| Finding | Final outcome | Follow-up | +|---|---|---| +| Validation content-download timeout | First attempt failed; automatic retry 1/1 succeeded | Largest remaining reliability/performance target. Investigate upstream download-stage telemetry; do not bypass validation or shorten its timeout without evidence | +| BGInfo command not found despite installation | Historical startup report remains Failed; later independent guest Full tests passed | Fixed for future deployments in `8f284c0b`: explicit binary discovery, early prerequisite check, persistent upstream test-runner patch and checked exit code. The current immutable payload was not changed | +| Startup watcher failed after task unregistration | Generic historical monitor failure did not prevent later ARM/guest progress | Fixed in `8f284c0b`: inspect the guarded process mutex, reread terminal reports on exit races, preserve bounded/redacted errors | +| Blob permission-propagation errors | Two initial retries, then successful reads/writes; final diagnostic upload succeeded | Existing bounded gate worked; no new role grants needed | +| `aksarc` post-submission `ResourceNotFound` | New matching ARM receipt accepted; original AKS deployment succeeded and runtime tests passed | Existing reconciliation worked; no duplicate create | +| Missing initial MetalLB runtime pool/L2 advertisement | One unchanged Azure-managed pool reapply succeeded in about 20 seconds; Full checks then passed | Keep bounded recovery and final Kubernetes checks. This remains an upstream/controller reconciliation concern, not justification for deleting/recreating the cluster | +| Reference VM inspection-only warning | VM is running, Arc connected, MDE installed and update assessment successful | Inspection-only is guidance, not read-only RBAC. Defender cloud reporting and actual participant sign-in still need verification | +| Az version advisory / `Get-AzStackHciCluster` breaking-change advisory | Current upstream tests passed; the latter warns about changes to `ICluster` output properties/types | Track upstream module compatibility before changing versions. Do not suppress these warnings or upgrade modules in the active run merely to remove them | + +The BGInfo failure occurred before registration of the upstream future-logon +test task; it was not merely cosmetic logging. The next fresh deployment must +verify that startup and recurring-logon behavior complete with the published +fix. The current run's successful independent Full suites do not retroactively +repair that historical startup failure. + +## Evidence and limits + +Deployment start times are derived from ARM terminal timestamp minus duration. +The original failed validation was recovered from Activity Log using its original +correlation, because the retry replaced the deployment record. Extension activity +events, original guest transcript timestamps, report file times, NUnit metadata +and the correlated final status supply the remaining boundaries. + +Managed Run Command output can be buffered: the monitor first displayed some +image-start lines hours after their embedded timestamps. This report uses the +embedded timestamps, not the later polling observation time. + +Private reports and operational snapshots remain outside the repository. +Console's screenshot does not establish a consistent UTC start/end pair; +the total Console job duration is intentionally not inferred from it. +See the [TEST06 analysis](test-06-timing-analysis.md) for the earlier baseline +and [hosted provisioning guidance](readme.md) for readiness and retry contracts. From 00487257537f54c37258dc7f2deed969fa4fc60b Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Sun, 8 Nov 2026 10:54:10 +0000 Subject: [PATCH 32/37] Add bounded participant K3s installation recovery Retry release resolution, installer download and installation with fixed-version attempts, explicit deadlines and final node readiness checks. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../labautomation/sovereign-lab.bicep | 57 ++++- .../resources/tests/k3s-install.tests.ps1 | 198 ++++++++++++++++++ .../resources/tests/readme.md | 24 +++ 3 files changed, 274 insertions(+), 5 deletions(-) create mode 100644 03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/k3s-install.tests.ps1 diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/sovereign-lab.bicep b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/sovereign-lab.bicep index 31c810d9e..6636107db 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/sovereign-lab.bicep +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/sovereign-lab.bicep @@ -49,6 +49,24 @@ var k3sInstallScript = ''' #!/usr/bin/env bash set -euo pipefail +retry_install_step() { + local step="$1" limit="$2" attempt result + shift 2 + for attempt in 1 2 3; do + echo "K3s $step: attempt $attempt/3." >&2 + if timeout --kill-after=15s "$limit" "$@"; then + return 0 + else + result=$? + fi + echo "ERROR: K3s $step failed (exit $result, attempt $attempt/3)." >&2 + if [ "$attempt" -lt 3 ]; then + sleep "$((attempt * 20))" + fi + done + return "$result" +} + install -d -m 0755 /etc/rancher/k3s cat >/etc/rancher/k3s/config.yaml <<'EOF' tls-san: @@ -59,16 +77,45 @@ disable: - traefik EOF -if command -v k3s >/dev/null 2>&1; then +if command -v k3s >/dev/null 2>&1 && systemctl cat k3s >/dev/null 2>&1; then systemctl enable --now k3s systemctl restart k3s else - curl --fail --location --silent --show-error https://get.k3s.io --output /tmp/install-k3s.sh - sh /tmp/install-k3s.sh - rm -f /tmp/install-k3s.sh + installer=$(mktemp /tmp/microhack-k3s-install.XXXXXX) + trap 'rm -f -- "$installer"' EXIT + release_url=$(retry_install_step 'release lookup' 75s curl --fail --location --silent --show-error \ + --connect-timeout 15 --max-time 60 --output /dev/null --write-out '%{url_effective}' \ + https://update.k3s.io/v1-release/channels/stable) + export INSTALL_K3S_VERSION="${release_url##*/}" + if [[ ! "$INSTALL_K3S_VERSION" =~ ^v[0-9]+\.[0-9]+\.[0-9]+\+k3s[0-9]+$ ]]; then + echo 'ERROR: Stable channel did not resolve to a valid K3s release.' >&2 + exit 1 + fi + echo "K3s selected release: $INSTALL_K3S_VERSION" + retry_install_step 'installer download' 75s curl --fail --location --silent --show-error \ + --connect-timeout 15 --max-time 60 --output "$installer" https://get.k3s.io + # Retry the installer too: it downloads and verifies the checksum and binary. + retry_install_step 'installation (including release downloads)' 600s sh "$installer" fi -systemctl is-active --quiet k3s +if ! systemctl is-active --quiet k3s; then + echo 'ERROR: K3s service is not active after installation/startup.' >&2 + exit 1 +fi +registered=false +for attempt in {1..24}; do + if nodes=$(k3s kubectl get nodes -o name --request-timeout=5s) && [ -n "$nodes" ]; then + registered=true + break + fi + echo "Waiting for K3s node registration ($attempt/24)." >&2 + sleep 5 +done +if [ "$registered" != true ]; then + echo 'ERROR: No K3s node registered within the bounded readiness window.' >&2 + exit 1 +fi +timeout --kill-after=15s 150s k3s kubectl wait --for=condition=Ready nodes --all --timeout=120s --request-timeout=10s ''' resource natPublicIp 'Microsoft.Network/publicIPAddresses@2024-05-01' = { diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/k3s-install.tests.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/k3s-install.tests.ps1 new file mode 100644 index 000000000..e31a767ea --- /dev/null +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/k3s-install.tests.ps1 @@ -0,0 +1,198 @@ +BeforeAll { + $template = Get-Content "$PSScriptRoot/../../labautomation/sovereign-lab.bicep" -Raw + $match = [regex]::Match($template, "(?s)var k3sInstallScript = '''\r?\n(.*?)\r?\n'''") + if (-not $match.Success) { throw 'K3s extension script not found.' } + $installScript = $match.Groups[1].Value + $fixture = @' +export TEST_ROOT="$1" +export SCENARIO="$2" +record() { printf '%s\n' "$*" >>"$TEST_ROOT/calls"; } +count_call() { + local count=0 + if [ -f "$TEST_ROOT/count-$1" ]; then read -r count <"$TEST_ROOT/count-$1"; fi + count=$((count + 1)) + printf '%s\n' "$count" >"$TEST_ROOT/count-$1" + printf '%s' "$count" +} +command() { + if [ "$1" = -v ] && [ "$2" = k3s ]; then + [[ "$SCENARIO" = existing || "$SCENARIO" = partial ]] + else + builtin command "$@" + fi +} +install() { record install; } +mktemp() { builtin command mktemp "$TEST_ROOT/installer.XXXXXX"; } +sleep() { record "sleep $*"; } +timeout() { + record "timeout $*" + shift 2 + if [ "$SCENARIO" = timeout ] && [ "$1" = sh ]; then return 124; fi + "$@" +} +curl() { + record "curl $*" + if [[ "$*" = *update.k3s.io* ]]; then + local count + count=$(count_call lookup) + if [ "$SCENARIO" = lookup-failure ]; then return 22; fi + if [ "$SCENARIO" = lookup-retry ] && [ "$count" -lt 3 ]; then return 28; fi + if [ "$SCENARIO" = invalid-release ]; then + printf 'https://example.invalid/stable' + else + printf 'https://github.com/k3s-io/k3s/releases/tag/v1.37.2+k3s1' + fi + else + local count + count=$(count_call installer) + if [ "$SCENARIO" = installer-failure ]; then return 22; fi + if [ "$SCENARIO" = installer-retry ] && [ "$count" -lt 3 ]; then return 28; fi + fi +} +sh() { + local count + count=$(count_call installation) + record "installation $count version=$INSTALL_K3S_VERSION" + if [ "$SCENARIO" = persistent ]; then return 1; fi + if [[ "$SCENARIO" = hash-retry || "$SCENARIO" = binary-retry ]] && [ "$count" -lt 3 ]; then + echo "ERROR: $SCENARIO download failed" >&2 + return 1 + fi +} +systemctl() { + record "systemctl $*" + if [ "$1" = cat ]; then [ "$SCENARIO" = existing ]; return; fi + if [ "$1" = is-active ] && [ "$SCENARIO" = inactive ]; then return 3; fi + return 0 +} +k3s() { + record "k3s $*" + if [ "$2" = get ]; then + local count + count=$(count_call registration) + if [ "$SCENARIO" = no-node ]; then return 0; fi + if [ "$SCENARIO" = api-failure ]; then echo 'API unavailable' >&2; return 1; fi + if [ "$SCENARIO" = delayed-node ] && [ "$count" -lt 3 ]; then return 0; fi + echo node/test-node + elif [ "$SCENARIO" = not-ready ]; then + echo 'ERROR: Node readiness timed out' >&2 + return 1 + fi +} +'@ + function Invoke-K3sFixture { + param([string]$Scenario) + $directory = Join-Path $TestDrive ([guid]::NewGuid().ToString()) + $null = New-Item -ItemType Directory $directory + $scriptPath = Join-Path $directory 'run.sh' + $source = $installScript.Replace('/etc/rancher/k3s/config.yaml', "$directory/config.yaml") + [IO.File]::WriteAllText($scriptPath, ($fixture + "`n" + $source).Replace("`r`n", "`n")) + $output = & bash $scriptPath $directory $Scenario 2>&1 + $exitCode = $LASTEXITCODE + [pscustomobject]@{ + ExitCode = $exitCode + Output = $output -join "`n" + Calls = @(Get-Content "$directory/calls") + Installers = @(Get-ChildItem "$directory/installer.*") + } + } +} + +Describe 'K3s extension bounded recovery' -Skip:(-not (Get-Command bash -ErrorAction SilentlyContinue)) { + It 'has valid Bash syntax' { + $path = Join-Path $TestDrive 'syntax.sh' + [IO.File]::WriteAllText($path, $installScript) + & bash -n $path + $LASTEXITCODE | Should -Be 0 + } + It 'installs once and verifies node readiness' { + $result = Invoke-K3sFixture success + $result.ExitCode | Should -Be 0 + @($result.Calls -like 'installation *').Count | Should -Be 1 + @($result.Calls -like 'sleep *').Count | Should -Be 0 + @($result.Calls -like 'timeout --kill-after=15s 600s sh *').Count | Should -Be 1 + $result.Calls | Should -Contain 'timeout --kill-after=15s 150s k3s kubectl wait --for=condition=Ready nodes --all --timeout=120s --request-timeout=10s' + $result.Installers.Count | Should -Be 0 + } + It 'retries with fixed version and 20/40-second backoff' -TestCases @( + @{ Scenario = 'hash-retry' } + @{ Scenario = 'binary-retry' } + ) { + param($Scenario) + $result = Invoke-K3sFixture $Scenario + $result.ExitCode | Should -Be 0 + @($result.Calls -like 'curl *update.k3s.io*').Count | Should -Be 1 + @($result.Calls -like 'installation *version=v1.37.2+k3s1').Count | Should -Be 3 + ($result.Calls -like 'sleep *') -join ',' | Should -Be 'sleep 20,sleep 40' + $result.Installers.Count | Should -Be 0 + } + It 'bounds failures and never reports readiness' -TestCases @( + @{ Scenario = 'persistent'; Code = 1 } + @{ Scenario = 'timeout'; Code = 124 } + @{ Scenario = 'lookup-failure'; Code = 22 } + @{ Scenario = 'installer-failure'; Code = 22 } + ) { + param($Scenario, $Code) + $result = Invoke-K3sFixture $Scenario + $result.ExitCode | Should -Be $Code + ($result.Calls -like 'sleep *') -join ',' | Should -Be 'sleep 20,sleep 40' + $result.Output | Should -Match "failed \(exit $Code, attempt 3/3\)" + @($result.Calls -like 'k3s *').Count | Should -Be 0 + $result.Installers.Count | Should -Be 0 + } + It 'recovers before installing' -TestCases @( + @{ Scenario = 'lookup-retry' } + @{ Scenario = 'installer-retry' } + ) { + param($Scenario) + $result = Invoke-K3sFixture $Scenario + $result.ExitCode | Should -Be 0 + ($result.Calls -like 'sleep *') -join ',' | Should -Be 'sleep 20,sleep 40' + @($result.Calls -like 'installation *').Count | Should -Be 1 + } + It 'rejects malformed release selection without invoking the installer' { + $result = Invoke-K3sFixture invalid-release + $result.ExitCode | Should -Be 1 + $result.Output | Should -Match 'did not resolve to a valid K3s release' + @($result.Calls -like 'installation *').Count | Should -Be 0 + } + It 'preserves existing-service behavior without downloading again' { + $result = Invoke-K3sFixture existing + $result.ExitCode | Should -Be 0 + @($result.Calls -like 'curl *').Count | Should -Be 0 + $result.Calls | Should -Contain 'systemctl enable --now k3s' + $result.Calls | Should -Contain 'systemctl restart k3s' + } + It 'repairs a partial install with a binary but no service' { + $result = Invoke-K3sFixture partial + $result.ExitCode | Should -Be 0 + @($result.Calls -like 'installation *').Count | Should -Be 1 + } + It 'waits for registration before checking readiness' { + $result = Invoke-K3sFixture delayed-node + $result.ExitCode | Should -Be 0 + ($result.Calls -like 'sleep *') -join ',' | Should -Be 'sleep 5,sleep 5' + @($result.Calls -like 'k3s kubectl get *').Count | Should -Be 3 + @($result.Calls -like 'k3s kubectl wait *').Count | Should -Be 1 + } + It 'fails explicitly on after bounded registration checks' -TestCases @( + @{ Scenario = 'no-node' } + @{ Scenario = 'api-failure' } + ) { + param($Scenario) + $result = Invoke-K3sFixture $Scenario + $result.ExitCode | Should -Be 1 + @($result.Calls -like 'k3s kubectl get *').Count | Should -Be 24 + @($result.Calls -like 'k3s kubectl wait *').Count | Should -Be 0 + $result.Output | Should -Match 'No K3s node registered' + } + It 'does not reinstall to mask ' -TestCases @( + @{ Scenario = 'inactive'; Code = 1 } + @{ Scenario = 'not-ready'; Code = 1 } + ) { + param($Scenario, $Code) + $result = Invoke-K3sFixture $Scenario + $result.ExitCode | Should -Be $Code + @($result.Calls -like 'installation *').Count | Should -Be 1 + } +} diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md index 209fd9534..04b536859 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md @@ -16,6 +16,30 @@ From this directory, run `Invoke-Pester ./visual-attestation.tests.ps1 -Output D A fresh Codespace smoke test is still required to validate the devcontainer build, Docker daemon, `confcom` policy generation and Azure deployment together; the mocked tests do not establish end-to-end Codespaces support. +## Participant K3s installation checks + +Run `Invoke-Pester ./k3s-install.tests.ps1 -Output Detailed` on a host with Bash. +These offline tests execute the actual Bicep-embedded installation script with +mocked download, installer, service and Kubernetes commands. They cover successful +installation, checksum/binary download failures, three-attempt exhaustion, +20/40-second backoff, timeout propagation, stable-version selection across retries, +installer cleanup, partial/existing installations, delayed node registration, +API failures and node readiness failures. They do not download or install K3s. + +The extension retries each release lookup, installer download and installer +execution at most three times. Lookup/download attempts have 60-second curl +limits and 75-second process limits; each installer attempt has a 600-second +process limit, followed by forced termination after a 15-second grace period if +needed. The stable channel is resolved once successfully per extension execution, +then the same version is used for all installer attempts. Upstream checksum +verification remains enabled. Registration is checked up to 24 times, with +5-second request limits and 5-second delays, followed by a bounded Ready check. +Persistent failures remain extension failures. No retry deletes infrastructure. + +The observed failed VM was recovered by a separately authorized manual retry; +that does not validate this new automatic retry in a fresh deployment. Event-lead +testing should use the feature branch containing these changes, not `main`. + ## MetalLB preparation checks Run `Invoke-Pester ./localbox-metallb-recovery.tests.ps1,./localbox-automation.tests.ps1 -Output Detailed` From b21ac0b748676f70550f493b785d15a80cbb6939 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Sun, 8 Nov 2026 20:08:06 +0000 Subject: [PATCH 33/37] Check BGInfo process exit instead of stale PowerShell status Await the resolved GUI process with a bounded timeout, retain explicit BGInfo error logs, and verify stale-exit and real-failure handling. Validated with 302 tests and live PowerShell 7/5.1 probes. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../labautomation/localbox-startup.ps1 | 37 ++++++- .../tests/localbox-startup.tests.ps1 | 96 ++++++++++++++++--- .../resources/tests/readme.md | 21 ++++ 3 files changed, 137 insertions(+), 17 deletions(-) diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-startup.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-startup.ps1 index 235323b0c..356503351 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-startup.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-startup.ps1 @@ -291,15 +291,44 @@ function Resolve-LocalBoxBgInfo { } function Invoke-LocalBoxBgInfo { - param([string]$ConfigurationPath) + param([string]$ConfigurationPath, [ValidateRange(1, 120)][int]$TimeoutSeconds = 60) $ErrorActionPreference = 'Stop' if (-not (Test-Path -LiteralPath $ConfigurationPath -PathType Leaf)) { throw 'The LocalBox BGInfo configuration is missing.' } $client = Resolve-LocalBoxBgInfo - $PSNativeCommandUseErrorActionPreference = $false - & $client $ConfigurationPath /timer:0 /NOLICPROMPT - if ($LASTEXITCODE -ne 0) { throw "BGInfo failed with exit code $LASTEXITCODE." } + $log = Join-Path ([IO.Path]::GetTempPath()) "localbox-bginfo-$([guid]::NewGuid()).log" + $process = $null + try { + # GUI applications can leave LASTEXITCODE unchanged; inspect the actual process. + $process = Start-Process -FilePath $client -ArgumentList @("`"$ConfigurationPath`"", + '/timer:0', '/NOLICPROMPT', '/silent', "/log:`"$log`"") -PassThru -ErrorAction Stop + if (-not $process.WaitForExit($TimeoutSeconds * 1000)) { + throw "BGInfo exceeded its $TimeoutSeconds-second budget. Inspect $log if present." + } + $process.WaitForExit() + if ($null -eq $process.ExitCode -or $process.ExitCode -ne 0) { + throw "BGInfo failed with exit code '$($process.ExitCode)'. Inspect $log if present." + } + if (Test-Path -LiteralPath $log -PathType Leaf) { + $message = ([string](Get-Content -LiteralPath $log -Raw -ErrorAction Stop)).Trim() + if ($message) { + throw "BGInfo reported an error: $(ConvertTo-LocalBoxDiagnosticText $message 400). Inspect $log." + } + Remove-Item -LiteralPath $log -ErrorAction Stop + } + } + finally { + if ($process) { + try { + if (-not $process.HasExited) { + $process.Kill() + if (-not $process.WaitForExit(10000)) { throw 'BGInfo process did not stop after its timeout.' } + } + } + finally { $process.Dispose() } + } + } } function Wait-LocalBoxStartupPrerequisites { diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-startup.tests.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-startup.tests.ps1 index a90307783..0c659549e 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-startup.tests.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-startup.tests.ps1 @@ -696,30 +696,100 @@ Describe 'Checked BGInfo invocation' { BeforeEach { $configuration = "$TestDrive/configuration with spaces.bgi" Set-Content $configuration fixture - $executable = "$TestDrive/bginfo fixture.ps1" + $executable = "$TestDrive/bginfo fixture.exe" Mock Resolve-LocalBoxBgInfo { $executable } + $log = "$TestDrive/bginfo diagnostics.log" + Mock Join-Path { $log } -ParameterFilter { $ChildPath -like 'localbox-bginfo-*.log' } + $processState = @{ Finished = $true; Killed = $false; Disposed = $false; Waits = [Collections.Generic.List[int]]::new() } + $fakeProcess = [pscustomobject]@{ HasExited = $true; ExitCode = 0 } + $fakeProcess | Add-Member ScriptMethod WaitForExit { + param($milliseconds) + if ($null -ne $milliseconds) { + $processState.Waits.Add($milliseconds) + return $processState.Finished + } + } + $fakeProcess | Add-Member ScriptMethod Kill { + $processState.Killed = $true + $processState.Finished = $true + $this.HasExited = $true + } + $fakeProcess | Add-Member ScriptMethod Dispose { $processState.Disposed = $true } + Mock Start-Process { $fakeProcess } $previousExitCode = $global:LASTEXITCODE } AfterEach { $global:LASTEXITCODE = $previousExitCode } It 'passes the exact configuration and noninteractive flags to the resolved executable' { - @' -$args | ConvertTo-Json | Set-Content "$PSScriptRoot/bginfo-arguments.json" -$global:LASTEXITCODE = 0 -'@ | Set-Content $executable Invoke-LocalBoxBgInfo $configuration - $arguments = Get-Content "$TestDrive/bginfo-arguments.json" -Raw | ConvertFrom-Json - $arguments.Count | Should -Be 3 - $arguments[0] | Should -Be $configuration - $arguments[1] | Should -Be '/timer:0' - $arguments[2] | Should -Be '/NOLICPROMPT' + Should -Invoke Start-Process -Times 1 -Exactly -ParameterFilter { + $FilePath -eq $executable -and $PassThru -and $ErrorAction -eq 'Stop' -and + $ArgumentList.Count -eq 5 -and $ArgumentList[0] -eq "`"$configuration`"" -and + $ArgumentList[1] -eq '/timer:0' -and $ArgumentList[2] -eq '/NOLICPROMPT' -and + $ArgumentList[3] -eq '/silent' -and $ArgumentList[4] -eq "/log:`"$log`"" + } + $processState.Waits | Should -Contain 60000 + $processState.Killed | Should -BeFalse + $processState.Disposed | Should -BeTrue + } + It 'continues after a successful GUI process despite stale LASTEXITCODE ' -TestCases @( + @{ Seed = 1 }; @{ Seed = 5 }; @{ Seed = $null } + ) { + param($Seed) + $global:LASTEXITCODE = $Seed + Invoke-LocalBoxBgInfo $configuration + $global:LASTEXITCODE | Should -Be $Seed + $processState.Disposed | Should -BeTrue } - It 'fails on a nonzero exit code' { - '$global:LASTEXITCODE = 5' | Set-Content $executable - { Invoke-LocalBoxBgInfo $configuration } | Should -Throw '*BGInfo failed with exit code 5*' + It 'rejects actual unsuccessful or missing process exit code ' -TestCases @( + @{ Code = 1 }; @{ Code = 5 }; @{ Code = $null } + ) { + param($Code) + $global:LASTEXITCODE = 0 + $fakeProcess.ExitCode = $Code + { Invoke-LocalBoxBgInfo $configuration } | Should -Throw '*BGInfo failed with exit code*' + $processState.Disposed | Should -BeTrue + } + It 'propagates process launch failures' { + Mock Start-Process { throw 'fixture-launch-denied' } + { Invoke-LocalBoxBgInfo $configuration } | Should -Throw '*fixture-launch-denied*' + $processState.Disposed | Should -BeFalse + } + It 'kills only its own timed-out process and disposes the handle' { + $processState.Finished = $false + $fakeProcess.HasExited = $false + { Invoke-LocalBoxBgInfo $configuration -TimeoutSeconds 1 } | Should -Throw '*exceeded its 1-second budget*' + $processState.Waits | Should -Contain 1000 + $processState.Waits | Should -Contain 10000 + $processState.Killed | Should -BeTrue + $processState.Disposed | Should -BeTrue + } + It 'surfaces termination failure and still disposes the handle' { + $processState.Finished = $false + $fakeProcess.HasExited = $false + $fakeProcess | Add-Member ScriptMethod Kill { throw 'fixture-kill-denied' } -Force + { Invoke-LocalBoxBgInfo $configuration -TimeoutSeconds 1 } | Should -Throw '*fixture-kill-denied*' + $processState.Disposed | Should -BeTrue + } + It 'rejects an error log even when the process exits zero and redacts its diagnostic' { + Set-Content $log ('token=fixture-secret ' + ('x' * 800)) + $failure = $null + try { Invoke-LocalBoxBgInfo $configuration } catch { $failure = $_ } + $failure | Should -Not -BeNullOrEmpty + $failure.Exception.Message | Should -Match 'BGInfo reported an error' + $failure.Exception.Message | Should -Not -Match 'fixture-secret' + $failure.Exception.Message.Length | Should -BeLessThan 650 + Test-Path $log | Should -BeTrue + $processState.Disposed | Should -BeTrue + } + It 'removes an empty success log' { + Set-Content $log '' + Invoke-LocalBoxBgInfo $configuration + Test-Path $log | Should -BeFalse } It 'does not launch BGInfo without its configuration' { { Invoke-LocalBoxBgInfo "$TestDrive/missing.bgi" } | Should -Throw '*configuration is missing*' Should -Invoke Resolve-LocalBoxBgInfo -Times 0 + Should -Invoke Start-Process -Times 0 } } diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md index 04b536859..73e3b93ec 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md @@ -106,6 +106,27 @@ monitor and verified redacted failure details. It did not execute BGInfo, start startup tasks or change the current test runner. A fresh deployment must still verify the complete startup and subsequent-logon behavior. +TEST15 isolated the remaining BGInfo failure to stale PowerShell `LASTEXITCODE`, +not a failed BGInfo process. In the existing desktop session, direct GUI invocation +preserved seeded exit codes of both 1 and 0, while an explicitly awaited process +returned 0. The helper now uses `Start-Process -PassThru`, waits up to 60 seconds, +checks the actual process exit code and rejects errors written by BGInfo's `/log` +option. `/silent` prevents unattended error dialogs; it does not suppress failure +reporting. A timed-out process is terminated by its own handle, and failure logs +remain in the invoking user's temporary directory for diagnosis. + +The updated coupled suites passed **302 tests**. Tests cover stale caller exit +codes, actual nonzero/missing exits, launch errors, timeout/termination cleanup, +quoted paths, error-log redaction and successful log cleanup. Real BGInfo 4.34 +was then verified in the TEST15 desktop session under PowerShell 7.6.9 and Windows +PowerShell 5.1: the original configuration succeeded despite a seeded exit code +of 1, while an invalid configuration failed with the actual exit code 2. +Temporary probe tasks/files were removed. The upstream Bicep and script transforms +compiled with a **34,442-byte** startup payload. This is a future-deployment fix: +the existing immutable startup payload and historical failure report were not +rewritten, and storage initialization was not replayed. A fresh deployment remains +the integration check for the original runner's subsequent-logon task registration. + The combined WinGet, startup, deployment, automation, validation-retry, global LocalBox and failure-summary suites passed **267 tests** after these changes. The real upstream Bicep compiled with the 30,679-byte guarded payload, and From e1b47fa693711d726e61c3d5c2cc68f0877b1f15 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Mon, 9 Nov 2026 10:17:42 +0000 Subject: [PATCH 34/37] Account for existing participant compute when retrying quota checks Preserve the saved region and confidential SKU, credit allocated lab compute, and reserve only missing capacity. Add quota resume regression tests. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../labautomation/deploy-lab.ps1 | 28 +- .../labautomation/quota-helpers.ps1 | 169 +++++++++++ .../labautomation/shared-deploy-lab.ps1 | 44 ++- .../resources/tests/global-localbox.tests.ps1 | 3 + .../resources/tests/quota-reuse.tests.ps1 | 271 ++++++++++++++++++ .../resources/tests/readme.md | 20 ++ .../resources/tests/shared-aks.tests.ps1 | 2 +- 7 files changed, 506 insertions(+), 31 deletions(-) create mode 100644 03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/quota-reuse.tests.ps1 diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/deploy-lab.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/deploy-lab.ps1 index bc01d6568..18e40e2b8 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/deploy-lab.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/deploy-lab.ps1 @@ -54,15 +54,9 @@ if($PreferredLocation.Count -gt 0) { $deploymentLocations = @('swedencentral') } -$selection = Get-MhhConfidentialComputeSelection -SubscriptionId $SubscriptionId -$validCandidates = @(Get-MhhConfidentialComputeCandidates -PreferredLocation $deploymentLocations) -$selectedCandidate = $validCandidates | - Where-Object { - $_.Location -ieq $selection.Location -and - $_.VmSize -ieq $selection.VmSize -and - $_.QuotaName -ieq $selection.QuotaName - } | - Select-Object -First 1 +$existingCompute = Get-MhhSovereignExistingCompute -SubscriptionId $SubscriptionId ` + -NameSuffixes @($stableHash.Substring(0, 8)) -ResourceGroupName $ResourceGroupName +$selectedCandidate = Get-MhhSovereignResumeCandidate -SubscriptionId $SubscriptionId -ExistingCompute $existingCompute if(-not $selectedCandidate) { throw 'The shared subscription preparation did not persist a valid confidential compute selection for this deployment.' } @@ -70,10 +64,13 @@ if(-not $selectedCandidate) { $confidentialVmSize = $selectedCandidate.VmSize $confidentialQuotaName = $selectedCandidate.QuotaName $deploymentLocations = @($selectedCandidate.Location) -$quotaRequirements = @(Get-MhhSovereignComputeRequirements -ConfidentialQuotaName $confidentialQuotaName) +$quotaRequirements = @(Get-MhhSovereignAdditionalCompute -Candidate $selectedCandidate -ExistingCompute $existingCompute) $regionReadinessSummary = @() foreach($candidateLocation in $deploymentLocations) { - $requiredVmSizes = @($confidentialVmSize, 'Standard_D4s_v5') + $requiredVmSizes = @( + if (($quotaRequirements | Where-Object Name -eq $confidentialQuotaName).Additional -gt 0) { $confidentialVmSize } + if (($quotaRequirements | Where-Object Name -eq 'StandardDSv5Family').Additional -gt 0) { 'Standard_D4s_v5' } + ) $availableVmSizes = Get-AzComputeResourceSku -Location $candidateLocation -ErrorAction Stop | Where-Object { $_.ResourceType -ieq 'virtualMachines' -and @@ -84,8 +81,7 @@ foreach($candidateLocation in $deploymentLocations) { $unavailableVmSizes = @($requiredVmSizes | Where-Object { $_ -inotin $availableVmSizes }) if($unavailableVmSizes.Count -gt 0) { - $regionReadinessSummary += "${candidateLocation}: unavailable VM sizes $($unavailableVmSizes -join ', ')" - continue + throw "${candidateLocation}: unavailable VM sizes $($unavailableVmSizes -join ', ') for missing participant compute." } $regionalUsage = Get-AzVMUsage -Location $candidateLocation -ErrorAction Stop @@ -96,10 +92,10 @@ foreach($candidateLocation in $deploymentLocations) { $current = if($null -eq $quota) { 0 } else { $quota.CurrentValue } $limit = if($null -eq $quota) { 0 } else { $quota.Limit } $available = $limit - $current - $regionReadinessSummary += "${candidateLocation}/$($requirement.Name): ${current}/${limit} vCPUs used" + $regionReadinessSummary += "${candidateLocation}/$($requirement.Name): ${current}/${limit} vCPUs used; $($requirement.Existing) already allocated to this lab; $($requirement.Additional) additional required" - if($available -lt $requirement.PerParticipant) { - throw "The shared compute selection is no longer ready ($($regionReadinessSummary -join '; ')). $($requirement.PerParticipant) available $($requirement.Name) vCPUs are required per participant." + if($requirement.Additional -gt 0 -and $available -lt $requirement.Additional) { + throw "The shared compute selection is no longer ready ($($regionReadinessSummary -join '; ')). $($requirement.Additional) additional $($requirement.Name) vCPUs are required for this participant." } } } diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/quota-helpers.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/quota-helpers.ps1 index e5a9c57f4..aaf39a6a9 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/quota-helpers.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/quota-helpers.ps1 @@ -79,6 +79,175 @@ function Get-MhhConfidentialComputeSelection { } } +function Get-MhhSovereignArmData { + param([Parameter(Mandatory)][string]$Path) + + do { + $arguments = @{ Method = 'GET'; ErrorAction = 'Stop' } + if ($Path.StartsWith('https://')) { $arguments.Uri = $Path } + else { $arguments.Path = $Path } + $response = Invoke-AzRestMethod @arguments + if ($response.StatusCode -ne 200) { + throw "Unable to inventory existing lab compute: GET $Path returned HTTP $($response.StatusCode)." + } + $data = $response.Content | ConvertFrom-Json -ErrorAction Stop + if ($null -eq $data) { throw "Empty compute inventory response from $Path." } + if ($data.PSObject.Properties.Name -contains 'value') { + if ($null -eq $data.value) { throw "Invalid compute inventory collection from $Path." } + $data.value + } + else { $data } + $Path = $data.nextLink + } while ($Path) +} + +function Test-MhhSovereignAllocatedInstance { + param($Statuses, [string]$ResourceId) + + $power = @($Statuses | Where-Object { $_.code -like 'PowerState/*' }) + if ($power.Count -ne 1) { + Write-Warning "No unambiguous power state for $ResourceId; not crediting it against new compute quota." + return $false + } + return $power[0].code -in @('PowerState/running', 'PowerState/stopped') +} + +function Get-MhhSovereignExistingCompute { + param( + [Parameter(Mandatory)][string]$SubscriptionId, + [string[]]$NameSuffixes = @(), + [string]$ResourceGroupName + ) + + $scope = "/subscriptions/$SubscriptionId" + $resources = @() + foreach ($kind in @( + @{ Type = 'Microsoft.Compute/virtualMachines'; Version = '2024-11-01'; Prefix = 'vm-k3s-' }, + @{ Type = 'Microsoft.ContainerService/managedClusters'; Version = '2024-10-01'; Prefix = 'aks-sovereign-' }, + @{ Type = 'Microsoft.Network/virtualNetworks'; Version = '2024-05-01'; Prefix = 'vnet-sovereign-' } + )) { + $names = @($NameSuffixes | ForEach-Object { "$($kind.Prefix)$_" }) + $listed = @(Get-MhhSovereignArmData -Path "$scope/providers/$($kind.Type)?api-version=$($kind.Version)") + if (-not $NameSuffixes.Count -and @($listed | Where-Object { $_.tags.workload -eq 'sovereign-lab' }).Count) { + throw 'Participant IDs are required to safely account for existing Sovereign Cloud labs.' + } + $resources += @($listed | Where-Object { + $_.name -iin $names -and + (-not $ResourceGroupName -or $_.id.Split('/')[4] -ieq $ResourceGroupName) + }) + } + foreach ($resource in $resources) { + if ($resource.tags.workload -ne 'sovereign-lab') { + throw "Existing resource $($resource.id) does not carry the expected sovereign-lab ownership tag." + } + } + $duplicates = @($resources | Group-Object type, name | Where-Object Count -gt 1) + if ($duplicates.Count) { throw 'Ambiguous participant compute names across resource groups; refusing quota reuse.' } + + $allocations = @() + $confidentialSizes = @() + $clusters = @($resources | Where-Object type -eq 'Microsoft.ContainerService/managedClusters') + $scaleSets = if ($clusters.Count) { + @(Get-MhhSovereignArmData -Path "$scope/providers/Microsoft.Compute/virtualMachineScaleSets?api-version=2024-11-01") + } else { @() } + foreach ($vm in @($resources | Where-Object type -eq 'Microsoft.Compute/virtualMachines')) { + if ($vm.properties.hardwareProfile.vmSize -ine 'Standard_D4s_v5') { + throw "Existing K3s VM $($vm.id) has an incompatible size; refusing automatic resize or quota reuse." + } + $view = Get-MhhSovereignArmData -Path "$($vm.id)/instanceView?api-version=2024-11-01" + if (Test-MhhSovereignAllocatedInstance -Statuses $view.statuses -ResourceId $vm.id) { + $allocations += [pscustomobject]@{ Location = $vm.location; VmSize = 'Standard_D4s_v5'; Count = 1 } + } + } + foreach ($cluster in $clusters) { + foreach ($poolName in @('system', 'cvmnodepool')) { + $pool = @($cluster.properties.agentPoolProfiles | Where-Object name -eq $poolName) + if (-not $pool.Count) { continue } + if ($pool.Count -ne 1) { throw "Ambiguous $poolName pool on $($cluster.id)." } + $size = $pool[0].vmSize + if (($poolName -eq 'system' -and $size -ine 'Standard_D4s_v5') -or + ($poolName -eq 'cvmnodepool' -and $size -inotin @('Standard_DC2as_v5', 'Standard_DC2as_v6'))) { + throw "Existing AKS pool $($cluster.id)/$poolName has an incompatible size." + } + if ($poolName -eq 'cvmnodepool') { $confidentialSizes += $size } + $allocated = 0 + foreach ($scaleSet in @($scaleSets | Where-Object { + $_.id.Split('/')[4] -ieq $cluster.properties.nodeResourceGroup -and + $_.tags.'aks-managed-poolName' -eq $poolName + })) { + if ($scaleSet.sku.name -ine $size -or $scaleSet.location -ine $cluster.location) { + throw "Existing AKS scale set $($scaleSet.id) conflicts with its pool configuration." + } + $instances = @(Get-MhhSovereignArmData -Path "$($scaleSet.id)/virtualMachines?api-version=2024-11-01&`$expand=instanceView") + foreach ($instance in $instances) { + if (Test-MhhSovereignAllocatedInstance -Statuses $instance.properties.instanceView.statuses -ResourceId $instance.id) { + $allocated++ + } + } + } + # Do not let surge nodes or an oversized pool pay for another participant's missing nodes. + $allocations += [pscustomobject]@{ Location = $cluster.location; VmSize = $size; Count = [Math]::Min(2, $allocated) } + } + } + return [pscustomobject]@{ + Resources = $resources + Allocations = $allocations + ConfidentialVmSizes = @($confidentialSizes | Select-Object -Unique) + } +} + +function Get-MhhSovereignResumeCandidate { + param([string]$SubscriptionId, $ExistingCompute) + + $selection = Get-MhhConfidentialComputeSelection -SubscriptionId $SubscriptionId + if (-not $selection.Location -and -not $selection.VmSize -and -not $selection.QuotaName) { + if ($ExistingCompute.Resources.Count) { + throw 'Existing lab resources have no saved compute selection. Restore the subscription selection tags before retrying.' + } + return $null + } + $candidate = @(Get-MhhConfidentialComputeCandidates -PreferredLocation @($selection.Location) | Where-Object { + $_.Location -and $_.VmSize -ieq $selection.VmSize -and $_.QuotaName -ieq $selection.QuotaName + }) + if ($candidate.Count -ne 1) { throw 'The saved compute selection is invalid; refusing to relocate existing labs.' } + if (@($ExistingCompute.Resources | Where-Object { $_.location -ine $selection.Location }).Count -or + @($ExistingCompute.ConfidentialVmSizes | Where-Object { $_ -ine $selection.VmSize }).Count) { + throw 'Existing lab regions or confidential VM sizes conflict with the saved compute selection; refusing to relocate or resize labs.' + } + return $candidate[0] +} + +function Get-MhhSovereignAdditionalCompute { + param( + [Parameter(Mandatory)]$Candidate, + [Parameter(Mandatory)]$ExistingCompute, + [ValidateRange(1, 100000)][int]$ParticipantCount = 1 + ) + + $credited = @{ StandardDSv5Family = 0; cores = 0 } + $credited[$Candidate.QuotaName] = 0 + foreach ($allocation in $ExistingCompute.Allocations) { + if ($allocation.Location -ine $Candidate.Location) { continue } + if ($allocation.VmSize -ieq 'Standard_D4s_v5') { + $credited.StandardDSv5Family += 4 * $allocation.Count + $credited.cores += 4 * $allocation.Count + } + elseif ($allocation.VmSize -ieq $Candidate.VmSize) { + $credited[$Candidate.QuotaName] += 2 * $allocation.Count + $credited.cores += 2 * $allocation.Count + } + } + foreach ($requirement in @(Get-MhhSovereignComputeRequirements -ConfidentialQuotaName $Candidate.QuotaName)) { + $desired = $requirement.PerParticipant * $ParticipantCount + [pscustomobject]@{ + Name = $requirement.Name + Desired = $desired + Existing = [Math]::Min($desired, $credited[$requirement.Name]) + Additional = [Math]::Max(0, $desired - $credited[$requirement.Name]) + } + } +} + function Wait-MhhQuotaRequest { param( [string]$OperationStatusUrl, diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/shared-deploy-lab.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/shared-deploy-lab.ps1 index c267482d1..ad8426f36 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/shared-deploy-lab.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/shared-deploy-lab.ps1 @@ -53,26 +53,42 @@ if($participantCount -lt 1) { } $confidentialCandidates = @(Get-MhhConfidentialComputeCandidates -PreferredLocation $PreferredLocation) +$nameSuffixes = @($AllowedEntraUserIds | Where-Object { $_ } | Select-Object -Unique | ForEach-Object { + (Get-MhhStableHash -Value @($_) -Length 24).Substring(0, 8) +}) $selectedCandidate = $null $quotaReadinessSummary = @() -for($attempt = 1; $attempt -le 30; $attempt++) { - Update-MhhToken | Out-Null - $quotaProvider = Get-AzResourceProvider -ProviderNamespace Microsoft.Quota -ErrorAction Stop - if($quotaProvider.RegistrationState -eq 'Registered') { - break +foreach ($providerNamespace in @('Microsoft.Compute', 'Microsoft.ContainerService', 'Microsoft.Network', 'Microsoft.Quota')) { + for($attempt = 1; $attempt -le 30; $attempt++) { + Update-MhhToken | Out-Null + $provider = Get-AzResourceProvider -ProviderNamespace $providerNamespace -ErrorAction Stop + if($provider.RegistrationState -eq 'Registered') { + break + } + Write-Host "Waiting for $providerNamespace registration ($attempt/30)..." + Start-Sleep -Seconds 10 + } + if($provider.RegistrationState -ne 'Registered') { + throw "$providerNamespace did not reach Registered state within 5 minutes." } - Write-Host "Waiting for Microsoft.Quota registration ($attempt/30)..." - Start-Sleep -Seconds 10 } -if($quotaProvider.RegistrationState -ne 'Registered') { - throw 'Microsoft.Quota did not reach Registered state within 5 minutes.' + +$existingCompute = Get-MhhSovereignExistingCompute -SubscriptionId $SubscriptionId -NameSuffixes $nameSuffixes +$resumeCandidate = Get-MhhSovereignResumeCandidate -SubscriptionId $SubscriptionId -ExistingCompute $existingCompute +if ($resumeCandidate) { + $confidentialCandidates = @($resumeCandidate) + Write-Host "Preserving the saved compute selection: $($resumeCandidate.VmSize) in $($resumeCandidate.Location)." } foreach($candidate in $confidentialCandidates) { Update-MhhToken | Out-Null $candidateLocation = $candidate.Location - $requiredVmSizes = @($candidate.VmSize, 'Standard_D4s_v5') + $quotaRequirements = @(Get-MhhSovereignAdditionalCompute -Candidate $candidate -ExistingCompute $existingCompute -ParticipantCount $participantCount) + $requiredVmSizes = @( + if (($quotaRequirements | Where-Object Name -eq $candidate.QuotaName).Additional -gt 0) { $candidate.VmSize } + if (($quotaRequirements | Where-Object Name -eq 'StandardDSv5Family').Additional -gt 0) { 'Standard_D4s_v5' } + ) $availableVmSizes = Get-AzComputeResourceSku -Location $candidateLocation -ErrorAction Stop | Where-Object { $_.ResourceType -ieq 'virtualMachines' -and @@ -89,23 +105,23 @@ foreach($candidate in $confidentialCandidates) { $regionalUsage = Get-AzVMUsage -Location $candidateLocation -ErrorAction Stop $locationReady = $true - $quotaRequirements = @(Get-MhhSovereignComputeRequirements -ConfidentialQuotaName $candidate.QuotaName) foreach($requirement in $quotaRequirements) { $quotaName = $requirement.Name - $quotaRequired = $requirement.PerParticipant * $participantCount + $quotaRequired = $requirement.Additional $quota = $regionalUsage | Where-Object { $_.Name.Value -ieq $quotaName } | Select-Object -First 1 $current = if($null -eq $quota) { 0 } else { $quota.CurrentValue } $limit = if($null -eq $quota) { 0 } else { $quota.Limit } $available = $limit - $current + Write-Host "$candidateLocation/${quotaName}: $current/$limit used; $($requirement.Existing)/$($requirement.Desired) lab vCPUs already allocated; $quotaRequired additional required." - if($available -ge $quotaRequired) { + if($quotaRequired -eq 0 -or $available -ge $quotaRequired) { continue } $newLimit = $current + $quotaRequired - Write-Host "Requesting exactly $newLimit $quotaName vCPUs in $candidateLocation for $participantCount participants..." + Write-Host "Requesting exactly $newLimit $quotaName vCPUs in $candidateLocation for the remaining participant compute..." $quotaResult = Request-MhhComputeQuota ` -SubscriptionId $SubscriptionId ` -Location $candidateLocation ` diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/global-localbox.tests.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/global-localbox.tests.ps1 index 9f6e50126..a43feb458 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/global-localbox.tests.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/global-localbox.tests.ps1 @@ -36,6 +36,7 @@ Submit-TestLocalBox @PSBoundParameters . "$root/labautomation/quota-helpers.ps1" function Get-AzComputeResourceSku { param($Location) } function Get-AzVMUsage { param($Location) } + function Get-MhhStableHash { param($Value, $Length) '123456781234567812345678' } } Describe 'Global LocalBox Console contract' { @@ -102,6 +103,8 @@ Describe 'Event-wide LocalBox orchestration' { } } Mock Set-MhhConfidentialComputeSelection {} + Mock Get-MhhSovereignExistingCompute { [pscustomobject]@{ Resources = @(); Allocations = @(); ConfidentialVmSizes = @() } } + Mock Get-MhhSovereignResumeCandidate { $null } Mock Request-MhhComputeQuota { throw 'Unexpected quota request' } } It 'deploys once for an event and gives users from every subscription the same discovery scope' { diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/quota-reuse.tests.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/quota-reuse.tests.ps1 new file mode 100644 index 000000000..e135d597f --- /dev/null +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/quota-reuse.tests.ps1 @@ -0,0 +1,271 @@ +BeforeAll { + $root = (Resolve-Path "$PSScriptRoot/../..").Path + . "$root/labautomation/quota-helpers.ps1" + function Invoke-AzRestMethod { [CmdletBinding()]param($Path, $Uri, $Method) } + function Get-AzTag { [CmdletBinding()]param($ResourceId) } + function Update-MhhToken {} + function Get-AzContext { [CmdletBinding()]param() } + function Get-MhhStableHash { param($Value, $Length) "$($Value[0])0000000000000000" } + function Get-AzResourceProvider { [CmdletBinding()]param($ProviderNamespace) } + function Get-AzComputeResourceSku { [CmdletBinding()]param($Location) } + function Get-AzVMUsage { [CmdletBinding()]param($Location) } + function Initialize-MhhHostedTagPolicy { param($SubscriptionId, $Location) } + function Initialize-MhhHostedDefender { param($SubscriptionId) } + function New-MhhStablePassword { param($Purpose, $Length) 'test-only-password' } + function Invoke-MhhDeploymentWithRegionFallback { + param($PreferredLocations, $ResourceGroupName, $RgOwnerEntraObjectIds, $TemplateFile, $TemplateParameterObject, $DeploymentNamePrefix, $Tag) + } + function Get-AzRoleAssignment { [CmdletBinding()]param($ObjectId, $RoleDefinitionId, $Scope) } + function New-AzSubscriptionDeployment { param($Name, $Location, $TemplateFile, $TemplateParameterObject) } + $scripts = Join-Path $TestDrive 'scripts' + New-Item -ItemType Directory $scripts | Out-Null + foreach ($name in @('deploy-lab.ps1', 'shared-deploy-lab.ps1', 'lab-defaults.json')) { + Copy-Item "$root/labautomation/$name" $scripts + } + foreach ($name in @('quota-helpers.ps1', 'hosted-tag-policy.ps1', 'hosted-defender.ps1', 'resource-providers.ps1')) { + Set-Content "$scripts/$name" '' + } + $subscription = '10000000-0000-0000-0000-000000000001' + $scope = "/subscriptions/$subscription" + $candidate = [pscustomobject]@{ Location = 'spaincentral'; VmSize = 'Standard_DC2as_v6'; QuotaName = 'standardDCasv6Family' } + function New-TestInventory { + param([int]$Participants = 1, [int]$ConfidentialNodes = 2, [int]$SystemNodes = 2, [int]$K3sVms = 1) + [pscustomobject]@{ + Resources = @([pscustomobject]@{ location = 'spaincentral' }) + ConfidentialVmSizes = @('Standard_DC2as_v6') + Allocations = @( + [pscustomobject]@{ Location = 'spaincentral'; VmSize = 'Standard_D4s_v5'; Count = ($SystemNodes + $K3sVms) * $Participants } + [pscustomobject]@{ Location = 'spaincentral'; VmSize = 'Standard_DC2as_v6'; Count = $ConfidentialNodes * $Participants } + ) + } + } +} + +Describe 'Existing participant compute discovery' { + BeforeEach { + $vmId = "$scope/resourceGroups/lab/providers/Microsoft.Compute/virtualMachines/vm-k3s-12345678" + $clusterId = "$scope/resourceGroups/lab/providers/Microsoft.ContainerService/managedClusters/aks-sovereign-12345678" + $setId = "$scope/resourceGroups/nodes/providers/Microsoft.Compute/virtualMachineScaleSets/aks-cvm" + $vm = @{ id = $vmId; name = 'vm-k3s-12345678'; type = 'Microsoft.Compute/virtualMachines'; location = 'SpainCentral' + tags = @{ workload = 'sovereign-lab' }; properties = @{ hardwareProfile = @{ vmSize = 'Standard_D4s_v5' } } } + $pool = @{ name = 'cvmnodepool'; vmSize = 'Standard_DC2as_v6'; count = 2 } + $cluster = @{ id = $clusterId; name = 'aks-sovereign-12345678'; type = 'Microsoft.ContainerService/managedClusters'; location = 'spaincentral' + tags = @{ workload = 'sovereign-lab' }; properties = @{ nodeResourceGroup = 'nodes'; agentPoolProfiles = @($pool) } } + $scaleSet = @{ id = $setId; location = 'spaincentral'; sku = @{ name = 'Standard_DC2as_v6' }; tags = @{ 'aks-managed-poolName' = 'cvmnodepool' } } + $routes = @{ + "$scope/providers/Microsoft.Compute/virtualMachines?api-version=2024-11-01" = @{ value = @($vm) } + "$scope/providers/Microsoft.ContainerService/managedClusters?api-version=2024-10-01" = @{ value = @($cluster) } + "$scope/providers/Microsoft.Network/virtualNetworks?api-version=2024-05-01" = @{ value = @() } + "$scope/providers/Microsoft.Compute/virtualMachineScaleSets?api-version=2024-11-01" = @{ value = @($scaleSet) } + "$vmId/instanceView?api-version=2024-11-01" = @{ statuses = @(@{ code = 'PowerState/running' }) } + "$setId/virtualMachines?api-version=2024-11-01&`$expand=instanceView" = @{ value = @( + @{ id = "$setId/virtualMachines/0"; properties = @{ instanceView = @{ statuses = @(@{ code = 'PowerState/running' }) } } } + @{ id = "$setId/virtualMachines/1"; properties = @{ instanceView = @{ statuses = @(@{ code = 'PowerState/stopped' }) } } } + ) } + } + Mock Invoke-AzRestMethod { + $key = if ($Uri) { $Uri } else { $Path } + if (-not $routes.ContainsKey($key)) { throw "Unexpected request: $key" } + @{ StatusCode = 200; Content = ($routes[$key] | ConvertTo-Json -Depth 20 -Compress) } + } + } + It 'credits allocated VMs and actual node instances, including stopped but allocated nodes' { + $inventory = Get-MhhSovereignExistingCompute -SubscriptionId $subscription -NameSuffixes 12345678 + ($inventory.Allocations | Where-Object VmSize -eq 'Standard_D4s_v5').Count | Should -Be 1 + ($inventory.Allocations | Where-Object VmSize -eq 'Standard_DC2as_v6').Count | Should -Be 2 + Should -Invoke Invoke-AzRestMethod -Times 0 -ParameterFilter { $Method -ne 'GET' } + } + It 'does not credit desired nodes that were not allocated or a deallocated K3s VM' { + $routes["$vmId/instanceView?api-version=2024-11-01"].statuses[0].code = 'PowerState/deallocated' + $routes["$setId/virtualMachines?api-version=2024-11-01&`$expand=instanceView"].value[1].properties.instanceView.statuses[0].code = 'PowerState/deallocated' + $inventory = Get-MhhSovereignExistingCompute -SubscriptionId $subscription -NameSuffixes 12345678 + @($inventory.Allocations | Where-Object VmSize -eq 'Standard_D4s_v5').Count | Should -Be 0 + ($inventory.Allocations | Where-Object VmSize -eq 'Standard_DC2as_v6').Count | Should -Be 1 + } + It 'caps surge instances at the two nodes this participant requires' { + $page = $routes["$setId/virtualMachines?api-version=2024-11-01&`$expand=instanceView"] + $page.value += $page.value[0] + $inventory = Get-MhhSovereignExistingCompute -SubscriptionId $subscription -NameSuffixes 12345678 + ($inventory.Allocations | Where-Object VmSize -eq 'Standard_DC2as_v6').Count | Should -Be 2 + } + It 'follows pagination when listing actual instances' { + $page = $routes["$setId/virtualMachines?api-version=2024-11-01&`$expand=instanceView"] + $routes['https://management.azure.com/next-page'] = @{ value = @($page.value[1]) } + $page.value = @($page.value[0]) + $page.nextLink = 'https://management.azure.com/next-page' + $inventory = Get-MhhSovereignExistingCompute -SubscriptionId $subscription -NameSuffixes 12345678 + ($inventory.Allocations | Where-Object VmSize -eq 'Standard_DC2as_v6').Count | Should -Be 2 + } + It 'ignores unrelated resources and other participant resource groups' { + $inventory = Get-MhhSovereignExistingCompute -SubscriptionId $subscription -NameSuffixes 87654321 + $inventory.Resources.Count | Should -Be 0 + $inventory.Allocations.Count | Should -Be 0 + $inventory = Get-MhhSovereignExistingCompute -SubscriptionId $subscription -NameSuffixes 12345678 -ResourceGroupName other + $inventory.Resources.Count | Should -Be 0 + } + It 'requires participant identities when existing labs are present' { + { Get-MhhSovereignExistingCompute -SubscriptionId $subscription } | Should -Throw '*Participant IDs are required*' + } + It 'rejects incompatible K3s sizes' { + $vm.properties.hardwareProfile.vmSize = 'Standard_D8s_v5' + { Get-MhhSovereignExistingCompute -SubscriptionId $subscription -NameSuffixes 12345678 } | Should -Throw '*incompatible size*' + } + It 'rejects conflicting ownership rather than borrowing unrelated quota' { + $vm.tags.workload = 'other' + { Get-MhhSovereignExistingCompute -SubscriptionId $subscription -NameSuffixes 12345678 } | Should -Throw '*ownership tag*' + } + It 'propagates inventory API failures' { + Mock Invoke-AzRestMethod { throw 'Forbidden' } + { Get-MhhSovereignExistingCompute -SubscriptionId $subscription -NameSuffixes 12345678 } | Should -Throw '*Forbidden*' + } + It 'warns and withholds quota credit for missing power-state evidence' { + Mock Write-Warning {} + $routes["$vmId/instanceView?api-version=2024-11-01"].statuses = @() + $inventory = Get-MhhSovereignExistingCompute -SubscriptionId $subscription -NameSuffixes 12345678 + @($inventory.Allocations | Where-Object VmSize -eq 'Standard_D4s_v5').Count | Should -Be 0 + Should -Invoke Write-Warning -Times 1 -Exactly + } + It 'rejects duplicate participant resources across groups' { + $copy = $vm.Clone() + $copy.id = $vmId.Replace('/lab/', '/other/') + $routes["$scope/providers/Microsoft.Compute/virtualMachines?api-version=2024-11-01"].value += $copy + { Get-MhhSovereignExistingCompute -SubscriptionId $subscription -NameSuffixes 12345678 } | Should -Throw '*Ambiguous participant compute*' + } + It 'rejects non-success and malformed inventory responses' { + Mock Invoke-AzRestMethod { @{ StatusCode = 403; Content = '{}' } } + { Get-MhhSovereignArmData -Path '/test' } | Should -Throw '*HTTP 403*' + Mock Invoke-AzRestMethod { @{ StatusCode = 200; Content = '{"value":null}' } } + { Get-MhhSovereignArmData -Path '/test' } | Should -Throw '*Invalid compute inventory*' + } +} + +Describe 'Reuse-aware compute demand and selection' { + BeforeEach { + Mock Get-MhhConfidentialComputeSelection { $candidate } + } + It 'needs no extra quota for four fully allocated labs' { + $demand = @(Get-MhhSovereignAdditionalCompute -Candidate $candidate -ExistingCompute (New-TestInventory -Participants 4) -ParticipantCount 4) + $demand.Additional | Should -Be @(0, 0, 0) + $demand.Existing | Should -Be @(16, 48, 64) + } + It 'requires the complete quota for fresh labs' { + $demand = @(Get-MhhSovereignAdditionalCompute -Candidate $candidate -ExistingCompute @{ Allocations = @() } -ParticipantCount 4) + $demand.Additional | Should -Be @(16, 48, 64) + } + It 'reserves only missing resources for a partial participant' { + $demand = @(Get-MhhSovereignAdditionalCompute -Candidate $candidate -ExistingCompute (New-TestInventory -ConfidentialNodes 1 -K3sVms 0)) + $demand.Additional | Should -Be @(2, 4, 6) + } + It 'does not borrow allocations from another location or confidential family' { + $inventory = New-TestInventory + $inventory.Allocations[0].Location = 'westus2' + $inventory.Allocations[1].VmSize = 'Standard_DC2as_v5' + $demand = @(Get-MhhSovereignAdditionalCompute -Candidate $candidate -ExistingCompute $inventory) + $demand.Additional | Should -Be @(4, 12, 16) + } + It 'preserves a valid saved selection' { + (Get-MhhSovereignResumeCandidate -SubscriptionId $subscription -ExistingCompute (New-TestInventory)).Location | Should -Be spaincentral + } + It 'rejects mismatched regions and confidential sizes' { + $inventory = New-TestInventory + $inventory.Resources[0].location = 'westus2' + { Get-MhhSovereignResumeCandidate -SubscriptionId $subscription -ExistingCompute $inventory } | Should -Throw '*conflict*' + $inventory.Resources[0].location = 'spaincentral' + $inventory.ConfidentialVmSizes = @('Standard_DC2as_v5') + { Get-MhhSovereignResumeCandidate -SubscriptionId $subscription -ExistingCompute $inventory } | Should -Throw '*conflict*' + } + It 'refuses existing resources without valid selection tags but allows fresh discovery' { + Mock Get-MhhConfidentialComputeSelection { $null } + { Get-MhhSovereignResumeCandidate -SubscriptionId $subscription -ExistingCompute (New-TestInventory) } | Should -Throw '*Restore*selection tags*' + Get-MhhSovereignResumeCandidate -SubscriptionId $subscription -ExistingCompute @{ Resources = @() } | Should -BeNullOrEmpty + Mock Get-MhhConfidentialComputeSelection { @{ Location = 'spaincentral'; VmSize = 'Standard_DC2as_v6'; QuotaName = 'wrong' } } + { Get-MhhSovereignResumeCandidate -SubscriptionId $subscription -ExistingCompute (New-TestInventory) } | Should -Throw '*invalid*' + } +} + +Describe 'Shared and participant quota retry orchestration' { + BeforeEach { + $inventory = New-TestInventory -Participants 4 + $usage = @( + @{ Name = @{ Value = 'standardDCasv6Family' }; CurrentValue = 16; Limit = 16 } + @{ Name = @{ Value = 'StandardDSv5Family' }; CurrentValue = 48; Limit = 100 } + @{ Name = @{ Value = 'cores' }; CurrentValue = 96; Limit = 120 } + ) + Mock Get-AzContext { @{ Subscription = @{ Id = $subscription } } } + Mock Get-MhhSovereignExistingCompute { $inventory } + Mock Get-MhhConfidentialComputeSelection { $candidate } + Mock Get-AzResourceProvider { @{ RegistrationState = 'Registered' } } + Mock Get-AzComputeResourceSku { + foreach ($size in @('Standard_D4s_v5', 'Standard_DC2as_v5', 'Standard_DC2as_v6')) { + @{ ResourceType = 'virtualMachines'; Name = $size; Restrictions = @() } + } + } + Mock Get-AzVMUsage { $usage } + Mock Request-MhhComputeQuota { @{ Succeeded = $true } } + Mock Set-MhhConfidentialComputeSelection {} + Mock Initialize-MhhHostedTagPolicy {} + Mock Initialize-MhhHostedDefender {} + Mock Invoke-MhhDeploymentWithRegionFallback { @{ LocationUsed = 'spaincentral'; Outputs = @{} } } + Mock Get-AzRoleAssignment { @() } + Mock New-AzSubscriptionDeployment {} + Mock Start-Sleep {} + $users = @('12345678', '23456789', '34567890', '45678901') + } + It 'does not request doubled quota on a complete rerun or switch to a newly preferred region' { + & "$scripts/shared-deploy-lab.ps1" -SubscriptionId $subscription -PreferredLocation westus2,spaincentral -AllowedEntraUserIds $users + Should -Invoke Request-MhhComputeQuota -Times 0 + Should -Invoke Set-MhhConfidentialComputeSelection -Times 1 -Exactly -ParameterFilter { + $Candidate.Location -eq 'spaincentral' -and $Candidate.VmSize -eq 'Standard_DC2as_v6' + } + } + It 'waits for inventory providers on fresh subscriptions before querying their resources' { + Mock Get-AzResourceProvider { @{ RegistrationState = 'Registering' } } -ParameterFilter { $ProviderNamespace -eq 'Microsoft.ContainerService' } + { & "$scripts/shared-deploy-lab.ps1" -SubscriptionId $subscription -PreferredLocation spaincentral -AllowedEntraUserIds $users } | Should -Throw '*Microsoft.ContainerService did not reach Registered*' + Should -Invoke Get-MhhSovereignExistingCompute -Times 0 + Should -Invoke Request-MhhComputeQuota -Times 0 + } + It 'requests only missing capacity while retaining unrelated LocalBox usage' { + $inventory = New-TestInventory -Participants 3 + $usage[0].CurrentValue = 12 + $usage[0].Limit = 12 + $usage[1].CurrentValue = 36 + $usage[2].CurrentValue = 80 + $usage[2].Limit = 80 + & "$scripts/shared-deploy-lab.ps1" -SubscriptionId $subscription -PreferredLocation spaincentral -AllowedEntraUserIds $users + Should -Invoke Request-MhhComputeQuota -Times 2 -Exactly + Should -Invoke Request-MhhComputeQuota -Times 1 -Exactly -ParameterFilter { $QuotaName -eq 'standardDCasv6Family' -and $NewLimit -eq 16 } + Should -Invoke Request-MhhComputeQuota -Times 1 -Exactly -ParameterFilter { $QuotaName -eq 'cores' -and $NewLimit -eq 96 } + } + It 'does not change region when a remaining quota request fails' { + $inventory = New-TestInventory -Participants 3 + Mock Request-MhhComputeQuota { @{ Succeeded = $false; ErrorCode = 'QuotaNotAvailableForResource' } } + { & "$scripts/shared-deploy-lab.ps1" -SubscriptionId $subscription -PreferredLocation westus2,spaincentral -AllowedEntraUserIds $users } | Should -Throw '*No preferred region*' + Should -Invoke Request-MhhComputeQuota -Times 1 -Exactly -ParameterFilter { $Location -eq 'spaincentral' } + Should -Invoke Set-MhhConfidentialComputeSelection -Times 0 + } + It 'still reserves the full requirement on a fresh deployment' { + $inventory = [pscustomobject]@{ Resources = @(); Allocations = @(); ConfidentialVmSizes = @() } + Mock Get-MhhConfidentialComputeSelection { $null } + & "$scripts/shared-deploy-lab.ps1" -SubscriptionId $subscription -PreferredLocation spaincentral -AllowedEntraUserIds $users + Should -Invoke Request-MhhComputeQuota -Times 1 -Exactly -ParameterFilter { $QuotaName -eq 'standardDCASv5Family' -and $NewLimit -eq 16 } + } + It 'lets an allocated participant reach deployment RBAC and credential output at full quota' { + $inventory = New-TestInventory + Mock Get-AzComputeResourceSku { @() } + $result = @(& "$scripts/deploy-lab.ps1" -DeploymentType resourcegroup -SubscriptionId $subscription -ResourceGroupName lab -PreferredLocation westus2 -AllowedEntraUserIds @($users[0])) + Should -Invoke Invoke-MhhDeploymentWithRegionFallback -Times 1 -Exactly -ParameterFilter { $PreferredLocations -eq 'spaincentral' } + Should -Invoke New-AzSubscriptionDeployment -Times 1 -Exactly + @($result.HackboxCredential | Where-Object name -eq 'K3s VM Admin Password').Count | Should -Be 1 + } + It 'still blocks missing confidential nodes without additional quota' { + $inventory = New-TestInventory -ConfidentialNodes 1 + { & "$scripts/deploy-lab.ps1" -DeploymentType resourcegroup -SubscriptionId $subscription -ResourceGroupName lab -AllowedEntraUserIds @($users[0]) } | Should -Throw '*2 additional standardDCasv6Family*' + Should -Invoke Invoke-MhhDeploymentWithRegionFallback -Times 0 + } + It 'blocks missing compute when its SKU is unavailable instead of submitting anyway' { + $inventory = New-TestInventory -K3sVms 0 + Mock Get-AzComputeResourceSku { @() } + { & "$scripts/deploy-lab.ps1" -DeploymentType resourcegroup -SubscriptionId $subscription -ResourceGroupName lab -AllowedEntraUserIds @($users[0]) } | Should -Throw '*unavailable VM sizes*' + Should -Invoke Invoke-MhhDeploymentWithRegionFallback -Times 0 + } +} diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md index 73e3b93ec..13b8ea987 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md @@ -18,6 +18,26 @@ A fresh Codespace smoke test is still required to validate the devcontainer buil ## Participant K3s installation checks +### Quota-safe deployment retries + +Run `Invoke-Pester ./quota-reuse.tests.ps1,./shared-aks.tests.ps1,./global-localbox.tests.ps1 -Output Detailed`. +Shared and participant quota checks reserve only the missing compute for the current +participants. Discovery uses their stable resource names and ownership tags, reads +actual K3s VM and AKS scale-set instance power states, and caps credit at the lab's +one K3s VM and two nodes per pool. Deallocated, missing, unrelated and unconfirmed +instances receive no credit; desired AKS node counts alone are not allocation evidence. +Unrelated usage, including LocalBox, remains part of the subscription's used quota. +Inventory failures stop the deployment rather than masquerading as an empty inventory. + +Retries preserve the saved subscription region and confidential SKU, even if preferred +regions are reordered. Conflicting resources, invalid selection tags, or existing labs +without participant identities stop with an error instead of choosing another region. +Fresh subscriptions still evaluate the configured candidates and request their full +aggregate compute requirement. These checks do not repair failed K3s extensions, +skip Console stages, or change the deployment helper's resource-group fallback behavior. + +### Bounded K3s downloads + Run `Invoke-Pester ./k3s-install.tests.ps1 -Output Detailed` on a host with Bash. These offline tests execute the actual Bicep-embedded installation script with mocked download, installer, service and Kubernetes commands. They cover successful diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/shared-aks.tests.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/shared-aks.tests.ps1 index d5015abd9..117b34c2b 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/shared-aks.tests.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/shared-aks.tests.ps1 @@ -142,7 +142,7 @@ Describe 'Shared AKS infrastructure contract' { ($requirements | Where-Object Name -eq 'StandardDSv5Family').PerParticipant | Should -Be 12 ($requirements | Where-Object Name -eq 'cores').PerParticipant | Should -Be 16 foreach ($scriptName in @('deploy-lab.ps1', 'shared-deploy-lab.ps1')) { - Get-Content "$root/labautomation/$scriptName" -Raw | Should -Match 'Get-MhhSovereignComputeRequirements' + Get-Content "$root/labautomation/$scriptName" -Raw | Should -Match 'Get-MhhSovereignAdditionalCompute' } $defaults = Get-Content "$root/labautomation/lab-defaults.json" -Raw | ConvertFrom-Json $defaults.estimatedDailyCostsUsd | Should -Be 45 From 3d79d79755939a5f5c92382f0314a1551a3e157a Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Mon, 9 Nov 2026 12:13:33 +0000 Subject: [PATCH 35/37] Document targeted K3s extension recovery for event leads Add scoped forceUpdateTag reruns, bounded waits, live readiness checks, and Console retry guidance without recreating lab infrastructure. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../resources/hosted-events/readme.md | 151 ++++++++++++++++++ 1 file changed, 151 insertions(+) diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md index 8b011e8ea..f20adf96d 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md @@ -14,6 +14,8 @@ This runbook covers the FY27 EMEA Cloud & AI Hackathons program described in the When automatic provisioning is not scheduled, use the Console's **Create Users**, content-pack license defaults and **Deploy Lab Environment** lifecycle in order. Investigate failures before continuing; do not compensate by running subscription-wide manual utilities. +For participant K3s installation failures, follow [event-lead troubleshooting](#retry-a-failed-install-k3s-extension) before retrying the Console deployment. + ## One LocalBox per event Re-import the content pack after updating it. The Console discovers @@ -777,6 +779,155 @@ The global hook calls the Console-supported `Get-MhhDefaultLabGroup` helper and The [platform contract](../../../../../99-MicroHack-Template/labautomation/README.md#groups-supported-values) still uses `groups` for licensing activation (`GHCPUsers`, `M365-E5-Users`), not arbitrary group creation. No new schema fields or Graph permissions are needed. The Console owns the event group, its membership and teardown; the facilitator must verify intended users/coaches and late registrations. The preparation script still prompts for the ID on the Client, where Console helpers are unavailable, and does not create groups or validate membership through Graph. +## Event-lead troubleshooting + +### Retry a failed install-k3s extension + +**Symptom:** a participant deployment fails with `VMExtensionProvisioningError` +on `install-k3s`, often with `[ERROR] Download failed`. AKS and the K3s VM can +already exist even though the participant job failed. + +An unchanged Custom Script Extension does not necessarily execute again when +Console resubmits the ARM template. Azure can report the **previous failure**: +a new ARM deployment timestamp is not evidence of a new download attempt. +In the observed recovery, all three failed extensions retained their original +execution records until explicitly force-rerun; all three then installed +successfully on the first attempt. + +A download failure alone does not distinguish Azure networking from GitHub/CDN +availability. Inspect the error first. The procedure below reruns the existing +extension; it does not fix a persistent network, configuration or policy problem. + +#### 1. Identify and inspect one failed participant + +Wait for the current Console deployment to finish. Do not race an active +deployment or rerun healthy participants. Use an authenticated **Azure CLI in +Bash**, with permission to read/update VM extensions and invoke VM Run Command +on the affected participant VM, such as Virtual Machine Contributor scoped to +that participant resource group. No VM administrator password is needed. + +Copy the subscription, resource group and VM name from the failed deployment +and confirm they belong to the intended participant. Run the following blocks +in the same Bash session, once per failed VM; do not target LocalBox-Client. + +```bash +set -euo pipefail + +SUBSCRIPTION_ID='' +RESOURCE_GROUP='rg-labuser-' +VM_NAME='vm-k3s-' +EXTENSION_NAME='install-k3s' + +extension_id="/subscriptions/$SUBSCRIPTION_ID/resourceGroups/$RESOURCE_GROUP/providers/Microsoft.Compute/virtualMachines/$VM_NAME/extensions/$EXTENSION_NAME" +extension_uri="https://management.azure.com$extension_id?api-version=2024-11-01" + +az vm extension show \ + --subscription "$SUBSCRIPTION_ID" --resource-group "$RESOURCE_GROUP" \ + --vm-name "$VM_NAME" --name "$EXTENSION_NAME" --instance-view \ + --query '{state:provisioningState,status:instanceView.statuses}' \ + --output json --only-show-errors +``` + +Confirm that the error is the K3s installation failure you intend to retry. +For recurring failures, preserve the extension output and guest log timestamps; +the original installer may not include the underlying HTTP status. + +#### 2. Force one execution without replacing the configuration + +This PATCH changes **only `forceUpdateTag`**. It preserves the extension's +public/protected settings and does not recreate the VM, AKS or LocalBox. The +existing installation script can restart K3s if it is already installed, so +limit this action to the failed extension being repaired. + +```bash +identity="$(az rest --subscription "$SUBSCRIPTION_ID" --method get \ + --url "$extension_uri" \ + --query "join('|', [properties.provisioningState, properties.publisher, properties.type])" \ + --output tsv --only-show-errors)" + +if [[ "$identity" != 'Failed|Microsoft.Azure.Extensions|CustomScript' ]]; then + printf 'STOP: expected a failed Microsoft CustomScript extension, got: %s\n' "$identity" >&2 + exit 1 +fi + +force_tag="event-lead-k3s-retry-$(date -u +%Y%m%dT%H%M%SZ)-$RANDOM" +az rest --subscription "$SUBSCRIPTION_ID" --method patch \ + --url "$extension_uri" \ + --body "{\"properties\":{\"forceUpdateTag\":\"$force_tag\"}}" \ + --query '{state:properties.provisioningState,forceUpdateTag:properties.forceUpdateTag}' \ + --output json --only-show-errors + +if ! az vm extension wait \ + --subscription "$SUBSCRIPTION_ID" --resource-group "$RESOURCE_GROUP" \ + --vm-name "$VM_NAME" --name "$EXTENSION_NAME" \ + --updated --interval 15 --timeout 2400 --only-show-errors; then + az vm extension show \ + --subscription "$SUBSCRIPTION_ID" --resource-group "$RESOURCE_GROUP" \ + --vm-name "$VM_NAME" --name "$EXTENSION_NAME" --instance-view \ + --query '{state:provisioningState,status:instanceView.statuses}' \ + --output json --only-show-errors + printf 'STOP: rerun failed or timed out. Inspect its status before taking further action.\n' >&2 + exit 1 +fi + +az vm extension show \ + --subscription "$SUBSCRIPTION_ID" --resource-group "$RESOURCE_GROUP" \ + --vm-name "$VM_NAME" --name "$EXTENSION_NAME" --instance-view \ + --query '{state:provisioningState,forceUpdateTag:forceUpdateTag,status:instanceView.statuses}' \ + --output json --only-show-errors +``` + +The initial `Updating` response means only that Azure accepted the request. +Require the final extension state **Succeeded**. The wait is bounded to +40 minutes; a timeout does not cancel the extension. Do not submit another +force-rerun while it is still running. + +#### 3. Verify K3s readiness + +Run this read-only check on the same VM: + +```bash +az vm run-command invoke \ + --subscription "$SUBSCRIPTION_ID" --resource-group "$RESOURCE_GROUP" \ + --name "$VM_NAME" --command-id RunShellScript \ + --scripts ' +set -eu +systemctl is-active k3s +systemctl is-enabled k3s +timeout 45s k3s kubectl wait --for=condition=Ready nodes --all --timeout=30s --request-timeout=10s +timeout 20s k3s kubectl get nodes --request-timeout=15s +echo K3S_RECOVERY_VERIFIED +' \ + --query 'value[].message' --output json --only-show-errors +``` + +Require **active**, **enabled**, the expected VM's node **Ready**, and the final +**K3S_RECOVERY_VERIFIED** marker without script errors. Run Command's outer +`Enable succeeded` message alone is not proof that its script succeeded. +If verification fails, investigate that VM rather than restarting the whole event. + +#### 4. Retry Console only after the affected VMs are healthy + +Re-import the content revision containing the reuse-aware quota checks before +retrying. Keep the **same event, subscription assignments and preferred-region +order**. In particular, changing the first preferred region can conflict with +an existing global policy assignment's managed-identity location. + +Then retry the Console deployment so the failed participants can finish +**RBAC assignment and credential publication**. Neither extension recovery nor +Run Command publishes the missing K3s credentials or changes historical failed +ARM deployment records to successful. + +A full Console retry is not a failed-participant-only resume: it runs global +LocalBox checks, shared preparation and participant jobs again. Existing +LocalBox is reused, and participant resources normally receive incremental +updates. The region-fallback helper can still recycle resource groups for +other retryable errors; do not delete resources or broaden recovery merely to +clear a K3s extension failure. + +References: [Custom Script Extension for Linux](https://learn.microsoft.com/azure/virtual-machines/extensions/custom-script-linux) +and [VM extension update / forceUpdateTag](https://learn.microsoft.com/rest/api/compute/virtual-machine-extensions/update). + ## Run and close the event - Export Console user credentials and distribute them securely to the correct attendees/coaches. Console login credentials and Azure Temporary Access Passes (TAPs) are distinct. From 7f47eeed8e3f7ac2db9dd5f373bc36c54fdccede Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Mon, 9 Nov 2026 18:36:30 +0000 Subject: [PATCH 36/37] docs: align Sovereign Cloud guides with automated LocalBox readiness Clarify hosted and manual responsibilities, regional policy guidance, recovery evidence and remaining event checks. Include TEST14 Console completion and source-pin promotion requirements. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../01_Sovereign_Cloud/Readme.md | 8 +- .../resources/demo-vm-creator/README.md | 4 +- .../resources/hosted-events/readme.md | 110 +++++++++++++++++- .../hosted-events/test-14-timing-analysis.md | 35 +++++- .../resources/localbox/readme.md | 19 +-- .../01_Sovereign_Cloud/resources/readme.md | 12 +- .../resources/tests/readme.md | 72 +++++++++--- .../walkthrough/challenge-06/solution-06.md | 2 +- 8 files changed, 219 insertions(+), 43 deletions(-) diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/Readme.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/Readme.md index 8cc79fbe1..6481be90e 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/Readme.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/Readme.md @@ -107,22 +107,24 @@ This Hackathon has a few but important prerequisites In order to use the Hackathon time most effectively, the following tasks should be completed prior to starting the session. > [!NOTE] -> For eligible Microsoft-hosted events, use the [Hacks Console runbook](https://github.com/microsoft/MicroHack/blob/main/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md). Outside those events, use [manual infrastructure setup](https://github.com/microsoft/MicroHack/blob/main/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/manual-setup/readme.md). Both paths require LocalBox preparation and health validation before participants begin. +> For eligible Microsoft-hosted events, use the [Hacks Console runbook](resources/hosted-events/readme.md). Console automatically deploys/reuses LocalBox, prepares its shared resources and verifies infrastructure readiness before shared and participant deployments. No routine Client login or manual preparation is required. Event leads still verify participant access and Defender cloud reporting. Outside those events, use [manual infrastructure setup](resources/manual-setup/readme.md), then run preparation and health checks explicitly. 1. For hosted events, the hacker account and assigned Azure subscription/resource group from the Hackathon Console; otherwise, your own Azure subscription with Owner RBAC rights at the subscription level 2. The resource, policy and role-assignment permissions specified by each challenge in your assigned scope -3. Access to Azure Arc Jumpstart LocalBox if completing Challenge 6, including the [guest-management and Defender readiness checks](https://github.com/microsoft/MicroHack/blob/main/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/demo-vm-creator/README.md#step-6-test-the-environment) +3. Access to the prepared Azure Arc Jumpstart LocalBox environment if completing Challenge 6; the organizer must complete the [participant-access, guest-management and Defender readiness checks](resources/localbox/manual-preparation.md#step-6-test-the-environment). Participants still create their own Windows VM and challenge workloads. 4. The [recommended Codespaces environment](#recommended-environment-github-codespaces), or a local workstation with [Azure CLI](https://learn.microsoft.com/cli/azure/install-azure-cli) and the tools listed in each walkthrough 5. Challenge 7: `kubectl`, `jq`, OpenSSL, the [Radius CLI](https://docs.radapp.io/getting-started/install/), and the Azure CLI `bastion` extension (included in the Sovereign Cloud devcontainer) The shared Challenge 5/7 AKS platform requires **4 AMD SEV-SNP confidential-family vCPUs per participant** for two Ubuntu nodes, plus **12 DSv5 vCPUs** for two system nodes and one K3s VM: **16 regional vCPUs** in total. There is no standalone Confidential VM. Automation evaluates `Standard_DC2as_v5` and `Standard_DC2as_v6` across the configured regions. With two labs per subscription, allow 8 confidential-family, 24 DSv5 and 32 regional vCPUs before deployment. These are steady-state requirements; also reserve upgrade surge headroom (for example one additional node per pool: 2 confidential plus 4 DSv5 vCPUs per concurrent cluster upgrade). -The default candidate order is Sweden Central/v5, Spain Central/v5, Sweden Central/v6, then Spain Central/v6. Shared preparation persists the first candidate with supported VM sizes and sufficient confidential-family, DSv5-family, and regional quota so every participant deployment in that subscription uses the same selection. +The default candidate order is Spain Central/v5, Sweden Central/v5, Spain Central/v6, then Sweden Central/v6. Shared preparation persists the first candidate with supported VM sizes and sufficient confidential-family, DSv5-family, and regional quota so every participant deployment in that subscription uses the same selection. Retries preserve the saved selection and credit confirmed existing allocations rather than requiring the full lab capacity again as unused quota. On 2026-09-21, read-only `az vm list-skus --all` checks in the validation subscription returned `Standard_DC2as_v6` in **both Sweden Central and Spain Central**, with no restrictions; `Standard_DC2as_v5` was not listed in either. This is a dated subscription-specific observation, not a capacity guarantee or proof of AKS deployment success. Recheck SKUs, quota and AKS support for each event. Ubuntu CVM pools are supported as described in [AKS confidential VM guidance](https://learn.microsoft.com/azure/aks/use-cvm). Challenge 4 independently deploys ACI/ACR in **North Europe**, the walkthrough's validated Confidential ACI region. It does not use VM-family quota, the old standalone CVM, or a custom attestation provider. Confirm Confidential ACI quota/capacity separately; VM availability in Sweden/Spain does not establish Confidential ACI availability there. Shared automation registers `Microsoft.ContainerInstance` and `Microsoft.ContainerRegistry`. Challenge 5 uses the actual Console AKS region, not Challenge 4's `LOCATION` value. +Event-specific capacity fallback can place participant AKS/K3s resources outside the default regions, including **West US 2**. This is a synthetic-lab workaround, **not an EU data-residency recommendation**. Neither Spain Central nor West US 2 is in Challenge 1's example location allowlist. Keep exercise assignments in **DoNotEnforce** except for the explicitly scoped enforcement test, then restore that setting; inherited policies still apply. The Azure Local/custom-location registration defaults to **Australia East**, independently of the LocalBox host and participant AKS regions. See the [regional policy and retry guidance](resources/hosted-events/readme.md#participant-regions-and-exercise-policies) before changing an event's locations. + ```powershell ./resources/manual-setup/subscription-preparations/2-vcpu-quotas.ps1 -Region swedencentral -NumberOfLabUsers 2 -ConfidentialVmGeneration v5 -SubmitQuotaRequests ``` diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/demo-vm-creator/README.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/demo-vm-creator/README.md index b6c002da5..631768f67 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/demo-vm-creator/README.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/demo-vm-creator/README.md @@ -1,9 +1,9 @@ # LocalBox guide has moved -Choose [hosted-event setup](../hosted-events/readme.md) or [manual setup](../manual-setup/readme.md), then complete the common [LocalBox preparation](../localbox/readme.md). +Choose [hosted-event setup](../hosted-events/readme.md) or [manual setup](../manual-setup/readme.md). Hosted Console events automatically deploy, prepare and infrastructure-test LocalBox. Manual organizers run the common [LocalBox preparation](../localbox/readme.md) themselves; hosted leads use it only for deliberate recovery, never concurrently with the hosted job. The manual deployment script is now [manual-setup/localbox/deploy-localbox.ps1](../manual-setup/localbox/deploy-localbox.ps1). The Console continues to use the existing labautomation hook. ## Step 6: Test the Environment -Run the [Pester health checks](../tests/readme.md), then complete the [participant VM guest-management, Defender and Update Manager readiness exercise](../localbox/manual-preparation.md#step-6-test-the-environment). This page preserves existing challenge links; challenge and solution content has not changed. \ No newline at end of file +For hosted delivery, inspect the automated LocalBox health results; for manual delivery, run the [Pester health checks](../tests/readme.md). In either case, complete the [participant VM guest-management, Defender and Update Manager readiness exercise](../localbox/manual-preparation.md#step-6-test-the-environment), including actual participant access and Defender cloud reporting. This page preserves existing challenge links. \ No newline at end of file diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md index f20adf96d..cec991af0 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md @@ -2,6 +2,46 @@ This runbook covers the FY27 EMEA Cloud & AI Hackathons program described in the Event Lead Guide. Confirm eligibility and access with your program lead; the Console is not a general entitlement for all Microsoft events or 1:1 customer engagements. Other delivery uses [manual setup](../manual-setup/readme.md). +## Current verified state + +Hosted provisioning automates LocalBox deployment, nested readiness, preparation +and infrastructure health before shared and participant stages. Event leads do +not routinely log into LocalBox-Client or invoke preparation. This does **not** +automate participant exercises or prove attendee access and cloud security reporting. + +On **9 November 2026**, the final Console retry for a 25-lab, seven-subscription +event completed successfully: + +| Evidence | Verified outcome | +| --- | --- | +| Console job | Succeeded; 12:10:31-13:16:11 UTC, approximately **1h05m40s** | +| Shared LocalBox | Reused existing infrastructure; a new guest run passed **12/12 upstream and 14/14 Sovereign tests**, with no failed, skipped or unrun tests | +| Subscription shared stages | **7/7 succeeded** | +| Participant deployments | **25/25 succeeded**, including 20 labs in Spain Central and five in West US 2 | +| K3s recovery and credentials | Three failed extensions were explicitly force-rerun and independently verified healthy; the subsequent Console run completed, and the organizer confirmed their credentials were published | + +This is **reuse/recovery evidence**, not an uninterrupted first-attempt cold +deployment or a new cold-provisioning time estimate. Earlier runs ended at 22/25. +An unchanged ARM redeployment had reused the three failed extension results; +the [targeted recovery procedure](#retry-a-failed-install-k3s-extension) caused new +executions, each successful on installer attempt 1/3. The original download logs +do not establish whether Azure networking or remote binary availability caused +the failures. + +Actual participant sign-in, Bastion/proxy/application access and Defender cloud +reporting remain unverified for this event; use the [remaining validation checklist](#remaining-event-validation). +The successful cloud participant jobs also do **not** validate 25 simultaneous +nested Challenge 6 Windows VMs; retain the [LocalBox capacity limits and rehearsal guidance](../localbox/readme.md#tested-participant-vm-capacity). + +The BGInfo fix was present and the subsequent-logon Pester task was registered +on the event Client, but the historical startup watcher still reported +`StartupExited` without a successful terminal report. This diagnostic discrepancy +remains unresolved; do not equate it with a failed current infrastructure health +run or claim that all startup-monitor behavior has been validated. + +Older TEST06/12/13/14 observations below are dated diagnostic history, not the +current event status. Preserve their original failures and measurement boundaries. + ## Configure the event 1. Agree the Sovereign Cloud scenario with your GEO/program lead and request an Event Lead account through the program's access process. @@ -319,8 +359,10 @@ For a full cold-run breakdown, see the The [TEST14 section comparison and final warning review](test-14-timing-analysis.md) records the 8 November Full pass: 3h46m10s of guest execution, approximately one hour of actual AKS/image overlap, and a failed validation attempt that offset -the guest-stage savings. It distinguishes LocalBox completion from the -unverified downstream Console outcome. +the guest-stage savings. Subsequent Console output confirms global, shared and +participant deployment completion, with approximately 11h00m30s of reported +stage execution. This excludes initial setup and inter-stage gaps; participant +sign-in and Defender cloud reporting remain separate verification steps. The 4 November 2026 run took **10 hours 30 minutes 44 seconds** from Console start to finish, including **5 hours 24 minutes 9 seconds** of managed guest execution. All 26 guest tests passed, but Console result retrieval failed and @@ -643,6 +685,8 @@ not overwritten. After a completed/failed command, retrying starts a new tracked run and reuses preparation's resource checks; it does not delete the lab or reset existing VM passwords. Investigate failures before retrying. +### Source pins and upstream promotion + When changing any packaged guest script or health test, publish those sources first, then update the source manifest's repository, full commit SHA and file digests together. The offline tests require the pin's digests to match the @@ -650,6 +694,19 @@ checkout. Re-import the final content revision, not the intermediate source commit. The feature branch currently pins the tested guest sources in the fork; promotion upstream must retain an accessible immutable source revision. +For the PR, retain the working pin unless maintainers choose to publish the +guest sources upstream first. Confirm the current fork revision will remain +accessible, or publish the same reviewed guest files to an upstream commit and +then update the repository, full SHA and all digests together in a follow-up +content revision. Do not substitute `main`, an unpublished commit or the future +merge SHA. Verify all five packaged files against the manifest and exercise the +`labautomation`-only packaging path before re-importing a changed pin. + +Include the TEST14 Console closeout in [its timing report](test-14-timing-analysis.md) +and this runbook when promoting the documentation. Do not replace historical +failure reports or publish private run artifacts, TAPs, passwords or unredacted +credential screenshots as PR evidence. + ### Remaining event validation The automated gate verifies **infrastructure**, MDE extension installation and @@ -659,10 +716,20 @@ an installed extension, a resource-count check or admin kubeconfig access. - After participant deployments, run [health checks](../tests/readme.md) for each selected lab; they cannot run inside the earlier global stage. +- Before pre-event smoke tests, check the test account's TAP validity window and + event time zone. A TAP issued for the workshop date may not work the day before. + Arrange an authorized test credential/window; after an approved schedule change, + save it and select **Update TAPs**. Do not include credential values in evidence. +- Smoke-test at least one Spain Central and one West US 2 participant when both + regions are used, including sign-in, the Challenge 7 Bastion/K3s path and actual + AKS access. Include a participant in a different subscription from LocalBox. - Complete the [Defender readiness checklist](../localbox/manual-preparation.md#defender-for-servers-readiness-organizer), including actual device reporting and recommendation/assessment visibility. - Verify intended lab users/coaches belong to the Console group and test a real participant's Entra-authenticated proxy and private application port-forward. +- Complete the [participant VM exercise](../localbox/manual-preparation.md#step-6-test-the-environment) + in the assigned participant scope. Rehearse planned nested-VM concurrency and + capacity; successful AKS/K3s provisioning is not evidence of LocalBox workload capacity. - Preserve the [manual preparation guide](../localbox/readme.md) for BYOS and deliberate recovery only. Do not run it concurrently with hosted automation. @@ -674,6 +741,45 @@ fails setup explicitly. Organizer preparation reads that nonsecret tag; it does not require Graph permissions on the Client managed identity. The global hook invokes AKS and MetalLB preparation inside the Client after nested readiness. +### Participant regions and exercise policies + +The default participant candidate order is Spain Central/v5, Sweden Central/v5, +Spain Central/v6, then Sweden Central/v6. Event-specific SKU/quota restrictions +can require other configured candidates. West US 2 was used for five participants +in the 9 November validation; it is a **synthetic-lab capacity workaround, not +EU data residency**. Do not add it to sovereignty policy examples as an approved +European region. + +- Challenge 1's example allowlist includes four European regions and the + Australia East lab exception, **not Spain Central or West US 2**. + `DoNotEnforce` still reports noncompliance but does not deny requests through + that assignment. During Task 9's scoped enforcement test, deployments or + updates in disallowed regions can be denied; restore the three exercise + assignments to `DoNotEnforce` afterward and keep the bonus initiative in that + mode. Existing resources are not automatically stopped or deleted. +- For Challenges 2/3, keep the assigned resource group and deploy exercise + resources in the approved region specified by the walkthrough/organizer. + A resource group's metadata location does not require every resource to use + that location. Keep the Key Vault and storage account colocated. +- Challenge 4 retains **`LOCATION=northeurope`** for the validated confidential + ACI comparison. Challenges 5/7 use the actual provided AKS/K3s resources; + their region does not change Challenge 4's setting. +- Challenge 6 uses the shared LocalBox custom location. Its registration + defaults to **Australia East** independently of the Azure host and participant + regions; see the [registration-region guidance](#localbox-registration-region-and-location-policies). +- Inherited/organizer-managed policies still apply. `SecurityControl=Ignore` + and `CostControl=Ignore` do not override arbitrary location restrictions. + Escalate those denials to the policy owner rather than disabling governance. + +For an existing event, preserve the original preferred-region order on retry. +Participant quota recovery pins the saved region/SKU, but the global hosted-tag +policy assignment's managed identity has a separate location constraint: +changing the first preferred region can cause +`AlreadyExistServicePrincipalInDifferentRegion`. For the recovered event the +working order was **Spain Central, West US 2, Australia East, Sweden Central**; +this is not a replacement for the pack's default or a guarantee of capacity. +Do not delete the policy assignment or relocate resources to force a retry. + ### LocalBox registration region and location policies If LocalBox validation reports `RequestDisallowedByAzure` with diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/test-14-timing-analysis.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/test-14-timing-analysis.md index acb520bbf..671abb174 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/test-14-timing-analysis.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/test-14-timing-analysis.md @@ -15,9 +15,10 @@ or unrun. Update assessment succeeded at 07:18:55.558 UTC. 7 November to 07:26:19.066 on 8 November, **10h 35m 38s**. - This second interval excludes Console setup before ARM and all subsequent shared/participant stages. It is **not the total Console job duration**. -- The participant resource group had no deployments or resources at the - post-completion checks. Console's final status and downstream completion - were not established by this investigation. +- The participant resource group was empty at the earlier post-guest checks. + Subsequently supplied Console output confirms completion of the global, + shared and participant deployment stages; those earlier checks were not + evidence of a downstream failure. - Defender cloud reporting and real participant sign-in remain manual checks. The guest completed without operator resubmission or infrastructure recreation. @@ -26,6 +27,28 @@ reconciliation and MetalLB recovery were all exercised. The historical startup report still records the BGInfo failure described below; it was not rewritten to disguise that failure. +## Console completion + +The operator subsequently supplied Console output for the same TEST14 event +and subscription. It reports successful completion of all three deployment +stages, including the single participant job: + +| Console stage | Reported duration | Outcome | +|---|---:|---| +| Global event deployment | 10h43m | Completed, 1/1 | +| Shared deployment | 6m01s | Completed, 1/1 | +| Participant deployment | 11m29s | Completed, 1/1 | +| **Sum of reported stage durations** | **Approximately 11h00m30s** | **All deployment stages completed** | + +This closes the earlier uncertainty about the Console handoff and downstream +deployment stages. The global duration includes LocalBox work; do not add the +guest or ARM-to-guest duration to this total. The global duration is only +reported to minute precision, and the sum excludes initial Console setup and +gaps between stages. Without job start/end timestamps, it is not an exact +end-to-end wall-clock duration. Completion messages establish Console-reported +deployment success, not independent validation of participant access or +Defender cloud reporting. + ## Comparison with previous observations These are individual observations, not an SLA or a controlled benchmark. @@ -148,7 +171,9 @@ image-start lines hours after their embedded timestamps. This report uses the embedded timestamps, not the later polling observation time. Private reports and operational snapshots remain outside the repository. -Console's screenshot does not establish a consistent UTC start/end pair; -the total Console job duration is intentionally not inferred from it. +Neither the earlier Console screenshot nor the subsequently supplied completion +output establishes a consistent UTC start/end pair. The stage-duration sum above +is therefore reported separately from the still-unmeasured total Console job +wall-clock duration. See the [TEST06 analysis](test-06-timing-analysis.md) for the earlier baseline and [hosted provisioning guidance](readme.md) for readiness and retry contracts. diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/localbox/readme.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/localbox/readme.md index 4f67e0634..ca1f6eefe 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/localbox/readme.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/localbox/readme.md @@ -53,7 +53,9 @@ The script prompts for the Entra group object ID, or accepts `-AksAdminGroupObje The reference VM is **enabled by default**; pass `-IncludeReferenceVm:$false` to skip its preparation. Operation confirmation prompts are off under PowerShell's default confirmation preference, including storage changes and recovery of a matching Failed image. Use `-Confirm` to request confirmations, or `-Confirm:$false` if your session has a stricter preference. `-WhatIf` and all configuration/health safety checks remain active. Supply `-AksAdminGroupObjectId` to avoid the group-ID prompt; reference-VM credentials are generated without prompting. -Preparation runs in this order: **storage -> logical networks -> AKS, proxy access and MetalLB -> VM image -> reference VM (unless disabled)**. A late Marketplace image failure therefore leaves the networks and AKS resources in place for the next run to reuse; it does not roll them back. The final manifest is written only after the requested preparation finishes. +Preparation starts with **storage -> logical networks**. By default, a new AKS cluster is validated and submitted asynchronously so its provisioning overlaps the Windows VM image import. After image completion, preparation requires AKS readiness, proxy access and MetalLB before creating the reference VM (unless disabled). Matching accepted/running AKS deployments can also overlap image import on resume; completed or legacy existing clusters retain verification-before-image ordering. + +Use `-SequentialProvisioning` for manual preparation to restore **AKS, proxy access and MetalLB -> VM image -> reference VM** ordering if contention is suspected. An image failure does not roll back earlier resources, and an asynchronously submitted AKS deployment may continue in Azure. Inspect both states before retrying. The final manifest is written only after the requested preparation finishes. If the Console-owned group is not available yet, `-SkipAks` prepares storage, both networks, the VM image and the reference VM (unless disabled). It does not fabricate a group or grant anyone cluster access. The manifest records this as a partial run, and full health checks fail until preparation is rerun without `-SkipAks` using a valid admin-group ID. @@ -92,10 +94,11 @@ There is **one import attempt per run**, not an endless retry loop. If the new a ## MetalLB preparation -MetalLB is installed automatically **after AKS is ready**, by this organizer -preparation script, not by participants. The Console global event hook deploys -LocalBox only; it registers the Runtime provider and supplies the object-ID tag -needed for this later step. No Client VM access is granted to attendees. +MetalLB is installed automatically **after AKS is ready**, by this preparation +script, not by participants. The Console global event hook registers the Runtime +provider, supplies the object-ID tag, deploys/reuses LocalBox and invokes this +script inside the Client before running infrastructure health checks. Manual +organizers invoke the script themselves. No Client VM access is granted to attendees. The script uses ARM APIs for the extension and load-balancer pool, without adding CLI extensions or querying Microsoft Graph as the Client managed identity. @@ -191,7 +194,9 @@ Before the workshop, verify: 3. Defender for Cloud has assessment evidence for this VM. Review available recommendations or a completed assessment with no findings; an empty list alone is not sufficient. 4. Azure Update Manager shows a **successful assessment and timestamp**. Periodic assessment is asynchronous; an organizer can run **Check for updates** on this VM if no result is available yet. Do not install patches just to prepare the demonstration. -The script intentionally leaves Defender assessment and update-result readiness as `NotVerified`; these portal checks remain required even after automated health tests pass. See the [Defender readiness checklist](manual-preparation.md#defender-for-servers-readiness-organizer). +Standalone `prepare-localbox.ps1` intentionally leaves Defender assessment and update-result readiness as `NotVerified`; its manifest is not a completed health report. For hosted events, the [automation wrapper](../hosted-events/readme.md#complete-localbox-preparation) additionally waits for the VM to be Running, Arc Connected and MDE Succeeded, then requests an Update Manager assessment and requires success with a timestamp before Full health checks. Use that run's status and health reports for the hosted result, not the preparation snapshot alone. + +Neither path proves actual Defender device reporting or Defender for Cloud assessment visibility. Those portal checks and real participant access remain required even after the hosted gate succeeds. See the [Defender readiness checklist](manual-preparation.md#defender-for-servers-readiness-organizer). The following examples show three separate readiness milestones on `sov-ref-vm`. @@ -219,7 +224,7 @@ If existing workloads use the secondary volume, do not remove it. Use the [manua ## Verify readiness -Follow [Pester health checks](../tests/readme.md). Provisioning success is not a health verdict. The nonsecret manifest intentionally records `FullHealthVerified = false`. +For hosted events, inspect the [run-correlated automation results](../hosted-events/readme.md#complete-localbox-preparation); Console runs upstream and Full LocalBox health checks automatically. For manual delivery or deliberate diagnostics, follow [Pester health checks](../tests/readme.md). Provisioning success is not a health verdict. The preparation manifest intentionally records `FullHealthVerified = false`; a successful hosted health report is separate evidence and does not rewrite that snapshot. For AKS runtime access, follow [Jumpstart's AKS guide](https://jumpstart.azure.com/azure_jumpstart_localbox/AKS): use a separate CLI profile/session authenticated as a group member, and keep `az connectedk8s proxy` running with the kubeconfig used for health checks (default `$HOME/.kube/config`). Do not log out the provisioning identity or give its managed identity broader permissions to run tests. diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/readme.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/readme.md index 15efec3b0..758e1b50f 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/readme.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/readme.md @@ -4,10 +4,10 @@ Choose the delivery path before running any setup or cleanup scripts. | Delivery path | Start here | Infrastructure ownership | | --- | --- | --- | -| Eligible Microsoft-hosted event | [Hosted events](hosted-events/readme.md) | Hacks Console provisions lab scopes, identities and baseline infrastructure. Event Leads complete LocalBox preparation and validate readiness. | +| Eligible Microsoft-hosted event | [Hosted events](hosted-events/readme.md) | Hacks Console provisions lab scopes, identities and infrastructure, prepares LocalBox and verifies infrastructure readiness. Event Leads verify participant access and Defender cloud reporting. | | Personal learning, customer-led or other non-Console delivery | [Manual setup](manual-setup/readme.md) | The subscription owner provisions, funds and removes the infrastructure. | -Both paths use [LocalBox preparation](localbox/readme.md) and the [Pester health checks](tests/readme.md). The Console integration remains in [labautomation](../labautomation); do not run those platform hooks as standalone scripts without the documented platform helper environment. +Both paths use [LocalBox preparation](localbox/readme.md) and the [Pester health checks](tests/readme.md). Hosted Console automation runs LocalBox preparation and its infrastructure checks without an organizer login to the Client; manual delivery requires the owner to run them. Do not start a competing preparation session during hosted automation. The Console integration remains in [labautomation](../labautomation); do not run those platform hooks as standalone scripts without the documented platform helper environment. ## Responsibility boundary @@ -16,10 +16,12 @@ Both paths use [LocalBox preparation](localbox/readme.md) and the [Pester health | Subscriptions, users, initial lab RBAC, provider/quota preparation | Console and content automation | Subscription owner | | Participant shared Azure AKS with Ubuntu confidential pool, K3s and networking | Console content automation for Challenges 5/7 | Subscription owner using the lab template | | Challenge 4 ACI/ACR comparison and Challenge 5 applications | Participants using walkthrough scripts | Same walkthrough scripts against the prepared scope | -| Shared LocalBox deployment | Console submits once per event in the lowest-GUID event subscription; full readiness still requires verification | Subscription owner deploys LocalBox | -| Storage, Windows image, VM/AKS networks, AKS on Azure Local | Event Lead runs preparation on each LocalBox Client | Same preparation script | +| Shared LocalBox deployment | Console deploys/reuses once per event in the lowest-GUID event subscription and waits for nested readiness | Subscription owner deploys LocalBox | +| Storage, Windows image, VM/AKS networks, AKS on Azure Local, proxy RBAC and MetalLB | Console invokes preparation automatically on LocalBox-Client | Subscription owner runs the same preparation script | +| Reference VM, MDE installation, update assessment and LocalBox infrastructure health | Console prepares the reference VM, waits for onboarding, requests an assessment and runs upstream/Full health checks | Organizer prepares and validates explicitly | | AKS Local Entra admin group | Console-owned event group; use **Lab Group ObjectId** from the Credentials tab | Supply an existing security group with intended users as members | -| Runtime health and participant access | Event Lead | Organizer/subscription owner | +| Participant sign-in, Bastion/proxy access, Defender cloud reporting and workload smoke tests | Event Lead; not established by the automated infrastructure gate | Organizer/subscription owner | +| Challenge 6 participant Windows VM and challenge workloads | Participants using walkthroughs; automation does not complete their exercises | Same walkthroughs against prepared infrastructure | | Teardown | Console lifecycle; verify shared resources and group cleanup | Explicit owner-managed cleanup | One Console team is one lab. This content pack shares **one LocalBox across the entire event**, even when participant labs span several subscriptions. Use the Console's **Shared LocalBox Subscription ID** and **Shared LocalBox Resource Group ID** to find it. Participant VMs stay in their assigned scopes. The existing Azure AKS/K3s labs are not replaced by AKS on Azure Local. \ No newline at end of file diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md index 13b8ea987..eb8b2ed52 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md @@ -2,6 +2,26 @@ There are two kinds of tests: offline mocked script tests, and live health tests against explicitly selected Azure scopes. Live tests do not deploy attendee VMs, install agents, enable paid plans, apply patches or repair failures. They do not replace the [participant VM readiness exercise](../localbox/manual-preparation.md#step-6-test-the-environment). +## Current validation and historical evidence + +Hosted Console automation invokes the upstream and Full LocalBox suites itself; +manual delivery and deliberate diagnostics use the runner instructions below. +The hosted wrapper additionally prepares resources, waits for reference-VM +onboarding and requests an Update Manager assessment. Those operations are not +side effects of the read-only health-test runner. + +The [9 November 2026 verified state](../hosted-events/readme.md#current-verified-state) +records **26/26 LocalBox tests, 7/7 shared stages and 25/25 participant jobs** +passing after targeted K3s recovery, with credential publication confirmed. +It is a reuse/recovery result, not a clean first-attempt deployment benchmark. +Participant sign-in, private/proxy application access, Defender cloud reporting +and planned nested-VM capacity still require event-specific validation. + +The dated TEST observations below describe evidence available at each checkpoint. +They retain original failures and outstanding checks; use the current summary +for their later disposition. In particular, successful Full health does not +resolve the historical startup watcher's missing successful terminal report. + ## Tooling and authentication Use PowerShell 7, Pester 5.7.1 or later within major version 5, Azure CLI, and `kubectl` for Full checks. Offline tests were exercised with PowerShell 7.4.13 and Pester 5.7.1. Live Windows/Azure validation remains an integration gate; these versions are not a claim that all LocalBox versions were tested. @@ -56,9 +76,14 @@ verification remains enabled. Registration is checked up to 24 times, with 5-second request limits and 5-second delays, followed by a bounded Ready check. Persistent failures remain extension failures. No retry deletes infrastructure. -The observed failed VM was recovered by a separately authorized manual retry; -that does not validate this new automatic retry in a fresh deployment. Event-lead -testing should use the feature branch containing these changes, not `main`. +The 9 November event exercised the bounded installer, but three extensions still +exhausted their attempts. An unchanged ARM redeployment surfaced the same old +failure rather than executing the installer again. A separately authorized +`forceUpdateTag` update triggered new executions; all three then installed on +attempt 1/3, and service/Ready-node checks passed before the successful Console +retry. This validates the [targeted recovery workflow](../hosted-events/readme.md#retry-a-failed-install-k3s-extension), +not a guarantee that automatic retries overcome every download outage. Test +with the reviewed content revision imported into Console. ## MetalLB preparation checks @@ -73,7 +98,10 @@ Live verification on traininglab-02 confirmed that the recovery preflight perfor no PUT when the objects exist, followed by **14/14 Full health tests passing** and verified temporary-route cleanup. The missing-object/retry branches are tested with fixtures; no healthy live objects were removed to manufacture a failure. -A fresh deployment exercising automatic recovery remains a separate live gate. +At that checkpoint, live automatic recovery remained unverified. TEST13 later +exercised one bounded unchanged-pool reapply during resume, and +[TEST14](../hosted-events/test-14-timing-analysis.md) exercised recovery during +its cold deployment. Those results do not turn the read-only tests into repair tools. MetalLB preparation tests are included in `prepare-localbox.tests.ps1`. Run `Invoke-Pester ./prepare-localbox.tests.ps1 -FullName 'LocalBox MetalLB preparation*','MetalLB health evidence*','CLI output streams*' -Output Detailed` @@ -97,8 +125,9 @@ It verifies that both deployment entry points default registration to Australia the global hook passes that region independently of host-region selection, and all four Challenge 1 policy parameter examples include the Australia East lab-only exception while retaining the four European regions. These checks make no -Azure changes; a fresh Console deployment and participant VM creation remain -required to validate regional service availability and effective inherited policies. +Azure changes. The later hosted event verified LocalBox provisioning with this +registration setting; participant VM creation in the assigned scope remains an +event-specific check of service availability and effective inherited policies. ## Global event LocalBox checks @@ -123,8 +152,9 @@ and script transformations compiled with a **33,073-byte** startup payload. A read-only Windows PowerShell 5.1 probe resolved the actual installed BGInfo binary, verified cross-process mutex ownership, parsed the packaged standalone monitor and verified redacted failure details. It did not execute BGInfo, start -startup tasks or change the current test runner. A fresh deployment must still -verify the complete startup and subsequent-logon behavior. +startup tasks or change the current test runner. At this checkpoint, a fresh +deployment was still needed to verify startup and subsequent-logon behavior. +See the later BGInfo evidence below and the current validation summary. TEST15 isolated the remaining BGInfo failure to stale PowerShell `LASTEXITCODE`, not a failed BGInfo process. In the existing desktop session, direct GUI invocation @@ -144,8 +174,11 @@ of 1, while an invalid configuration failed with the actual exit code 2. Temporary probe tasks/files were removed. The upstream Bicep and script transforms compiled with a **34,442-byte** startup payload. This is a future-deployment fix: the existing immutable startup payload and historical failure report were not -rewritten, and storage initialization was not replayed. A fresh deployment remains -the integration check for the original runner's subsequent-logon task registration. +rewritten, and storage initialization was not replayed. The later event Client +contained the fix, showed no BGInfo failure and had the subsequent-logon Pester +task registered. Actual subsequent-logon execution is a separate check, and the +historical `StartupExited` watcher report still lacks a successful terminal +report; registration and passing Full health do not resolve that discrepancy. The combined WinGet, startup, deployment, automation, validation-retry, global LocalBox and failure-summary suites passed **267 tests** after these changes. @@ -155,8 +188,9 @@ existing-VM custom-data preservation was verified. Live TEST12 repair verified matching CLI/Client 1.29.380, successful COM enumeration, zero-exit package configuration after the retired-entry removal, and executable AzCopy. After safe resume, both base-image SHA256 checksums -passed at 18:15:09 UTC on 6 November 2026. The current instance is being used for the remaining -nested deployment and Full health gates; these prerequisite checks alone do not +passed at 18:15:09 UTC on 6 November 2026. That instance was then used for nested +deployment checks but was later torn down without a verified Full health pass, +as recorded in the TEST12 closeout below. These prerequisite checks alone do not establish end-to-end readiness. Run `Invoke-Pester ./localbox-startup.tests.ps1,./localbox-deployment.tests.ps1 -Output Detailed` @@ -167,9 +201,10 @@ handling, cleanup, persistent replay prevention, Console report correlation and one-hour/two-minute monitoring thresholds. Mocked transfers never download VHDs or restart a live VM. The modified real upstream Bicep was compiled, and both upstream script transformations were syntax-checked offline. A new Windows -deployment is still required to verify custom-data delivery, scheduled-task -execution and end-to-end Console reporting together; existing failed instances -are deliberately not retrofitted. +deployment was required at that checkpoint to verify custom-data delivery, +scheduled-task execution and Console reporting together. Later infrastructure +success does not close the startup-monitor discrepancy described above; +existing failed instances are deliberately not retrofitted. The startup suite also covers WinGet-before-storage ordering, the one-hour prerequisite timeout, stale-PATH AzCopy resolution using only the current user's @@ -181,8 +216,9 @@ restarting or overwriting anything. On traininglab-17, a read-only Windows probe ran the actual resolver with AzCopy absent from the process PATH. It resolved the startup user's existing WinGet link and successfully ran `azcopy --version`; no image was downloaded and no -deployment was resumed. The prerequisite race and tag-retention fixes still -require a fresh end-to-end Console deployment. +deployment was resumed. This isolated probe did not validate the prerequisite +race or tag retention end to end; later deployment evidence is tracked separately +in the current validation summary. Error-boundary regressions cover a non-terminating upstream task lookup followed by its readiness gate, a subsequent terminating readiness failure, explicit @@ -477,7 +513,7 @@ The organizer also manually created a storage account in `rg-labuser-0011` and o Download [test-sovereign-cloud.ps1](../test-sovereign-cloud.ps1) from the same trusted ref as preparation. `-DownloadTests` defaults to `$true`: every run downloads the helpers and two suites to a temporary directory and prints the repository/ref being used. Internet access is required unless you pass `-DownloadTests:$false` with a complete local checkout. `-GitHubRef` must identify the **same reviewed ref** as the runner, preferably a commit SHA; it defaults to `main`. Downloaded scripts execute under your privileges, so review and pin them. Download failures stop the run rather than silently falling back to other code. -For a fork, also specify `-GitHubRepository 'owner/MicroHack'`. For example, testing this branch uses `-GitHubRepository 'janegilring/MicroHack' -GitHubRef 'sov-cloud-localbox-post-automation'`. The default repository remains `microsoft/MicroHack`. +For a fork, also specify `-GitHubRepository 'owner/MicroHack'`. For example, use `-GitHubRepository 'janegilring/MicroHack' -GitHubRef ''`, replacing the placeholder with the published commit containing both the runner and suites. Do not assume an old test branch or moving `main` contains the content imported into Console. The default repository remains `microsoft/MicroHack`. Hosted packaging has its own [immutable source-pin promotion contract](../hosted-events/readme.md#source-pins-and-upstream-promotion). ```powershell $ref = 'main' diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/walkthrough/challenge-06/solution-06.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/walkthrough/challenge-06/solution-06.md index 6183ac7fa..cb2e2cb12 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/walkthrough/challenge-06/solution-06.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/walkthrough/challenge-06/solution-06.md @@ -29,7 +29,7 @@ Use your [Sovereign Cloud Codespace](https://github.com/microsoft/MicroHack/blob > Complete Challenge 1's [Preparing for Next Challenges](https://github.com/microsoft/MicroHack/blob/main/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/walkthrough/challenge-01/solution-01.md#preparing-for-next-challenges): your exercise policy assignments, including **Allowed locations** and any bonus initiative, must be in **DoNotEnforce**. This does not override inherited or organizer-managed policies. The organizer must also confirm that the shared LocalBox **custom location's Azure region** is permitted for resources created in your participant resource group. > [!NOTE] -> LocalBox is typically deployed by the workshop facilitator due to resource requirements and deployment time. See the [LocalBox deployment and readiness guide](https://github.com/microsoft/MicroHack/blob/main/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/demo-vm-creator/README.md). For a personal subscription, an authorized owner must also enable Defender for Servers before this challenge and review the plan's charges. Students in hosted labs should not change subscription-level Defender plans. +> For hosted events, [Console automation](../../resources/hosted-events/readme.md#complete-localbox-preparation) deploys and prepares shared LocalBox infrastructure, including AKS, proxy permissions, MetalLB, the VM image and a reference VM, then verifies infrastructure health. The facilitator verifies actual participant access and Defender cloud reporting. You do not log into LocalBox-Client or run preparation; you still create your own Windows VM and deploy the challenge workloads. For a personal subscription, follow [manual setup](../../resources/manual-setup/readme.md) and [LocalBox preparation](../../resources/localbox/readme.md); an authorized owner must enable Defender for Servers and review its charges. Students in hosted labs should not change subscription-level Defender plans. --- From cc125761d94bc90d9eb3523a1b47aa277260dba6 Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Tue, 10 Nov 2026 05:38:26 +0000 Subject: [PATCH 37/37] fix: recover safely from asynchronous LocalBox capacity failures Inspect nested ARM failures before advancing candidates, permit v5 fallback only without an existing VM, and preserve partial regional resources. Check subscription SKU restrictions and family/regional quota; retain the hosted policy identity location across retries. Add regression coverage and document the authorized TEST16 cleanup. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../labautomation/deploy-localbox.ps1 | 26 ++-- .../labautomation/global-event-deploy.ps1 | 87 +++++++---- .../labautomation/hosted-tag-policy.ps1 | 15 ++ .../labautomation/localbox-credentials.ps1 | 73 ++++++++- .../resources/hosted-events/readme.md | 97 ++++++++++-- .../resources/tests/global-localbox.tests.ps1 | 72 ++++++++- .../tests/hosted-tag-policy.tests.ps1 | 26 ++++ .../tests/localbox-arm-recovery.tests.ps1 | 146 ++++++++++++++++++ .../tests/localbox-deployment.tests.ps1 | 50 +++++- .../tests/prepare-localbox.tests.ps1 | 32 +++- .../resources/tests/readme.md | 24 +++ 11 files changed, 573 insertions(+), 75 deletions(-) create mode 100644 03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-arm-recovery.tests.ps1 diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/deploy-localbox.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/deploy-localbox.ps1 index 559565c15..6c30b9ec0 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/deploy-localbox.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/deploy-localbox.ps1 @@ -225,27 +225,27 @@ Invoke-AzJson -Arguments @( '--namespace', 'Microsoft.Network', '--wait' ) | Out-Null -$availableVmSize = & az vm list-sizes ` - --subscription $SubscriptionId ` - --location $Location ` - --query "[?name=='$VmSize'].name | [0]" ` - --output tsv ` - --only-show-errors -if ($LASTEXITCODE -ne 0) { - throw "Unable to query VM size availability in $Location." -} -if ($availableVmSize -ne $VmSize) { - throw "VM size $VmSize is unavailable in $Location." +$availableVmSizes = @(Invoke-AzJson -Arguments @( + 'vm', 'list-skus', '--subscription', $SubscriptionId, '--location', $Location, + '--resource-type', 'virtualMachines', '--size', $VmSize, '--all' +) | Where-Object { $_.name -eq $VmSize }) +if ($availableVmSizes.Count -ne 1 -or + @($availableVmSizes[0].restrictions | Where-Object { $_.type -ne 'Zone' }).Count -gt 0) { + throw "VM size $VmSize is unavailable in $Location for this subscription (missing SKU or regional restriction). No deployment was submitted." } -$familyName = if ($VmSize -eq 'Standard_E32s_v6') { 'Standard Esv6 Family vCPUs' } else { 'Standard Esv5 Family vCPUs' } +$familyName = if ($VmSize -eq 'Standard_E32s_v6') { 'standardEsv6Family' } else { 'standardESv5Family' } $usageEntries = Invoke-AzJson -Arguments @( 'vm', 'list-usage', '--subscription', $SubscriptionId, '--location', $Location ) -$usage = @($usageEntries | Where-Object { $_.name.localizedValue -eq $familyName }) | Select-Object -First 1 +$usage = @($usageEntries | Where-Object { $_.name.value -eq $familyName }) | Select-Object -First 1 if (-not $usage -or ($usage.limit - $usage.currentValue) -lt 32) { throw "At least 32 unused $familyName vCPUs are required in $Location." } +$regionalUsage = @($usageEntries | Where-Object { $_.name.value -eq 'cores' }) | Select-Object -First 1 +if (-not $regionalUsage -or ($regionalUsage.limit - $regionalUsage.currentValue) -lt 32) { + throw "At least 32 unused regional vCPUs are required in $Location." +} if ([string]::IsNullOrWhiteSpace($AzureLocalResourceProviderObjectId)) { $servicePrincipals = Invoke-AzJson -Arguments @( diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/global-event-deploy.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/global-event-deploy.ps1 index 985ae755c..fb4529a2a 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/global-event-deploy.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/global-event-deploy.ps1 @@ -82,7 +82,7 @@ $localBoxLocations = @( Select-Object -Unique ) $activeDeploymentStates = @('Accepted', 'Running', 'Ready', 'Creating', 'Created', 'Succeeded') -$resourceGroup = Get-AzResourceGroup -Name $localBoxResourceGroupName -ErrorAction SilentlyContinue +$resourceGroup = Get-AzResourceGroup -ErrorAction Stop | Where-Object ResourceGroupName -eq $localBoxResourceGroupName if ($resourceGroup) { Update-AzTag ` -ResourceId $resourceGroup.ResourceId ` @@ -122,44 +122,68 @@ if ($localBoxDeployment) { -Tag @{ CostControl = 'Ignore' } ` -ErrorAction Stop | Out-Null } + try { + Wait-LocalBoxDeployment -ResourceGroupName $localBoxResourceGroupName -DeploymentName $localBoxDeployment.DeploymentName + } + catch { + if (-not (Test-LocalBoxCapacityFailure $_)) { throw } + Write-Warning $_.Exception.Message + $localBoxDeployment = $null + } } -else { - foreach ($localBoxLocation in $localBoxLocations) { - try { - Write-Host "Submitting shared LocalBox deployment in $localBoxLocation..." -ForegroundColor Cyan - $localBoxDeployment = & (Join-Path $PSScriptRoot 'deploy-localbox.ps1') ` - -SubscriptionId $SubscriptionId ` - -ResourceGroupName $localBoxResourceGroupName ` - -Location $localBoxLocation ` - -AzureLocalResourceProviderObjectId $azureLocalResourceProviderObjectIds[0] ` - -AzureLocalInstanceLocation 'australiaeast' ` - -UseConsoleCredentials ` - -NoWait - - if (-not $localBoxDeployment -or $localBoxDeployment.ProvisioningState -ne 'Submitted') { - throw "LocalBox deployment submission did not return the expected Submitted state." + +if (-not $localBoxDeployment) { + $candidateFailures = [Collections.Generic.List[string]]::new() + :locations foreach ($localBoxLocation in $localBoxLocations) { + foreach ($localBoxVmSize in @('Standard_E32s_v6', 'Standard_E32s_v5')) { + Update-MhhToken | Out-Null + $existingGroup = Get-AzResourceGroup -ErrorAction Stop | Where-Object ResourceGroupName -eq $localBoxResourceGroupName + $resources = if ($existingGroup) { + @(Get-AzResource -ResourceGroupName $localBoxResourceGroupName -ErrorAction Stop) + } else { @() } + if (@($resources | Where-Object ResourceType -eq 'Microsoft.Compute/virtualMachines').Count) { + throw "A VM already exists in $localBoxResourceGroupName. Automatic LocalBox SKU/region changes are blocked; inspect the failed deployment. No VM was resized or deleted." } - break - } - catch { - $failureDetails = @( - $_.Exception.ToString() - $_.ErrorDetails.Message - ) | Where-Object { $_ } - $failureDetails = $failureDetails -join ' ' - $retryableRegionalFailure = $failureDetails -match 'RequestDisallowedByAzure|locationineligible|SkuNotAvailable|QuotaExceeded|OperationNotAllowed|AllocationFailed|VM size .+ unavailable|At least 32 unused' - - if ($retryableRegionalFailure -and $localBoxLocation -ne $localBoxLocations[-1]) { - Write-Warning "LocalBox is unavailable in $localBoxLocation; trying the next preferred region. $($_.Exception.Message)" + $conflicts = @($resources | Where-Object { + $_.ResourceType -notin @('Microsoft.Storage/storageAccounts', 'Microsoft.Resources/deployments') -and + ($_.Location -replace '\s', '') -ine ($localBoxLocation -replace '\s', '') -and $_.Location -ine 'global' + }) + if ($conflicts.Count) { + $message = "Skipping ${localBoxLocation}: existing LocalBox resources are in another or unknown region. Preserve the partial environment; a region change requires a separately approved migration or fresh scope." + Write-Warning $message + $candidateFailures.Add($message) + continue locations + } + try { + Write-Host "Submitting shared LocalBox deployment in $localBoxLocation using $localBoxVmSize..." -ForegroundColor Cyan + $localBoxDeployment = & (Join-Path $PSScriptRoot 'deploy-localbox.ps1') ` + -SubscriptionId $SubscriptionId ` + -ResourceGroupName $localBoxResourceGroupName ` + -Location $localBoxLocation ` + -VmSize $localBoxVmSize ` + -AzureLocalResourceProviderObjectId $azureLocalResourceProviderObjectIds[0] ` + -AzureLocalInstanceLocation 'australiaeast' ` + -UseConsoleCredentials ` + -NoWait + + if (-not $localBoxDeployment -or $localBoxDeployment.ProvisioningState -ne 'Submitted') { + throw "LocalBox deployment submission did not return the expected Submitted state." + } + Wait-LocalBoxDeployment -ResourceGroupName $localBoxResourceGroupName -DeploymentName $localBoxDeployment.DeploymentName + break locations + } + catch { + if (-not (Test-LocalBoxCapacityFailure $_)) { throw } + $message = "$localBoxLocation/$localBoxVmSize : $(ConvertTo-LocalBoxDiagnosticText $_.Exception.Message 2000)" + Write-Warning "LocalBox capacity candidate failed: $message" + $candidateFailures.Add($message) $localBoxDeployment = $null - continue } - throw } } if (-not $localBoxDeployment) { - throw "LocalBox deployment submission failed in all candidate regions for subscription $SubscriptionId." + throw "LocalBox capacity candidates exhausted for subscription $SubscriptionId. No resources were deleted or relocated. Retry after availability changes or arrange an approved fresh scope/migration. $($candidateFailures -join '; ')" } } @@ -205,7 +229,6 @@ foreach ($participantObjectId in $participants) { } } -Wait-LocalBoxDeployment -ResourceGroupName $localBoxResourceGroupName -DeploymentName $localBoxDeployment.DeploymentName $groupId = ($groupCredentials.HackboxCredential | Where-Object name -eq 'Lab Group ObjectId').value $automation = Invoke-LocalBoxHostedAutomation -SubscriptionId $SubscriptionId -ResourceGroupName $localBoxResourceGroupName ` -AksAdminGroupObjectId $groupId -Package $localBoxAutomationPackage diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/hosted-tag-policy.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/hosted-tag-policy.ps1 index 5bc97b60d..40e884113 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/hosted-tag-policy.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/hosted-tag-policy.ps1 @@ -14,6 +14,21 @@ function Initialize-MhhHostedTagPolicy { throw "The Azure context must target hosted subscription $SubscriptionId before assigning the tag initiative." } + $assignment = Invoke-AzRestMethod -Method GET ` + -Path "/subscriptions/$SubscriptionId/providers/Microsoft.Authorization/policyAssignments/sov-hosted-control-tags?api-version=2026-06-01" ` + -ErrorAction Stop + if ($assignment.StatusCode -eq 200) { + $existing = $assignment.Content | ConvertFrom-Json -ErrorAction Stop + if ([string]::IsNullOrWhiteSpace($existing.location)) { + throw 'Existing hosted control-tag assignment has no identity location. Inspect it before retrying; no location change was attempted.' + } + $Location = $existing.location + Write-Host "Preserving hosted control-tag policy identity location: $Location." + } + elseif ($assignment.StatusCode -ne 404) { + throw "Unable to read hosted control-tag assignment (HTTP $($assignment.StatusCode)); no policy deployment was attempted." + } + Write-Host "Assigning hosted MCAPS control-tag initiative in subscription $SubscriptionId..." $deployment = New-AzSubscriptionDeployment ` -Name "sov-hosted-tags-$Location" ` diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-credentials.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-credentials.ps1 index f70540c39..311aba4a3 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-credentials.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/labautomation/localbox-credentials.ps1 @@ -14,6 +14,62 @@ function Get-LocalBoxConsoleGroupCredential { @{ HackboxCredential = @{ name = 'Lab Group DisplayName'; value = $group.DisplayName; note = '' } } } +function Get-LocalBoxDeploymentFailure { + [CmdletBinding()] + param( + [Parameter(Mandatory)][string]$ResourceGroupName, + [Parameter(Mandatory)][string]$DeploymentName, + [ValidateRange(0, 5)][int]$Depth = 0 + ) + + Update-MhhToken | Out-Null + $operations = @(Get-AzResourceGroupDeploymentOperation -ResourceGroupName $ResourceGroupName -DeploymentName $DeploymentName -ErrorAction Stop | + Where-Object { $_.Properties.ProvisioningState -eq 'Failed' }) + if (-not $operations.Count) { + return [pscustomobject]@{ Code = 'UnknownDeploymentFailure'; Message = 'No failed operation details were returned; inspect the deployment in Azure.' } + } + foreach ($operation in $operations) { + $status = $operation.Properties.StatusMessage + if ($status -is [string]) { + $status = $status | ConvertFrom-Json -ErrorAction Stop + } + $pending = [Collections.Generic.Queue[object]]::new() + if ($status.error) { $pending.Enqueue($status.error) } + $details = @( + while ($pending.Count) { + $errorDetail = $pending.Dequeue() + if ($errorDetail.details.Count) { + foreach ($child in $errorDetail.details) { $pending.Enqueue($child) } + } + else { + [pscustomobject]@{ Code = [string]$errorDetail.code; Message = [string]$errorDetail.message } + } + } + ) + $target = $operation.Properties.TargetResource + if ($Depth -lt 5 -and $target.ResourceType -eq 'Microsoft.Resources/deployments' -and + $target.Id -match "^/subscriptions/$([regex]::Escape((Get-AzContext -ErrorAction Stop).Subscription.Id))/resourceGroups/$([regex]::Escape($ResourceGroupName))/providers/Microsoft.Resources/deployments/[^/]+$" -and + ($details.Count -eq 0 -or @($details | Where-Object { $_.Code -notin @('DeploymentFailed', 'InvalidTemplateDeployment') }).Count -eq 0)) { + Get-LocalBoxDeploymentFailure -ResourceGroupName $ResourceGroupName -DeploymentName $target.ResourceName -Depth ($Depth + 1) + } + elseif ($details.Count) { $details } + else { + [pscustomobject]@{ Code = 'UnknownDeploymentFailure'; Message = 'Failed operation has no usable error details; inspect the deployment in Azure.' } + } + } +} + +function Test-LocalBoxCapacityFailure { + param([Parameter(Mandatory)][System.Management.Automation.ErrorRecord]$Failure) + + if ($Failure.Exception.Data.Contains('LocalBoxCapacityFailure')) { + return $Failure.Exception.Data['LocalBoxCapacityFailure'] -eq $true + } + $message = "$($Failure.Exception.Message) $($Failure.ErrorDetails.Message)" + if ($message -match 'AuthorizationFailed|Forbidden|RequestDisallowedByPolicy|InvalidAuthenticationToken') { return $false } + return $message -match '\b(SkuNotAvailable|AllocationFailed|ZonalAllocationFailed|QuotaExceeded)\b|VM size .+ unavailable|At least 32 unused' +} + function Wait-LocalBoxDeployment { [CmdletBinding()] param( @@ -30,7 +86,22 @@ function Wait-LocalBoxDeployment { break } if ($deployment.ProvisioningState -in @('Failed', 'Canceled')) { - throw "LocalBox deployment '$DeploymentName' ended as $($deployment.ProvisioningState)." + $failure = [InvalidOperationException]::new("LocalBox deployment '$DeploymentName' ended as $($deployment.ProvisioningState).") + $failure.Data['LocalBoxCapacityFailure'] = $false + if ($deployment.ProvisioningState -eq 'Failed') { + try { + $details = @(Get-LocalBoxDeploymentFailure -ResourceGroupName $ResourceGroupName -DeploymentName $DeploymentName) + $summary = ($details | ForEach-Object { "[$($_.Code)] $($_.Message)" }) -join '; ' + $failure = [InvalidOperationException]::new("$($failure.Message) $(ConvertTo-LocalBoxDiagnosticText $summary 2000)") + $failure.Data['LocalBoxCapacityFailure'] = $details.Count -gt 0 -and + @($details | Where-Object { $_.Code -notin @('SkuNotAvailable', 'AllocationFailed', 'ZonalAllocationFailed', 'QuotaExceeded') }).Count -eq 0 + } + catch { + $failure = [InvalidOperationException]::new("$($failure.Message) Could not retrieve deployment failure details: $(ConvertTo-LocalBoxDiagnosticText $_.Exception.Message 400). Inspect Azure deployment operations before retrying.") + $failure.Data['LocalBoxCapacityFailure'] = $false + } + } + throw $failure } if ((Get-Date -AsUTC) -ge $deadline) { throw "Timed out waiting for LocalBox deployment '$DeploymentName'. Rerun the global event hook after investigating the deployment." diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md index cec991af0..46b1b98ca 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/hosted-events/readme.md @@ -42,6 +42,14 @@ run or claim that all startup-monitor behavior has been validated. Older TEST06/12/13/14 observations below are dated diagnostic history, not the current event status. Preserve their original failures and measurement boundaries. +**Later cold test (TEST16, reviewed 10 November):** the initial LocalBox ARM +deployment failed with `SkuNotAvailable` for `Standard_E32s_v6` in Spain Central, +before a Client VM or guest preparation existed. Shared and participant stages +were skipped. This was a capacity restriction, not a bootstrap/download failure. +The [ARM capacity recovery handling](#localbox-arm-capacity-failures) below addresses +the missed asynchronous failure path; a new Console run is still required to +validate recovery end to end. It does not change the earlier event's successful result. + ## Configure the event 1. Agree the Sovereign Cloud scenario with your GEO/program lead and request an Event Lead account through the program's access process. @@ -69,9 +77,11 @@ verify the global deployment job actually ran before continuing. The Console runs the global hook **once per event in the lowest-GUID event subscription**, passing all event lab-user object IDs. It completes before `shared-deploy-lab.ps1` runs in each subscription, followed by participant -deployments. A global failure stops the event deployment. Host-region fallback -uses the preferred regions within that selected subscription; it does not choose -a different subscription if quota or SKU availability is insufficient. +deployments. A global failure stops the event deployment. Host capacity selection +tries `Standard_E32s_v6`, then `Standard_E32s_v5`, in each preferred region within +that selected subscription. It does not choose another subscription, delete +partial infrastructure or resize an existing VM. Regional resources constrain +recovery to their existing region; see [ARM capacity failures](#localbox-arm-capacity-failures). The global hook deploys/reuses one `rg-localbox-shared`, including one LocalBox Client. It publishes **Shared LocalBox Subscription ID**, **Shared LocalBox Resource @@ -338,7 +348,7 @@ attempt to decrypt them from a different Windows account. ### Timeouts and scheduling -The initial ARM wait allows six hours, nested readiness eight hours, and the +The ARM wait allows six hours per submitted capacity candidate, nested readiness eight hours, and the Managed Run Command twelve hours (with five additional minutes for host polling). These are independent ceilings, **not** expected durations. Confirm with the Console operator that its global-job limit accommodates the intended run; token @@ -372,6 +382,59 @@ bounded intervals and potential optimizations. ### Diagnostics and reruns +#### LocalBox ARM capacity failures + +The global hook waits for ARM completion inside capacity selection, including +when resuming a running deployment. `Submitted` is not success. Failed operations +are inspected for nested error codes and bounded, redacted messages so Console +shows the underlying `SkuNotAvailable`/allocation/quota error rather than only +`ended as Failed`. + +For a new host, each candidate is tried at most once per hook invocation: +**E32s_v6 then E32s_v5 in the first preferred region**, followed by the same +sequence in subsequent regions when safe. The deployer checks subscription SKU +restrictions, at least 32 unused family vCPUs and at least 32 unused regional +vCPUs before submission. These checks are not a capacity reservation; ARM can +still report an allocation failure afterward. + +Automatic fallback is allowed only for recognized capacity failures. Canceled +deployments, timeouts, policy/authorization errors, mixed failure causes and +missing/unreadable diagnostics stop the job. A timeout does not establish that +the original deployment stopped, so no competing deployment is submitted. + +Before every new candidate, the hook rereads the resource inventory: + +- If any VM exists in the LocalBox group, stop for organizer inspection rather + than resize/recreate it or change its immutable startup payload. +- Existing networking/workspace and other regional resources must match the + candidate region. Otherwise that region is skipped. Resource-group metadata + alone does not pin the host region; fixed-registration staging storage is + retained independently. +- No cleanup is automatic. When a partial deployment pins the group to a region + where both host sizes are unavailable, retry after availability changes or + arrange a separately authorized cleanup/migration or fresh event. Merely + reordering regions cannot move the partial infrastructure. + +For TEST16, read-only checks on 10 November found both E32s sizes restricted in +Spain Central for the selected subscription; Sweden Central listed both without +restrictions and had sufficient family/regional quota. The existing Spain +networking prevented an in-place cross-region retry. These are dated planning +observations, not proof that a Sweden deployment will succeed. Do not remove +the failed shared group without checking for new work and obtaining organizer +approval; leave participant scopes and subscription policies intact. + +After explicit organizer approval, the failed TEST16 shared group was removed +on 10 November, following confirmation that only the seven known leftover +resources remained and no VM or active deployment existed. A subsequent read +confirmed the group was gone, all eight participant groups remained, and the +subscription's hosted-tag policy identity still used Spain Central. This was +operator-authorized cleanup, not automatic fallback; no replacement deployment +was started. The next Console run remains pending. + +Re-import the updated content pack before retrying Console. A failed global +stage correctly prevents shared and participant deployments; guest health tests +cannot run until the host and nested infrastructure exist. + #### Guarded startup for new Client VMs Re-import the complete `labautomation` directory, including @@ -771,14 +834,16 @@ European region. and `CostControl=Ignore` do not override arbitrary location restrictions. Escalate those denials to the policy owner rather than disabling governance. -For an existing event, preserve the original preferred-region order on retry. -Participant quota recovery pins the saved region/SKU, but the global hosted-tag -policy assignment's managed identity has a separate location constraint: -changing the first preferred region can cause -`AlreadyExistServicePrincipalInDifferentRegion`. For the recovered event the -working order was **Spain Central, West US 2, Australia East, Sweden Central**; -this is not a replacement for the pack's default or a guarantee of capacity. -Do not delete the policy assignment or relocate resources to force a retry. +For an existing event, normally retain the original preferred-region order. +Participant quota recovery pins the saved region/SKU. Hosted-tag setup now reads +and preserves the existing policy assignment's identity location independently +of the preferred host region; lookup failures stop setup rather than treating +the assignment as absent. This prevents the earlier +`AlreadyExistServicePrincipalInDifferentRegion` failure when preferences changed. +The recovered 25-lab event used **Spain Central, West US 2, Australia East, Sweden +Central**; this is not a replacement for the pack's default or a capacity guarantee. +Existing LocalBox regional resources still constrain host fallback. Do not delete +the policy assignment or relocate resources to force a retry. ### LocalBox registration region and location policies @@ -792,10 +857,10 @@ passes `azureLocalInstanceLocation` to the staging storage module, whose account name is `localbox` plus a generated suffix. In the 4 October 2026 hosted test, this account was rejected in West Europe while the host was selected in Spain Central. **Host-region fallback cannot resolve a rejection in the fixed -registration/staging-storage region.** The shared setup tries -the next configured preferred host region for recognized regional failures; -it does not change the Azure Local registration region or delete the shared -resource group. The CLI wrapper retains Azure's error details in the exception +registration/staging-storage region.** Host capacity fallback does not change the +Azure Local registration region or delete the shared resource group. Fixed-region +eligibility failures require organizer review, not repeated host-region attempts. +The CLI wrapper retains Azure's error details in the exception so PowerShell's generic native-command exit error cannot hide them from the fallback check. Policy denials, authorization failures, and unrecognized errors still stop the run. If all preferred regions fail, the final error is reported; diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/global-localbox.tests.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/global-localbox.tests.ps1 index a43feb458..1f29be5d4 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/global-localbox.tests.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/global-localbox.tests.ps1 @@ -10,7 +10,7 @@ BeforeAll { } Set-Content "$fixture/resource-providers.ps1" 'Register-TestProviders' @' -param($SubscriptionId, $ResourceGroupName, $Location, $AzureLocalResourceProviderObjectId, $AzureLocalInstanceLocation, [switch]$UseConsoleCredentials, [switch]$NoWait) +param($SubscriptionId, $ResourceGroupName, $Location, $VmSize, $AzureLocalResourceProviderObjectId, $AzureLocalInstanceLocation, [switch]$UseConsoleCredentials, [switch]$NoWait) Submit-TestLocalBox @PSBoundParameters '@ | Set-Content "$fixture/deploy-localbox.ps1" function Update-MhhToken {} @@ -30,13 +30,15 @@ Submit-TestLocalBox @PSBoundParameters function Invoke-LocalBoxHostedAutomation { param($SubscriptionId, $ResourceGroupName, $AksAdminGroupObjectId, $Package) } function New-LocalBoxAutomationPackage {} function Submit-TestLocalBox { - param($SubscriptionId, $ResourceGroupName, $Location, $AzureLocalResourceProviderObjectId, $AzureLocalInstanceLocation, [switch]$UseConsoleCredentials, [switch]$NoWait) + param($SubscriptionId, $ResourceGroupName, $Location, $VmSize, $AzureLocalResourceProviderObjectId, $AzureLocalInstanceLocation, [switch]$UseConsoleCredentials, [switch]$NoWait) } function az {} . "$root/labautomation/quota-helpers.ps1" function Get-AzComputeResourceSku { param($Location) } function Get-AzVMUsage { param($Location) } function Get-MhhStableHash { param($Value, $Length) '123456781234567812345678' } + . "$root/labautomation/localbox-credentials.ps1" + . "$root/labautomation/localbox-automation.ps1" } Describe 'Global LocalBox Console contract' { @@ -83,7 +85,7 @@ Describe 'Event-wide LocalBox orchestration' { Mock Initialize-MhhHostedDefender { $state.Events.Add("defender:$SubscriptionId") } Mock Get-AzResourceGroup {} Mock Get-AzResourceGroupDeployment { @{ DeploymentName = 'localbox-existing'; ProvisioningState = 'Succeeded'; Timestamp = Get-Date } } - Mock Get-AzResource { @{ Name = 'LocalBox-Client'; ResourceId = "$scope/providers/Microsoft.Compute/virtualMachines/LocalBox-Client" } } + Mock Get-AzResource {} Mock Update-AzTag {} Mock Get-AzResourceProvider { @{ RegistrationState = 'Registered' } } Mock Get-AzRoleAssignment {} @@ -141,7 +143,7 @@ Describe 'Event-wide LocalBox orchestration' { @{ ProvisioningState = 'Running' }; @{ ProvisioningState = 'Succeeded' } ) { param($ProvisioningState) - Mock Get-AzResourceGroup { @{ ResourceId = $scope } } + Mock Get-AzResourceGroup { @{ ResourceId = $scope; ResourceGroupName = 'rg-localbox-shared' } } Mock Get-AzResourceGroupDeployment { @{ DeploymentName = 'localbox-existing'; ProvisioningState = $ProvisioningState } } Mock Get-AzRoleAssignment { @{ RoleAssignmentId = 'existing' } } & "$fixture/global-event-deploy.ps1" -SubscriptionId $state.Subscription -PreferredLocation spaincentral -AllowedEntraUserIds $users | Out-Null @@ -152,9 +154,69 @@ Describe 'Event-wide LocalBox orchestration' { It 'tries the next host region without changing the global subscription' { Mock Submit-TestLocalBox { throw 'SkuNotAvailable' } -ParameterFilter { $Location -eq 'spaincentral' } & "$fixture/global-event-deploy.ps1" -SubscriptionId $state.Subscription -PreferredLocation @('spaincentral, swedencentral') -AllowedEntraUserIds $users | Out-Null - Should -Invoke Submit-TestLocalBox -Times 2 -Exactly + Should -Invoke Submit-TestLocalBox -Times 3 -Exactly Should -Invoke Submit-TestLocalBox -Times 1 -ParameterFilter { $Location -eq 'swedencentral' -and $SubscriptionId -eq $state.Subscription } } + It 'falls back to v5 after asynchronous capacity failure and only then runs guest automation' { + Mock Submit-TestLocalBox { + $state.Events.Add("submit:$VmSize") + @{ DeploymentName = "localbox-$VmSize"; ProvisioningState = 'Submitted' } + } + Mock Wait-LocalBoxDeployment { + $state.Events.Add("wait:$DeploymentName") + if ($DeploymentName -eq 'localbox-Standard_E32s_v6') { throw 'SkuNotAvailable' } + } + & "$fixture/global-event-deploy.ps1" -SubscriptionId $state.Subscription -PreferredLocation spaincentral -AllowedEntraUserIds $users | Out-Null + Should -Invoke Submit-TestLocalBox -Times 2 -Exactly + Should -Invoke Submit-TestLocalBox -Times 1 -Exactly -ParameterFilter { $VmSize -eq 'Standard_E32s_v5' -and $Location -eq 'spaincentral' } + $state.Events[-1] | Should -Be 'automation' + Should -Invoke Invoke-LocalBoxHostedAutomation -Times 1 -Exactly + } + It 'does not move partial regional infrastructure even when preferences are reordered' { + Mock Get-AzResourceGroup { @{ ResourceId = $scope; ResourceGroupName = 'rg-localbox-shared' } } + Mock Get-AzResourceGroupDeployment { @{ DeploymentName = 'localbox-failed'; ProvisioningState = 'Failed' } } + Mock Get-AzResource { @{ Name = 'LocalBox-VNet'; ResourceType = 'Microsoft.Network/virtualNetworks'; Location = 'spaincentral' } } + & "$fixture/global-event-deploy.ps1" -SubscriptionId $state.Subscription -PreferredLocation @('swedencentral', 'spaincentral') -AllowedEntraUserIds $users | Out-Null + Should -Invoke Submit-TestLocalBox -Times 0 -Exactly -ParameterFilter { $Location -eq 'swedencentral' } + Should -Invoke Submit-TestLocalBox -Times 1 -Exactly -ParameterFilter { $Location -eq 'spaincentral' } + } + It 'fails explicitly when both sizes fail and another region contains incompatible resources' { + Mock Get-AzResourceGroup { @{ ResourceId = $scope; ResourceGroupName = 'rg-localbox-shared' } } + Mock Get-AzResourceGroupDeployment { @{ DeploymentName = 'localbox-failed'; ProvisioningState = 'Failed' } } + Mock Get-AzResource { @{ ResourceType = 'Microsoft.Network/virtualNetworks'; Location = 'spaincentral' } } + Mock Submit-TestLocalBox { throw 'SkuNotAvailable' } + { & "$fixture/global-event-deploy.ps1" -SubscriptionId $state.Subscription -PreferredLocation @('spaincentral', 'swedencentral') -AllowedEntraUserIds $users } | + Should -Throw '*capacity candidates exhausted*No resources were deleted*' + Should -Invoke Submit-TestLocalBox -Times 2 -Exactly + Should -Invoke Invoke-LocalBoxHostedAutomation -Times 0 + } + It 'rechecks inventory after failure and never changes size when a VM appeared' { + Mock Get-AzResourceGroup { @{ ResourceId = $scope; ResourceGroupName = 'rg-localbox-shared' } } + Mock Get-AzResourceGroupDeployment { @{ DeploymentName = 'localbox-failed'; ProvisioningState = 'Failed' } } + Mock Get-AzResource { + if ($state.Created) { @{ ResourceType = 'Microsoft.Compute/virtualMachines'; Location = 'spaincentral' } } + } + Mock Submit-TestLocalBox { $state.Created = $true; @{ DeploymentName = 'localbox-new'; ProvisioningState = 'Submitted' } } + Mock Wait-LocalBoxDeployment { throw 'SkuNotAvailable' } + { & "$fixture/global-event-deploy.ps1" -SubscriptionId $state.Subscription -PreferredLocation spaincentral -AllowedEntraUserIds $users } | + Should -Throw '*VM already exists*' + Should -Invoke Submit-TestLocalBox -Times 1 -Exactly + Should -Invoke Invoke-LocalBoxHostedAutomation -Times 0 + } + It 'handles capacity failure while resuming an active deployment without skipping the wait' { + Mock Get-AzResourceGroup { @{ ResourceId = $scope; ResourceGroupName = 'rg-localbox-shared' } } + Mock Get-AzResourceGroupDeployment { @{ DeploymentName = 'localbox-existing'; ProvisioningState = 'Running' } } + Mock Wait-LocalBoxDeployment { if ($DeploymentName -eq 'localbox-existing') { throw 'SkuNotAvailable' } } + & "$fixture/global-event-deploy.ps1" -SubscriptionId $state.Subscription -PreferredLocation spaincentral -AllowedEntraUserIds $users | Out-Null + Should -Invoke Wait-LocalBoxDeployment -Times 2 -Exactly + Should -Invoke Submit-TestLocalBox -Times 1 -Exactly + } + It 'does not convert scope discovery errors into an empty resource group' { + Mock Get-AzResourceGroup { throw 'AuthorizationFailed' } + { & "$fixture/global-event-deploy.ps1" -SubscriptionId $state.Subscription -PreferredLocation spaincentral -AllowedEntraUserIds $users } | + Should -Throw '*AuthorizationFailed*' + Should -Invoke Submit-TestLocalBox -Times 0 + } It 'stops without publishing discovery credentials when fails' -TestCases @( @{ Stage = 'tags' }; @{ Stage = 'defender' }; @{ Stage = 'roles' }; @{ Stage = 'wait' }; @{ Stage = 'deploy' }; @{ Stage = 'graph' }; @{ Stage = 'automation' }; @{ Stage = 'package' } ) { diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/hosted-tag-policy.tests.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/hosted-tag-policy.tests.ps1 index 53d03d126..90c85aef6 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/hosted-tag-policy.tests.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/hosted-tag-policy.tests.ps1 @@ -4,6 +4,7 @@ BeforeAll { function Update-MhhToken {} function Get-AzContext { [CmdletBinding()] param() } + function Invoke-AzRestMethod { [CmdletBinding()] param($Method, $Path) } function New-AzSubscriptionDeployment { [CmdletBinding()] param($Name, $Location, $TemplateFile) } function New-AzResourceGroup { [CmdletBinding()] param($Name, $Location, $Tag, [switch]$Force) } function New-AzNetworkSecurityGroup { [CmdletBinding()] param($Name, $ResourceGroupName, $Location, $Tag, [switch]$Force) } @@ -19,6 +20,7 @@ Describe 'Hosted control-tag setup' { $script:resourceTags = $script:groupTags.Clone() Mock Update-MhhToken {} Mock Get-AzContext { @{ Subscription = @{ Id = 'hosted-subscription' } } } + Mock Invoke-AzRestMethod { @{ StatusCode = 404 } } Mock New-AzSubscriptionDeployment { @{ ProvisioningState = 'Succeeded' } } Mock New-AzResourceGroup { $script:probeGroupName = $Name } Mock New-AzNetworkSecurityGroup {} @@ -58,12 +60,36 @@ Describe 'Hosted control-tag setup' { if ($script:reads -eq 1) { return [pscustomobject]@{ Tag = @{} } } return [pscustomobject]@{ Tag = $script:resourceTags } } + Initialize-MhhHostedTagPolicy -SubscriptionId 'hosted-subscription' -Location 'swedencentral' Should -Invoke New-AzNetworkSecurityGroup -Times 2 -Exactly Should -Invoke Start-Sleep -Times 1 -Exactly -ParameterFilter { $Seconds -eq 10 } Should -Invoke Remove-AzResourceGroup -Times 1 -Exactly } + It 'preserves an existing policy identity location when host preferences change' { + Mock Invoke-AzRestMethod { @{ StatusCode = 200; Content = '{"location":"spaincentral"}' } } + Initialize-MhhHostedTagPolicy -SubscriptionId 'hosted-subscription' -Location 'swedencentral' + Should -Invoke Invoke-AzRestMethod -Times 1 -Exactly -ParameterFilter { + $Method -eq 'GET' -and $Path -eq '/subscriptions/hosted-subscription/providers/Microsoft.Authorization/policyAssignments/sov-hosted-control-tags?api-version=2026-06-01' + } + Should -Invoke New-AzSubscriptionDeployment -Times 1 -Exactly -ParameterFilter { $Location -eq 'spaincentral' } + Should -Invoke New-AzResourceGroup -Times 1 -Exactly -ParameterFilter { $Location -eq 'spaincentral' } + } + + It 'stops on policy lookup failure or missing location instead of recreating the identity' -TestCases @( + @{ Status = 403; Content = '{}' } + @{ Status = 500; Content = '{}' } + @{ Status = 200; Content = '{}' } + @{ Status = 200; Content = 'invalid' } + ) { + param($Status, $Content) + Mock Invoke-AzRestMethod { @{ StatusCode = $Status; Content = $Content } } + { Initialize-MhhHostedTagPolicy -SubscriptionId 'hosted-subscription' -Location 'swedencentral' } | Should -Throw + Should -Invoke New-AzSubscriptionDeployment -Times 0 + Should -Invoke New-AzResourceGroup -Times 0 + } + It 'fails closed for missing or incorrect tags and cleans up' -TestCases @( @{ Target = 'group'; Key = 'SecurityControl'; Value = $null } @{ Target = 'group'; Key = 'CostControl'; Value = 'Enforce' } diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-arm-recovery.tests.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-arm-recovery.tests.ps1 new file mode 100644 index 000000000..67099bbaa --- /dev/null +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-arm-recovery.tests.ps1 @@ -0,0 +1,146 @@ +BeforeAll { + . "$PSScriptRoot/../../labautomation/localbox-credentials.ps1" + . "$PSScriptRoot/../../labautomation/localbox-automation.ps1" + function Update-MhhToken {} + function Get-AzContext { [CmdletBinding()] param() } + function Get-AzResourceGroupDeployment { [CmdletBinding()] param($ResourceGroupName, $Name) } + function Get-AzResourceGroupDeploymentOperation { [CmdletBinding()] param($ResourceGroupName, $DeploymentName) } +} + +Describe 'LocalBox asynchronous ARM failures' { + BeforeEach { + Mock Update-MhhToken {} + Mock Get-AzContext { @{ Subscription = @{ Id = 'test' } } } + Mock Get-AzResourceGroupDeployment { @{ ProvisioningState = 'Failed' } } + $state = @{ + Message = 'VM size Standard_E32s_v6 has Capacity Restrictions in SpainCentral' + Codes = @('SkuNotAvailable') + Serialize = $false + } + Mock Get-AzResourceGroupDeploymentOperation { + $status = @{ error = @{ code = 'InvalidTemplateDeployment'; details = @( + foreach ($code in $state.Codes) { @{ code = $code; message = $state.Message } } + ) } } + if ($state.Serialize) { $status = $status | ConvertTo-Json -Depth 10 } + @{ Properties = @{ ProvisioningState = 'Failed'; StatusMessage = $status } } + } + } + It 'reports nested capacity details and classifies both object and JSON-string status messages' -TestCases @( + @{ Serialize = $false }; @{ Serialize = $true } + ) { + param($Serialize) + $state.Serialize = $Serialize + $caught = $null + try { Wait-LocalBoxDeployment -ResourceGroupName 'shared' -DeploymentName 'localbox-test' } + catch { $caught = $_ } + $caught | Should -Not -BeNullOrEmpty + $caught.Exception.Message | Should -BeLike '*ended as Failed*[SkuNotAvailable]*Standard_E32s_v6*SpainCentral*' + Test-LocalBoxCapacityFailure $caught | Should -BeTrue + } + It 'does not retry mixed policy, unknown or authorization failures' -TestCases @( + @{ Code = 'RequestDisallowedByPolicy' } + @{ Code = 'AuthorizationFailed' } + @{ Code = 'OperationNotAllowed' } + @{ Code = '' } + ) { + param($Code) + $state.Codes = @('SkuNotAvailable', $Code) + $caught = $null + try { Wait-LocalBoxDeployment -ResourceGroupName 'shared' -DeploymentName 'localbox-test' } + catch { $caught = $_ } + Test-LocalBoxCapacityFailure $caught | Should -BeFalse + } + It 'fails closed on unavailable diagnostics and reports the read error' { + Mock Get-AzResourceGroupDeploymentOperation { throw 'AuthorizationFailed reading operations' } + $caught = $null + try { Wait-LocalBoxDeployment -ResourceGroupName 'shared' -DeploymentName 'localbox-test' } + catch { $caught = $_ } + $caught.Exception.Message | Should -BeLike '*Could not retrieve*AuthorizationFailed*' + Test-LocalBoxCapacityFailure $caught | Should -BeFalse + } + It 'never retries canceled deployments' { + Mock Get-AzResourceGroupDeployment { @{ ProvisioningState = 'Canceled' } } + $caught = $null + try { Wait-LocalBoxDeployment -ResourceGroupName 'shared' -DeploymentName 'localbox-test' } + catch { $caught = $_ } + Test-LocalBoxCapacityFailure $caught | Should -BeFalse + Should -Invoke Get-AzResourceGroupDeploymentOperation -Times 0 + } + It 'follows a same-group nested deployment when the parent only reports a wrapper error' { + Mock Get-AzResourceGroupDeploymentOperation { + @{ Properties = @{ + ProvisioningState = 'Failed' + StatusMessage = @{ error = @{ code = 'DeploymentFailed'; message = 'See operations' } } + TargetResource = @{ + ResourceType = 'Microsoft.Resources/deployments'; ResourceName = 'hostVmDeployment' + Id = '/subscriptions/test/resourceGroups/shared/providers/Microsoft.Resources/deployments/hostVmDeployment' + } + } } + } -ParameterFilter { $DeploymentName -eq 'localbox-test' } + $caught = $null + try { Wait-LocalBoxDeployment -ResourceGroupName 'shared' -DeploymentName 'localbox-test' } + catch { $caught = $_ } + Test-LocalBoxCapacityFailure $caught | Should -BeTrue + Should -Invoke Get-AzResourceGroupDeploymentOperation -Times 1 -Exactly -ParameterFilter { $DeploymentName -eq 'hostVmDeployment' } + } + It 'redacts sensitive values from deployment error messages' { + $state.Message = 'Capacity unavailable. password=private-fixture-value https://example.invalid/?sig=private-signature' + $caught = $null + try { Wait-LocalBoxDeployment -ResourceGroupName 'shared' -DeploymentName 'localbox-test' } + catch { $caught = $_ } + $caught.Exception.Message | Should -Not -Match 'private-fixture-value|private-signature' + } + It 'fails closed on empty or malformed operation diagnostics' -TestCases @( + @{ Malformed = $false }; @{ Malformed = $true } + ) { + param($Malformed) + Mock Get-AzResourceGroupDeploymentOperation { + if ($Malformed) { @{ Properties = @{ ProvisioningState = 'Failed'; StatusMessage = 'not-json' } } } + } + $caught = $null + try { Wait-LocalBoxDeployment -ResourceGroupName 'shared' -DeploymentName 'localbox-test' } + catch { $caught = $_ } + $caught | Should -Not -BeNullOrEmpty + Test-LocalBoxCapacityFailure $caught | Should -BeFalse + } + It 'bounds nested deployment traversal and does not infer a capacity failure' { + Mock Get-AzResourceGroupDeploymentOperation { + @{ Properties = @{ + ProvisioningState = 'Failed' + StatusMessage = @{ error = @{ code = 'DeploymentFailed' } } + TargetResource = @{ + ResourceType = 'Microsoft.Resources/deployments'; ResourceName = 'loop' + Id = '/subscriptions/test/resourceGroups/shared/providers/Microsoft.Resources/deployments/loop' + } + } } + } + $caught = $null + try { Wait-LocalBoxDeployment -ResourceGroupName 'shared' -DeploymentName 'localbox-test' } + catch { $caught = $_ } + Test-LocalBoxCapacityFailure $caught | Should -BeFalse + Should -Invoke Get-AzResourceGroupDeploymentOperation -Times 6 -Exactly + } + It 'does not follow a nested target outside the original subscription or group' -TestCases @( + @{ Target = '/subscriptions/other/resourceGroups/shared/providers/Microsoft.Resources/deployments/host' } + @{ Target = '/subscriptions/test/resourceGroups/other/providers/Microsoft.Resources/deployments/host' } + ) { + param($Target) + Mock Get-AzResourceGroupDeploymentOperation { + @{ Properties = @{ + ProvisioningState = 'Failed'; StatusMessage = @{ error = @{ code = 'DeploymentFailed' } } + TargetResource = @{ ResourceType = 'Microsoft.Resources/deployments'; ResourceName = 'host'; Id = $Target } + } } + } + $caught = $null + try { Wait-LocalBoxDeployment -ResourceGroupName 'shared' -DeploymentName 'localbox-test' } + catch { $caught = $_ } + Test-LocalBoxCapacityFailure $caught | Should -BeFalse + Should -Invoke Get-AzResourceGroupDeploymentOperation -Times 1 -Exactly + } + It 'does not treat an authorization failure mentioning a capacity error as retryable' { + $caught = $null + try { throw 'AuthorizationFailed while looking up SkuNotAvailable' } + catch { $caught = $_ } + Test-LocalBoxCapacityFailure $caught | Should -BeFalse + } +} diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-deployment.tests.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-deployment.tests.ps1 index f34959bf3..74f656c61 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-deployment.tests.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/localbox-deployment.tests.ps1 @@ -62,6 +62,10 @@ Describe 'LocalBox deployment entry points' { StartupTagUpdate = @() QueriesFail = $false ValidationFails = $false + Restrictions = @() + MissingSku = $false + FamilyQuota = 64 + RegionalQuota = 64 Submitted = 0 Builds = 0 Parameters = $null @@ -101,8 +105,22 @@ var settings = { '^provider show' { 'Registered'; break } '^feature show' { 'Registered'; break } '^provider register' { '{}'; break } - '^vm list-sizes' { 'Standard_E32s_v6'; break } - '^vm list-usage' { '[{"name":{"localizedValue":"Standard Esv6 Family vCPUs"},"limit":64,"currentValue":0}]'; break } + '^vm list-skus' { + if ($deploymentTestState.MissingSku) { '[]' } + else { + $size = $args[[array]::IndexOf($args, '--size') + 1] + ConvertTo-Json -InputObject @(@{ name = $size; restrictions = $deploymentTestState.Restrictions }) -Depth 10 + } + break + } + '^vm list-usage' { + ConvertTo-Json -InputObject @( + @{ name = @{ value = 'standardEsv6Family' }; limit = $deploymentTestState.FamilyQuota; currentValue = 0 } + @{ name = @{ value = 'standardESv5Family' }; limit = $deploymentTestState.FamilyQuota; currentValue = 0 } + @{ name = @{ value = 'cores' }; limit = $deploymentTestState.RegionalQuota; currentValue = 0 } + ) + break + } '^ad sp list' { '[{"id":"20000000-0000-0000-0000-000000000001"}]'; break } '^group exists' { $deploymentTestState.GroupExists.ToString().ToLowerInvariant(); break } '^network bastion list' { @@ -145,6 +163,34 @@ var settings = { $deploymentTestState.Parameters.tags.value.PSObject.Properties.Name | Should -Not -Contain 'microhack-startup-id' $deploymentTestState.StartupTagUpdate.Count | Should -Be 0 } + It 'rejects a region-restricted or missing SKU before creating infrastructure' -TestCases @( + @{ Missing = $true; Restrictions = @() } + @{ Missing = $false; Restrictions = @(@{ type = 'Location'; reasonCode = 'NotAvailableForSubscription' }) } + ) { + param($Missing, $Restrictions) + $deploymentTestState.MissingSku = $Missing + $deploymentTestState.Restrictions = $Restrictions + { & "$fixture/labautomation/deploy-localbox.ps1" @arguments } | Should -Throw '*VM size*unavailable*' + $deploymentTestState.Submitted | Should -Be 0 + $deploymentTestState.Builds | Should -Be 0 + } + It 'permits zone-only restrictions for a non-zonal v5 deployment' { + $deploymentTestState.Restrictions = @(@{ type = 'Zone' }) + $arguments.VmSize = 'Standard_E32s_v5' + & "$fixture/labautomation/deploy-localbox.ps1" @arguments + $deploymentTestState.Parameters.vmSize.value | Should -Be 'Standard_E32s_v5' + $deploymentTestState.Submitted | Should -Be 1 + } + It 'requires both regional and family headroom' -TestCases @( + @{ Family = 31; Regional = 64 } + @{ Family = 64; Regional = 31 } + ) { + param($Family, $Regional) + $deploymentTestState.FamilyQuota = $Family + $deploymentTestState.RegionalQuota = $Regional + { & "$fixture/labautomation/deploy-localbox.ps1" @arguments } | Should -Throw '*At least 32 unused*' + $deploymentTestState.Submitted | Should -Be 0 + } It 'does not redeploy a guarded VM whose progress update removed identity tags' { $deploymentTestState.Clients = @(@{ name = 'LocalBox-Client'; tags = @{} }) $deploymentTestState.Monitors = @(@{ name = 'sovereign-localbox-startup-watch' }) diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/prepare-localbox.tests.ps1 b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/prepare-localbox.tests.ps1 index eef41bbcd..ac977addf 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/prepare-localbox.tests.ps1 +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/prepare-localbox.tests.ps1 @@ -1060,9 +1060,11 @@ Describe 'Console lab group credentials' { Describe 'Console LocalBox credential isolation' { BeforeAll { + . "$PSScriptRoot/../../labautomation/localbox-automation.ps1" function New-MhhStablePassword { [CmdletBinding()] param($Purpose, $Length) } function Update-MhhToken { [CmdletBinding()] param() } function Get-AzResourceGroupDeployment { [CmdletBinding()] param($ResourceGroupName, $Name) } + function Get-AzResourceGroupDeploymentOperation { [CmdletBinding()] param($ResourceGroupName, $DeploymentName) } } BeforeEach { $deployment = @{ @@ -1070,6 +1072,7 @@ Describe 'Console LocalBox credential isolation' { } Mock Update-MhhToken {} Mock Get-AzResourceGroupDeployment { $deployment } + Mock Get-AzResourceGroupDeploymentOperation { @() } Mock New-MhhStablePassword { 'mock-password-not-a-secret' } Mock Start-Sleep {} } @@ -1174,6 +1177,12 @@ Describe 'LocalBox registration region contract' { Describe 'Console LocalBox CLI failure propagation' { BeforeAll { + . "$PSScriptRoot/../../labautomation/localbox-credentials.ps1" + . "$PSScriptRoot/../../labautomation/localbox-automation.ps1" + function Update-MhhToken {} + function Get-AzResourceGroup { [CmdletBinding()] param() } + function Get-AzResource { [CmdletBinding()] param($ResourceGroupName) } + function Wait-LocalBoxDeployment { param($ResourceGroupName, $DeploymentName) } $deployer = [Management.Automation.Language.Parser]::ParseFile( "$PSScriptRoot/../../labautomation/deploy-localbox.ps1", [ref]$null, [ref]$null) $helper = $deployer.Find({ @@ -1235,11 +1244,18 @@ Invoke-AzJson @('deployment', 'group', 'validate', '--location', $Location) | Ou [pscustomobject]@{ ProvisioningState = 'Submitted'; Location = $Location } '@ | Set-Content (Join-Path $TestDrive 'deploy-localbox.ps1') $localBoxLocations = @('first-region', 'second-region') + $localBoxResourceGroupName = 'shared' + $candidateFailures = [Collections.Generic.List[string]]::new() $azureLocalResourceProviderObjectIds = @('test-object-id') $localBoxDeployment = $null + $cliState.Command = '[Console]::Error.WriteLine("SkuNotAvailable: capacity restriction"); exit 1' + Mock Update-MhhToken {} + Mock Get-AzResourceGroup {} + Mock Get-AzResource {} + Mock Wait-LocalBoxDeployment {} Mock az { $cliState.Attempts++ - if ($cliState.Attempts -eq 2) { + if ($cliState.Attempts -eq 3) { & $nativeExecutable -NoProfile -NonInteractive -Command 'exit 0' } else { @@ -1247,25 +1263,29 @@ Invoke-AzJson @('deployment', 'group', 'validate', '--location', $Location) | Ou } } } - It 'tries the next region after the actual native regional failure' { + It 'tries both sizes then the next region after the actual native capacity failure' { . $fallback $localBoxDeployment.ProvisioningState | Should -Be 'Submitted' $localBoxDeployment.Location | Should -Be 'second-region' - Should -Invoke az -Times 2 -Exactly + Should -Invoke az -Times 3 -Exactly } It 'stops without retrying ' -ForEach @( @{ Code = 'RequestDisallowedByPolicy' } @{ Code = 'AuthorizationFailed' } @{ Code = 'UnexpectedFailure' } + @{ Code = 'RequestDisallowedByAzure' } ) { $cliState.Command = "[Console]::Error.WriteLine('$Code'); exit 1" { . $fallback } | Should -Throw "*$Code*" Should -Invoke az -Times 1 -Exactly } - It 'preserves the regional failure when no fallback regions remain' { + It 'retains native failure details when no capacity candidates remain' { $localBoxLocations = @('first-region') - { . $fallback } | Should -Throw '*RequestDisallowedByAzure*locationineligible*' - Should -Invoke az -Times 1 -Exactly + . $fallback + $localBoxDeployment | Should -BeNullOrEmpty + $candidateFailures.Count | Should -Be 2 + $candidateFailures -join '; ' | Should -Match 'SkuNotAvailable' + Should -Invoke az -Times 2 -Exactly } } } diff --git a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md index eb8b2ed52..4fb2b6cdb 100644 --- a/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md +++ b/03-Azure/01-03-Infrastructure/01_Sovereign_Cloud/resources/tests/readme.md @@ -131,6 +131,30 @@ event-specific check of service availability and effective inherited policies. ## Global event LocalBox checks +### ARM capacity and safe retries + +Run `Invoke-Pester ./localbox-arm-recovery.tests.ps1,./global-localbox.tests.ps1,./localbox-deployment.tests.ps1,./hosted-tag-policy.tests.ps1 -Output Detailed`. +These offline regressions cover the TEST16 asynchronous `SkuNotAvailable` path: +nested operation diagnostics, redaction, bounded traversal, mixed/unknown/read +failures, canceled deployments, completion before guest automation, E32s_v6 to +E32s_v5 fallback, inventory rechecks, existing-VM protection and refusal to move +partial regional resources. SKU restrictions and both family/regional headroom +are checked before submission. Hosted policy tests verify preservation of an +existing identity's location when host preferences change. + +The tests do not allocate capacity, delete resources or prove the alternate SKU +works in a live event. See [ARM recovery and the TEST16 status](../hosted-events/readme.md#localbox-arm-capacity-failures); +a new Console run remains the integration gate. + +On 10 November 2026, the combined ARM-recovery, global LocalBox, deployment, +preparation, hosted-tag-policy, hosted-Defender, shared-AKS and quota-reuse suites +passed **563 tests**, with none failed or skipped. A separate GET-only Azure CLI +adapter fed the actual TEST16 deployment/operation responses into the new wait +helper and verified the nested `SkuNotAvailable` classification and diagnostic +message. This read-only check did not deploy, retry or resize a VM. + +### Startup and guest automation + Run `Invoke-Pester ./localbox-winget.tests.ps1,./localbox-startup.tests.ps1,./localbox-deployment.tests.ps1 -Output Detailed` for WinGet provisioning failures. These checks cover bounded HTTP 504 retries, CLI/module version mismatch, COM activation failure, nonzero/missing native exit