From 51a08eb82e6c2a5b72a30cc94290f7cb3feff4eb Mon Sep 17 00:00:00 2001 From: Jan Egil Ring Date: Wed, 30 Sep 2026 19:28:39 +0000 Subject: [PATCH] feat: Enhance lab automation with region fallback and public IP feature registration Signed-off-by: Jan Egil Ring --- .../04-adaptive-apps/labautomation/README.md | 57 +++++--- .../labautomation/deploy-lab.ps1 | 37 +++-- .../labautomation/lab-defaults.json | 2 +- .../labautomation/shared-deploy-lab.ps1 | 31 +++- .../labautomation/tests/console.Tests.ps1 | 137 ++++++++++++++++-- 5 files changed, 216 insertions(+), 48 deletions(-) diff --git a/03-Azure/01-01-App Innovation/04-adaptive-apps/labautomation/README.md b/03-Azure/01-01-App Innovation/04-adaptive-apps/labautomation/README.md index c5853166e..b99ef2391 100644 --- a/03-Azure/01-01-App Innovation/04-adaptive-apps/labautomation/README.md +++ b/03-Azure/01-01-App Innovation/04-adaptive-apps/labautomation/README.md @@ -16,8 +16,9 @@ region, and participant object IDs. The participant already has Owner on that group. There is no interactive login, Graph application/service-principal creation, confidential computing, Azure Local, or shared participant Kubernetes cluster here. -Default scope: **resourcegroup**, **five labs per subscription**, in **Sweden Central** -(`swedencentral`). Organizers may override the event region before creating lab +Default scope: **resourcegroup**, **five labs per subscription**, with **Sweden Central** +first and **Spain Central** as fallback (`swedencentral,spaincentral`). +Organizers may override the ordered regions before creating lab scopes; the shared preflight must pass for that subscription and region. Every participant gets a dedicated two-node AKS cluster and a separate private K3s VM. The organizer needs a **Linux runner**, PowerShell 7, current Az.Accounts/Az.Resources/Az.Network, @@ -60,7 +61,11 @@ prerequisites. No GitHub or AI API credentials are provisioned by these hooks. `shared-deploy-lab.ps1` runs once per subscription: 1. Verify both authenticated contexts target the supplied subscription. -2. Register and wait for Compute, Network, ContainerService, ContainerRegistry, +2. Register `Microsoft.Network/AllowBringYourOwnPublicIpAddress`, as required + by these hosted workshop subscriptions for public IP allocation (also used by + LocalBox). Wait up to 15 minutes for `Registered`; pending approval, failure, + or timeout blocks participant provisioning. Then re-register Network so the + feature takes effect. Register and wait for Compute, Network, ContainerService, ContainerRegistry, ManagedIdentity, DBforPostgreSQL, OperationalInsights, PolicyInsights, Sql, KeyVault and Storage. SQL supports the published SQL recipe; Key Vault and Storage preserve the manual challenge 01 subscription prerequisites. @@ -89,14 +94,27 @@ regional service limits, policy and actual SKU capacity can still fail deploymen The stage fails explicitly rather than silently choosing a different VM size. Quota failure messages identify the required capacity and region. -Console creates participant groups in the **first preferred region**. The -PostgreSQL recipe defaults to `resourceGroup().location`, so the shared stage -requires that first region to pass even if an alternative is eligible. The -participant hook deploys everything in the **supplied RG's actual region**, and -rejects any region absent from the shared metadata. To use an alternative, -reconfigure the event location and explicitly recreate empty scopes through -Console; never move resource deployment independently of the RG metadata. -There is no destructive region fallback, automatic RG deletion or AKS recreation. +Console creates participant groups in the **first preferred region**. Shared +preflight retains eligible regions in preference order and fails if none pass. +For **initial infrastructure provisioning only**, the participant hook uses +`Invoke-MhhDeploymentWithRegionFallback` with those validated regions. The helper +classifies deployment failures, retries supported capacity/region failures, and +fails immediately for fatal errors rather than treating all errors as retryable. +It can **delete the participant RG and all its contents**, retry the same region, +or recreate it in the next region. Participant Owner access is restored on every +attempt. Use initial provisioning only in dedicated, disposable lab groups. + +After infrastructure succeeds, `adaptiveAppsPreserve=true` prevents subsequent +runs from invoking destructive fallback, including when Bash bootstrap failed. +Existing labs with `adaptiveAppsReady=true` also receive this persistent guard +before readiness is cleared. Do not remove the preservation tag to troubleshoot +a working lab. Such labs receive incremental deployment in place; if their region +fails preflight, the hook stops rather than relocating them. + +The parameter file deliberately omits `location`, letting the template follow +`resourceGroup().location`. After fallback the hook re-reads the RG location for +all bootstrap phases and credentials, keeping the PostgreSQL recipe in the same +region. These checks do not guarantee AKS capacity or region eligibility. ## Participant infrastructure and execution @@ -131,8 +149,9 @@ record its value in deployment history. Do not print or publish the parameter file. Stage messages identify context validation, parameter preparation and deployment submission without including credentials. -The participant hook uses an **incremental deployment**, retaining resources and -failed deployment records for diagnosis. It grants each supplied participant +The participant hook uses an **incremental deployment**; initial-provisioning +fallback may recycle failed attempts as described above. Protected labs retain +resources and failed deployment records for diagnosis. It grants each supplied participant **Reader on this AKS node RG only**, so the PostgreSQL recipe can discover AKS egress IPs. Role operations use object IDs and explicitly disable Graph name resolution. No subscription-wide participant role is added. @@ -267,12 +286,12 @@ the managed-cluster API requires the profile when `mode` is `Istio`, and Azure selects its default supported revision when no revision is specified. If a live deployment reports `AKSCapacityHeavyUsage`, quota/SKU validation has -not reserved AKS capacity. Resolve the capacity issue or explicitly select another -event region before retrying. A `SubscriptionNotRegisteredForFeature` error -mentioning `AllowBringYourOwnPublicIpAddress` on these ordinary Standard public -IPs needs subscription/provider investigation; the template does not request -BYOIP or a custom IP prefix. Do not enable an unrelated preview feature merely -to suppress that error. Preserve the deployment operation errors for support. +not reserved AKS capacity. Initial provisioning can use the validated fallback; +protected labs require organizer investigation. If +`SubscriptionNotRegisteredForFeature` still mentions +`AllowBringYourOwnPublicIpAddress`, verify shared setup completed both feature +registration and subsequent Network provider registration. The template itself +does not request BYOIP or a custom IP prefix. ## Hosted policy provenance diff --git a/03-Azure/01-01-App Innovation/04-adaptive-apps/labautomation/deploy-lab.ps1 b/03-Azure/01-01-App Innovation/04-adaptive-apps/labautomation/deploy-lab.ps1 index facdd7612..57fe879f3 100644 --- a/03-Azure/01-01-App Innovation/04-adaptive-apps/labautomation/deploy-lab.ps1 +++ b/03-Azure/01-01-App Innovation/04-adaptive-apps/labautomation/deploy-lab.ps1 @@ -39,14 +39,19 @@ try { $group = Get-AzResourceGroup -Name $ResourceGroupName -ErrorAction Stop $region = $group.Location.ToLowerInvariant() $metadata = (Get-AzTag -ResourceId "/subscriptions/$SubscriptionId" -ErrorAction Stop).Properties.TagsProperty - if (-not $metadata -or $region -notin ($metadata['microhack-adaptive-regions'] -split ',')) { - throw "RG location '$region' was not validated by shared-deploy-lab.ps1. Run the shared hook for this region; do not relocate resources independently of their RG." + if (-not $metadata) { throw 'Shared region metadata is missing. Run shared-deploy-lab.ps1 first.' } + $regions = @($metadata['microhack-adaptive-regions'] -split ',' | Where-Object { $_ }) + $preserve = $group.Tags -and ($group.Tags['adaptiveAppsReady'] -eq 'true' -or $group.Tags['adaptiveAppsPreserve'] -eq 'true') + if ($regions.Count -eq 0 -or ($preserve -and $region -notin $regions)) { + throw "No validated region is available for this lab. Run shared-deploy-lab.ps1 for the lab's region; an established lab will not be relocated." } - Update-AzTag -ResourceId $group.ResourceId -Operation Merge -Tag @{ + $deploymentTags = @{ adaptiveAppsReady = 'false' SecurityControl = 'Ignore' CostControl = 'Ignore' - } -ErrorAction Stop | Out-Null + } + if ($preserve) { $deploymentTags.adaptiveAppsPreserve = 'true' } + Update-AzTag -ResourceId $group.ResourceId -Operation Merge -Tag $deploymentTags -ErrorAction Stop | Out-Null Write-Host 'Preparing private infrastructure deployment parameters...' $parameterFile = Join-Path $scratch 'main.parameters.json' @@ -54,7 +59,6 @@ try { '$schema' = 'https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#' contentVersion = '1.0.0.0' parameters = @{ - location = @{ value = $region } adminPassword = @{ value = (New-MhhStablePassword -Purpose 'adaptive-apps-k3s-admin' -Length 24) } } } | ConvertTo-Json -Depth 4 | Set-Content -LiteralPath $parameterFile -Encoding utf8 @@ -62,15 +66,30 @@ try { if ($LASTEXITCODE -ne 0) { throw 'Unable to secure infrastructure parameter file.' } Write-Host 'Submitting incremental Adaptive Apps infrastructure deployment...' try { - $deployment = New-AzResourceGroupDeployment -Name 'adaptive-apps-console' ` - -ResourceGroupName $ResourceGroupName -TemplateFile (Join-Path $PSScriptRoot 'main.bicep') ` - -TemplateParameterFile $parameterFile -Mode Incremental -ErrorAction Stop + if ($preserve) { + $deployment = New-AzResourceGroupDeployment -Name 'adaptive-apps-console' ` + -ResourceGroupName $ResourceGroupName -TemplateFile (Join-Path $PSScriptRoot 'main.bicep') ` + -TemplateParameterFile $parameterFile -Mode Incremental -ErrorAction Stop + } else { + Write-Warning "Initial provisioning may delete and recreate '$ResourceGroupName' on retryable infrastructure failures. Validated regions: $($regions -join ', ')." + $result = Invoke-MhhDeploymentWithRegionFallback -PreferredLocations $regions ` + -ResourceGroupName $ResourceGroupName -RgOwnerEntraObjectIds $AllowedEntraUserIds ` + -TemplateFile (Join-Path $PSScriptRoot 'main.bicep') -TemplateParameterFile $parameterFile ` + -DeploymentNamePrefix 'adaptive-apps-console' -Tag $deploymentTags -ErrorAction Stop + if (-not $result.Success) { throw 'Initial infrastructure fallback did not report success.' } + $deployment = $result.DeploymentResult + } } finally { Remove-Item -LiteralPath $parameterFile -Force } if ($deployment.ProvisioningState -ne 'Succeeded') { - throw "Infrastructure deployment failed: $($deployment.ProvisioningState). Resources are retained for diagnosis; no destructive fallback is attempted." + throw "Infrastructure deployment failed: $($deployment.ProvisioningState). Bootstrap will not run." } + $group = Get-AzResourceGroup -Name $ResourceGroupName -ErrorAction Stop + $region = $group.Location.ToLowerInvariant() + if ($region -notin $regions) { throw "Deployment used unvalidated region '$region'." } + # Keep this marker through failed reconciliations; readiness alone is not a deletion guard. + Update-AzTag -ResourceId $group.ResourceId -Operation Merge -Tag @{ adaptiveAppsPreserve = 'true' } -ErrorAction Stop | Out-Null $acrName = $deployment.Outputs.acrName.Value $nodeGroup = $deployment.Outputs.nodeResourceGroup.Value if (-not $acrName -or -not $nodeGroup) { throw 'Infrastructure deployment omitted ACR or AKS node resource group outputs.' } diff --git a/03-Azure/01-01-App Innovation/04-adaptive-apps/labautomation/lab-defaults.json b/03-Azure/01-01-App Innovation/04-adaptive-apps/labautomation/lab-defaults.json index 6e3c9149d..063d8cf32 100644 --- a/03-Azure/01-01-App Innovation/04-adaptive-apps/labautomation/lab-defaults.json +++ b/03-Azure/01-01-App Innovation/04-adaptive-apps/labautomation/lab-defaults.json @@ -2,7 +2,7 @@ "$schema": "https://raw.githubusercontent.com/microsoft/MicroHack/refs/heads/main/lab-defaults-schema.json", "deploymentType": "resourcegroup", "labsPerSubscription": 5, - "preferredLocation": "swedencentral", + "preferredLocation": "swedencentral,spaincentral", "estimatedDailyCostsUsd": 35.0, "estimatedSharedDeploymentDailyCostsUsd": 0.0 } diff --git a/03-Azure/01-01-App Innovation/04-adaptive-apps/labautomation/shared-deploy-lab.ps1 b/03-Azure/01-01-App Innovation/04-adaptive-apps/labautomation/shared-deploy-lab.ps1 index 836ab1876..cb649cdc6 100644 --- a/03-Azure/01-01-App Innovation/04-adaptive-apps/labautomation/shared-deploy-lab.ps1 +++ b/03-Azure/01-01-App Innovation/04-adaptive-apps/labautomation/shared-deploy-lab.ps1 @@ -18,6 +18,27 @@ $locations = @($PreferredLocation | ForEach-Object { $_ -split ',' } | ForEach-O if ($locations.Count -eq 0) { throw 'At least one preferred location is required.' } $labCount = @($AllowedEntraUserIds | Where-Object { $_ } | Select-Object -Unique).Count if ($labCount -eq 0) { $labCount = $defaults.labsPerSubscription } +$feature = 'AllowBringYourOwnPublicIpAddress' +$featureState = (Invoke-AdaptiveAz -Arguments @('feature', 'show', '--namespace', 'Microsoft.Network', '--name', $feature)).properties.state +if ($featureState -ne 'Registered') { + if ($featureState -notin @('Registering', 'Pending')) { + Invoke-AdaptiveAz -Arguments @('feature', 'register', '--namespace', 'Microsoft.Network', '--name', $feature) | Out-Null + } + for ($attempt = 0; $attempt -lt 90; $attempt++) { + Start-Sleep -Seconds 10 + Update-MhhToken | Out-Null + $featureState = (Invoke-AdaptiveAz -Arguments @('feature', 'show', '--namespace', 'Microsoft.Network', '--name', $feature)).properties.state + if ($featureState -eq 'Registered') { break } + if ($featureState -notin @('Registering', 'Pending')) { + throw "Microsoft.Network/$feature registration ended in state '$featureState'." + } + Write-Host "Waiting for Microsoft.Network/$feature registration ($($attempt + 1)/90): $featureState" + } +} +if ($featureState -ne 'Registered') { + throw "Microsoft.Network/$feature did not reach Registered within 15 minutes (state: '$featureState'). Resolve feature registration before retrying." +} +# Re-register Network after the feature is registered so public IP allocation sees it. $providers = @('Microsoft.Compute', 'Microsoft.Network', 'Microsoft.ContainerService', 'Microsoft.ContainerRegistry', 'Microsoft.ManagedIdentity', 'Microsoft.DBforPostgreSQL', 'Microsoft.OperationalInsights', 'Microsoft.PolicyInsights', 'Microsoft.Sql', @@ -48,14 +69,12 @@ foreach ($location in $locations) { Write-Warning $failures[-1] } } -# Console pre-creates RGs in the first preferred location. Recipes inherit that location. -# Do not report shared success merely because a different candidate passed. -if ($locations[0] -notin $ready) { - throw "The supplied participant RG region '$($locations[0])' is not ready. Eligible alternatives: $($ready -join ', '). Reconfigure the event location and recreate EMPTY lab scopes explicitly; no automatic relocation/deletion is performed. $($failures -join '; ')" +if ($ready.Count -eq 0) { + throw "No preferred region passed Adaptive Apps preflight. $($failures -join '; ')" } -Initialize-MhhHostedTagPolicy -SubscriptionId $SubscriptionId -Location $locations[0] +Initialize-MhhHostedTagPolicy -SubscriptionId $SubscriptionId -Location $ready[0] Update-AzTag -ResourceId "/subscriptions/$SubscriptionId" -Operation Merge -Tag @{ - 'microhack-adaptive-location' = $locations[0] + 'microhack-adaptive-location' = $ready[0] 'microhack-adaptive-regions' = $ready -join ',' 'microhack-adaptive-lab-count' = "$labCount" } -ErrorAction Stop | Out-Null diff --git a/03-Azure/01-01-App Innovation/04-adaptive-apps/labautomation/tests/console.Tests.ps1 b/03-Azure/01-01-App Innovation/04-adaptive-apps/labautomation/tests/console.Tests.ps1 index 3262e6291..a31e3a630 100644 --- a/03-Azure/01-01-App Innovation/04-adaptive-apps/labautomation/tests/console.Tests.ps1 +++ b/03-Azure/01-01-App Innovation/04-adaptive-apps/labautomation/tests/console.Tests.ps1 @@ -10,6 +10,11 @@ BeforeAll { function New-AzResourceGroupDeployment { [CmdletBinding()] param($Name, $ResourceGroupName, $TemplateFile, $TemplateParameterObject, $TemplateParameterFile, $Mode) } + function Invoke-MhhDeploymentWithRegionFallback { + [CmdletBinding()] + param($PreferredLocations, $ResourceGroupName, $RgOwnerEntraObjectIds, $TemplateFile, + $TemplateParameterFile, $DeploymentNamePrefix, $Tag) + } function New-AzSubscriptionDeployment { [CmdletBinding()] param($Name, $Location, $TemplateFile) } function New-AzResourceGroup { [CmdletBinding()] param($Name, $Location, $Tag, [switch]$Force) } function New-AzNetworkSecurityGroup { [CmdletBinding()] param($Name, $ResourceGroupName, $Location, $Tag, [switch]$Force) } @@ -84,6 +89,10 @@ Describe 'Shared hook and hosted policy verification' { $script:blockedRegion = '' $script:policyTagsValid = $true $script:sharedTags = $null + $script:featureStates = [Collections.Generic.Queue[string]]::new() + $script:featureStates.Enqueue('Registered') + $script:featureState = 'Registered' + $script:featureCalls = [Collections.Generic.List[string]]::new() $script:registeredProviders = [Collections.Generic.List[string]]::new() Mock Update-MhhToken { } Mock Start-Sleep { } @@ -110,7 +119,16 @@ Describe 'Shared hook and hosted policy verification' { $global:LASTEXITCODE = 0 switch ("$args") { { $_ -like 'account show *' } { "{`"id`":`"$subscription`"}"; break } - { $_ -like 'provider register *' } { $script:registeredProviders.Add("$args"); '{}'; break } + { $_ -like 'feature show *' } { + $script:featureCalls.Add('show') + if ($script:featureStates.Count) { $script:featureState = $script:featureStates.Dequeue() } + "{`"properties`":{`"state`":`"$script:featureState`"}}"; break + } + { $_ -like 'feature register *' } { $script:featureCalls.Add('register'); '{}'; break } + { $_ -like 'provider register *' } { + $script:featureState | Should -Be 'Registered' + $script:registeredProviders.Add("$args"); '{}'; break + } { $_ -like 'provider show *' } { '{"registrationState":"Registered"}'; break } { $_ -like 'vm list-skus *' } { if ($script:blockedRegion -and "$args" -like "*--location $script:blockedRegion *") { '[]' } @@ -137,9 +155,41 @@ Describe 'Shared hook and hosted policy verification' { Should -Invoke New-AzSubscriptionDeployment -Times 1 Should -Invoke Remove-AzResourceGroup -Times 1 } - It 'refuses success when only an alternative region passes' { + It 'allows initial provisioning in the validated alternative when the primary fails' { + $script:blockedRegion = 'westeurope' + . (Join-Path $automation 'shared-deploy-lab.ps1') -SubscriptionId $subscription -PreferredLocation @('westeurope,northeurope') -AllowedEntraUserIds @('one', 'two') + $script:sharedTags['microhack-adaptive-location'] | Should -Be 'northeurope' + $script:sharedTags['microhack-adaptive-regions'] | Should -Be 'northeurope' + } + It 'fails when no region passes preflight' { $script:blockedRegion = 'westeurope' - { . (Join-Path $automation 'shared-deploy-lab.ps1') -SubscriptionId $subscription -PreferredLocation @('westeurope,northeurope') -AllowedEntraUserIds @('one', 'two') } | Should -Throw '*Reconfigure the event location*' + { . (Join-Path $automation 'shared-deploy-lab.ps1') -SubscriptionId $subscription -PreferredLocation @('westeurope') -AllowedEntraUserIds @('one') } | + Should -Throw '*No preferred region passed*' + Should -Invoke New-AzSubscriptionDeployment -Times 0 + } + It 'registers the public IP feature before re-registering Network' { + $script:featureStates.Clear() + foreach ($state in @('NotRegistered', 'Registering', 'Registered')) { $script:featureStates.Enqueue($state) } + . (Join-Path $automation 'shared-deploy-lab.ps1') -SubscriptionId $subscription -PreferredLocation @('westeurope') -AllowedEntraUserIds @('one') + ($script:featureCalls -join ',') | Should -Be 'show,register,show,show' + @($script:registeredProviders | Where-Object { $_ -like 'provider register --namespace Microsoft.Network *' }) | Should -HaveCount 1 + } + It 'waits for in-progress registration without resubmitting' { + $script:featureStates.Clear() + foreach ($state in @('Registering', 'Registered')) { $script:featureStates.Enqueue($state) } + . (Join-Path $automation 'shared-deploy-lab.ps1') -SubscriptionId $subscription -PreferredLocation @('westeurope') -AllowedEntraUserIds @('one') + $script:featureCalls | Should -Not -Contain 'register' + } + It 'blocks fanout when feature registration times out or fails' -ForEach @( + @{ State = 'Pending'; Error = '*within 15 minutes*' } + @{ State = 'Failed'; Error = "*ended in state 'Failed'*" } + ) { + $script:featureStates.Clear() + $script:featureStates.Enqueue('Registering') + $script:featureStates.Enqueue($State) + { . (Join-Path $automation 'shared-deploy-lab.ps1') -SubscriptionId $subscription -PreferredLocation @('westeurope') -AllowedEntraUserIds @('one') } | + Should -Throw $Error + $script:registeredProviders | Should -HaveCount 0 Should -Invoke New-AzSubscriptionDeployment -Times 0 Should -Invoke Update-AzTag -Times 0 } @@ -162,6 +212,8 @@ Describe 'Participant hook orchestration' { $script:tamperedDownload = $false $script:failedPhase = '' $script:location = 'westeurope' + $script:groupTags = @{ adaptiveAppsReady = 'true' } + $script:fallbackLocation = 'northeurope' $script:guestMarker = 'ADAPTIVE_K3S_READY' $script:deploymentState = 'Succeeded' $script:deploymentThrows = $false @@ -191,10 +243,25 @@ Describe 'Participant hook orchestration' { Mock Update-MhhToken { $script:calls.Add('refresh') } Mock Get-AzContext { @{ Subscription = @{ Id = $subscription } } } Mock Get-AzResourceGroup { - @{ Location = $script:location; ResourceId = "/subscriptions/$subscription/resourceGroups/lab-one" } + @{ Location = $script:location; ResourceId = "/subscriptions/$subscription/resourceGroups/lab-one"; Tags = $script:groupTags } } Mock Get-AzTag { @{ Properties = @{ TagsProperty = @{ 'microhack-adaptive-regions' = 'westeurope,northeurope' } } } } - Mock Update-AzTag { $script:tagWrites.Add($Tag.Clone()) } + Mock Update-AzTag { + $script:tagWrites.Add($Tag.Clone()) + foreach ($key in $Tag.Keys) { $script:groupTags[$key] = $Tag[$key] } + } + Mock Invoke-MhhDeploymentWithRegionFallback { + ($PreferredLocations -join ',') | Should -Be 'westeurope,northeurope' + $RgOwnerEntraObjectIds | Should -Not -BeNullOrEmpty + $Tag.adaptiveAppsReady | Should -Be 'false' + $Tag.SecurityControl | Should -Be 'Ignore' + $Tag.CostControl | Should -Be 'Ignore' + $script:location = $script:fallbackLocation + $script:groupTags = $Tag.Clone() + $deployment = New-AzResourceGroupDeployment -Name $DeploymentNamePrefix -ResourceGroupName $ResourceGroupName ` + -TemplateFile $TemplateFile -TemplateParameterFile $TemplateParameterFile -Mode Incremental + @{ Success = $true; LocationUsed = $script:location; DeploymentResult = $deployment } + } Mock New-MhhStablePassword { 'TestOnlyStablePassword123' } Mock New-AzResourceGroupDeployment { $Mode | Should -Be 'Incremental' @@ -209,7 +276,7 @@ Describe 'Participant hook orchestration' { $parameters = Get-Content -LiteralPath $TemplateParameterFile -Raw | ConvertFrom-Json $parameters.'$schema' | Should -Be 'https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#' $parameters.contentVersion | Should -Be '1.0.0.0' - $parameters.parameters.location.value | Should -Be $script:location + $parameters.parameters.PSObject.Properties.Name | Should -Not -Contain 'location' $parameters.parameters.adminPassword.value | Should -BeExactly 'TestOnlyStablePassword123' $serialized = [Management.Automation.PSSerializer]::Serialize($PesterBoundParameters, 10) $serialized | Should -Not -Match 'TestOnlyStablePassword|]' @@ -242,7 +309,7 @@ Describe 'Participant hook orchestration' { ProvisioningState = $script:deploymentState Outputs = @{ acrName = @{ Value = 'acadtestregistry' } - nodeResourceGroup = @{ Value = 'MC_lab-one_aks-adaptive-apps_westeurope' } + nodeResourceGroup = @{ Value = "MC_lab-one_aks-adaptive-apps_$script:location" } } } } @@ -290,7 +357,8 @@ Describe 'Participant hook orchestration' { $env:HOME | Should -Not -Be $script:originalHome $env:PATH | Should -BeLike "$env:HOME/.local/bin:*" $env:AZURE_CONFIG_DIR | Should -Be (Join-Path $TestDrive 'platform-azure') - $env:REGION | Should -Be 'westeurope' + $env:REGION | Should -Be $script:location + $env:AZURE_LOCATION | Should -Be $script:location $env:BASTION_NAME | Should -Be 'bas-adaptive-apps' [int]$env:K3S_LOCAL_PORT | Should -BeGreaterThan 0 $env:K3S_KUBECONFIG | Should -BeLike "$env:HOME/*" @@ -323,6 +391,47 @@ Describe 'Participant hook orchestration' { { . (Join-Path $automation 'deploy-lab.ps1') -DeploymentType resourcegroup -SubscriptionId $subscription -ResourceGroupName lab-one -AllowedEntraUserIds @('user') } | Should -Throw '*exit code 13*' @($script:tagWrites | Where-Object { $_.adaptiveAppsReady -eq 'true' }) | Should -HaveCount 0 } + It 'uses validated fallback regions only for initial infrastructure and bootstraps in the resulting RG region' { + $script:groupTags = @{} + $result = . (Join-Path $automation 'deploy-lab.ps1') -DeploymentType resourcegroup -SubscriptionId $subscription -ResourceGroupName lab-one -AllowedEntraUserIds @('user') + Should -Invoke Invoke-MhhDeploymentWithRegionFallback -Times 1 -Exactly -ParameterFilter { + $RgOwnerEntraObjectIds -contains 'user' -and $DeploymentNamePrefix -eq 'adaptive-apps-console' + } + ($result.HackboxCredential | Where-Object name -eq 'Adaptive Apps Region').value | Should -Be 'northeurope' + $script:groupTags.adaptiveAppsPreserve | Should -Be 'true' + } + It 'never invokes destructive fallback for a previously ready lab even after a failed retry' { + $script:deploymentThrows = $true + { . (Join-Path $automation 'deploy-lab.ps1') -DeploymentType resourcegroup -SubscriptionId $subscription -ResourceGroupName lab-one -AllowedEntraUserIds @('user') } | + Should -Throw '*Simulated ARM deployment exception*' + $script:groupTags.adaptiveAppsReady | Should -Be 'false' + $script:groupTags.adaptiveAppsPreserve | Should -Be 'true' + $script:downloadRoot = '' + { . (Join-Path $automation 'deploy-lab.ps1') -DeploymentType resourcegroup -SubscriptionId $subscription -ResourceGroupName lab-one -AllowedEntraUserIds @('user') } | + Should -Throw '*Simulated ARM deployment exception*' + Should -Invoke Invoke-MhhDeploymentWithRegionFallback -Times 0 + } + It 'preserves initial infrastructure after a bootstrap failure on subsequent retries' { + $script:groupTags = @{} + $script:fallbackLocation = 'westeurope' + $script:failedPhase = 'recipes' + { . (Join-Path $automation 'deploy-lab.ps1') -DeploymentType resourcegroup -SubscriptionId $subscription -ResourceGroupName lab-one -AllowedEntraUserIds @('user') } | + Should -Throw '*exit code 13*' + $script:groupTags.adaptiveAppsPreserve | Should -Be 'true' + $script:downloadRoot = '' + $script:observedWorkingRoot = '' + $script:failedPhase = '' + . (Join-Path $automation 'deploy-lab.ps1') -DeploymentType resourcegroup -SubscriptionId $subscription -ResourceGroupName lab-one -AllowedEntraUserIds @('user') | Out-Null + Should -Invoke Invoke-MhhDeploymentWithRegionFallback -Times 1 -Exactly + } + It 'propagates exhausted fallback without bootstrap or readiness' { + $script:groupTags = @{} + Mock Invoke-MhhDeploymentWithRegionFallback { throw 'RegionFallbackExhausted' } + { . (Join-Path $automation 'deploy-lab.ps1') -DeploymentType resourcegroup -SubscriptionId $subscription -ResourceGroupName lab-one -AllowedEntraUserIds @('user') } | + Should -Throw '*RegionFallbackExhausted*' + Should -Invoke bash -Times 0 + $script:groupTags.adaptiveAppsReady | Should -Be 'false' + } It 'passes deployment arguments through a real child job without serializing a SecureString' { $script:useDeploymentJob = $true $result = . (Join-Path $automation 'deploy-lab.ps1') -DeploymentType resourcegroup -SubscriptionId $subscription -ResourceGroupName lab-one -AllowedEntraUserIds @('user') @@ -370,8 +479,9 @@ Describe 'Participant hook orchestration' { } It 'rejects unvalidated RG region before deployment without deleting the scope' { $script:location = 'eastus' - { . (Join-Path $automation 'deploy-lab.ps1') -DeploymentType resourcegroup -SubscriptionId $subscription -ResourceGroupName lab-one -AllowedEntraUserIds @('user') } | Should -Throw '*was not validated*' + { . (Join-Path $automation 'deploy-lab.ps1') -DeploymentType resourcegroup -SubscriptionId $subscription -ResourceGroupName lab-one -AllowedEntraUserIds @('user') } | Should -Throw '*No validated region*' Should -Invoke New-AzResourceGroupDeployment -Times 0 + Should -Invoke Invoke-MhhDeploymentWithRegionFallback -Times 0 } It 'fails before Azure calls and cleans scratch when downloading fails' { $script:failedDownload = $true @@ -391,7 +501,8 @@ Describe 'Participant hook orchestration' { } It 'stops on ARM failure without fallback or bootstrap' { $script:deploymentState = 'Failed' - { . (Join-Path $automation 'deploy-lab.ps1') -DeploymentType resourcegroup -SubscriptionId $subscription -ResourceGroupName lab-one -AllowedEntraUserIds @('user') } | Should -Throw '*no destructive fallback*' + { . (Join-Path $automation 'deploy-lab.ps1') -DeploymentType resourcegroup -SubscriptionId $subscription -ResourceGroupName lab-one -AllowedEntraUserIds @('user') } | Should -Throw '*Bootstrap will not run*' + Should -Invoke Invoke-MhhDeploymentWithRegionFallback -Times 0 Should -Invoke bash -Times 0 } It 'requires guest readiness, not just Run Command HTTP success' { @@ -455,13 +566,13 @@ Describe 'Portable packaging and source contracts' { $errors | Should -BeNullOrEmpty } } - It 'uses configured platform defaults and shared first-region gating' { + It 'uses configured platform defaults and fails when no region is ready' { $defaults = Get-Content (Join-Path $automation 'lab-defaults.json') -Raw | ConvertFrom-Json $defaults.deploymentType | Should -Be 'resourcegroup' $defaults.labsPerSubscription | Should -Be 5 - $defaults.preferredLocation | Should -Be 'swedencentral' + $defaults.preferredLocation | Should -Be 'swedencentral,spaincentral' $shared = Get-Content (Join-Path $automation 'shared-deploy-lab.ps1') -Raw - $shared | Should -Match '\$locations\[0\] -notin \$ready' + $shared | Should -Match '\$ready.Count -eq 0' $shared | Should -Match 'microhack-adaptive-regions' $shared | Should -Not -Match 'az ad |AzureStackHCI|LocalBox|Confidential' }