diff --git a/03-Azure/01-04-AI/06_AI_Governance/challenges/challenge-01.md b/03-Azure/01-04-AI/06_AI_Governance/challenges/challenge-01.md index 483196695..87eb402c7 100644 --- a/03-Azure/01-04-AI/06_AI_Governance/challenges/challenge-01.md +++ b/03-Azure/01-04-AI/06_AI_Governance/challenges/challenge-01.md @@ -52,16 +52,27 @@ end of the event. Every resource in your lab is named with the same random token, and the `.env` values are built from it. In the portal, open your resource group (`rg-labuser-00XX`) and find -the AI Foundry account named **`aif-hub-`** — everything after the prefix is your -token. For example `aif-hub-fd79f0b6fee89` means the token is `fd79f0b6fee89`. +the API Management service named **`apim-citadel-`** — everything after the prefix +is your token. For example `apim-citadel-fd79f0b6fee89` means the token is +`fd79f0b6fee89`. From the CLI: ```bash -az cognitiveservices account list -g \ - --query "[?starts_with(name,'aif-hub-')].name" -o tsv +az apim list -g --query "[].name" -o tsv ``` +> [!IMPORTANT] +> The two AI Foundry accounts are the one exception: they carry a few extra random +> characters after the token (for example `aif-hub-fd79f0b6fee89-k3xq`). Azure reserves a +> deleted Foundry account's name for two weeks, so the lab gives them a fresh name on every +> deployment. Copy those two names **exactly as they appear** rather than building them +> from the token: +> +> ```bash +> az cognitiveservices account list -g --query "[].name" -o tsv +> ``` + #### A.3 — Fill in `.env` ```bash @@ -71,7 +82,7 @@ cp .env.template .env Open `.env` and set the values below, replacing `` throughout. The first three come straight from the dashboard / portal Overview blade; the rest are derived from the -token. +token, except the two Foundry account names you copied in A.2. ```bash AZURE_RESOURCE_GROUP=rg-labuser-00XX @@ -80,11 +91,11 @@ AZURE_SUBSCRIPTION_ID= SPOKE_RESOURCE_GROUP=rg-labuser-00XX # same RG — this lab uses one resource group SPOKE_KEY_VAULT_NAME=kv-spoke- -SPOKE_AI_FOUNDRY_ACCOUNT_NAME=aif-spoke- +SPOKE_AI_FOUNDRY_ACCOUNT_NAME= # the full aif-spoke-... name SPOKE_AI_FOUNDRY_PROJECT_NAME=citadel-agents-project SPOKE_ACR_NAME=acrcitadel SPOKE_ACR_LOGIN_SERVER=acrcitadel.azurecr.io -A2A_FOUNDRY_ACCOUNT_NAME=aif-spoke- +A2A_FOUNDRY_ACCOUNT_NAME= # same value as above ``` Challenges 1-6 need only the first four values plus `LLM_BACKEND_CONFIG`. @@ -92,10 +103,11 @@ Challenges 1-6 need only the first four values plus `LLM_BACKEND_CONFIG`. #### A.4 — Fill in `LLM_BACKEND_CONFIG` This one is a single line of JSON describing your gateway's backend and its six models. -Paste the block below as **one line**, with `` replaced in **both** places: +Paste the block below as **one line**, replacing `` in **both** +places with the full `aif-hub-...` name you copied in A.2: ``` -LLM_BACKEND_CONFIG=[{"backendId":"aif-hub-","backendType":"ai-foundry","endpoint":"https://aif-hub-.openai.azure.com","authScheme":"managedIdentity","authType":"managed-identity","priority":1,"weight":100,"supportedModels":[{"name":"gpt-4.1","sku":"GlobalStandard","capacity":20,"modelFormat":"OpenAI","modelVersion":"2025-04-14","retirementDate":"2027-04-14"},{"name":"gpt-5.4-mini","sku":"GlobalStandard","capacity":20,"modelFormat":"OpenAI","modelVersion":"2026-03-17","retirementDate":"2027-09-21"},{"name":"gpt-5.2","sku":"GlobalStandard","capacity":20,"modelFormat":"OpenAI","modelVersion":"2025-12-11","retirementDate":"2027-06-08"},{"name":"text-embedding-3-large","sku":"GlobalStandard","capacity":20,"modelFormat":"OpenAI","modelVersion":"1","retirementDate":"2028-02-09"},{"name":"Mistral-Large-3","sku":"GlobalStandard","capacity":20,"modelFormat":"Mistral AI","modelVersion":"1","retirementDate":"2099-12-31"},{"name":"Phi-4","sku":"GlobalStandard","capacity":1,"modelFormat":"Microsoft","modelVersion":"7","retirementDate":"2099-12-31"}]}] +LLM_BACKEND_CONFIG=[{"backendId":"","backendType":"ai-foundry","endpoint":"https://.openai.azure.com","authScheme":"managedIdentity","authType":"managed-identity","priority":1,"weight":100,"supportedModels":[{"name":"gpt-4.1","sku":"GlobalStandard","capacity":20,"modelFormat":"OpenAI","modelVersion":"2025-04-14","retirementDate":"2027-04-14"},{"name":"gpt-5.4-mini","sku":"GlobalStandard","capacity":20,"modelFormat":"OpenAI","modelVersion":"2026-03-17","retirementDate":"2027-09-21"},{"name":"gpt-5.2","sku":"GlobalStandard","capacity":20,"modelFormat":"OpenAI","modelVersion":"2025-12-11","retirementDate":"2027-06-08"},{"name":"text-embedding-3-large","sku":"GlobalStandard","capacity":20,"modelFormat":"OpenAI","modelVersion":"1","retirementDate":"2028-02-09"},{"name":"Mistral-Large-3","sku":"GlobalStandard","capacity":20,"modelFormat":"Mistral AI","modelVersion":"1","retirementDate":"2099-12-31"},{"name":"Phi-4","sku":"GlobalStandard","capacity":1,"modelFormat":"Microsoft","modelVersion":"7","retirementDate":"2099-12-31"}]}] ``` > [!IMPORTANT] diff --git a/03-Azure/01-04-AI/06_AI_Governance/labautomation/README.md b/03-Azure/01-04-AI/06_AI_Governance/labautomation/README.md index 7e74fc233..4c39f91c0 100644 --- a/03-Azure/01-04-AI/06_AI_Governance/labautomation/README.md +++ b/03-Azure/01-04-AI/06_AI_Governance/labautomation/README.md @@ -14,7 +14,7 @@ Adapted from the Citadel workshop (`azd up`), the lab deploys: | Resource | Purpose | |----------|---------| | **API Management (StandardV2)** | Unified AI gateway; all model calls route through APIM | -| **Azure AI Foundry Account (Hub)** | Primary Foundry account (`aif-hub-`) for governance backend | +| **Azure AI Foundry Account (Hub)** | Primary Foundry account (`aif-hub-`) for governance backend | | **Azure AI Foundry Project (Hub)** | Hosts hub-level agents and access-control demonstrations | | **Log Analytics Workspace** | Hub observability backend | | **Application Insights** | Connected to hub Foundry for agent tracing | @@ -27,7 +27,7 @@ Adapted from the Citadel workshop (`azd up`), the lab deploys: ### Spoke Resources (Folded into Same RG) | Resource | Purpose | |----------|---------| -| **Azure AI Foundry Account (Spoke)** | Sample workload Foundry account (`aif-spoke-`) for attendee agent development | +| **Azure AI Foundry Account (Spoke)** | Sample workload Foundry account (`aif-spoke-`) for attendee agent development | | **Azure AI Foundry Project (Spoke)** | Where attendees deploy their governed agents (`citadel-agents-project`) | | **Log Analytics Workspace (Spoke)** | Spoke observability | | **Application Insights (Spoke)** | Spoke agent tracing | @@ -65,7 +65,7 @@ infix plus the shared `resourceToken`. | Workshop `deploy-spoke-foundry.ps1` step | Equivalent here | |---|---| | Create spoke resource group | *Not applicable* — hub and spoke share the participant's single RG | -| Foundry account (`allowProjectManagement`, system-assigned identity) | `spokeFoundryAccount` → `aif-spoke-` | +| Foundry account (`allowProjectManagement`, system-assigned identity) | `spokeFoundryAccount` → `aif-spoke-` | | Foundry project (system-assigned identity) | `spokeFoundryProject` → `citadel-agents-project` | | Log Analytics workspace | `laSpoke` → `law-spoke-` | | Application Insights | `aiSpoke` → `appi-spoke-` | @@ -163,12 +163,39 @@ The script follows MicroHack conventions: - **Region fallback** — Retries across `$PreferredLocation` (honoring platform preference), falling back to [swedencentral, westeurope, norwayeast] if all fail. Retries are gated on `Test-MhhDeploymentFailureRetryable`: a non-retryable failure (template or permission bug) - aborts immediately with the real error rather than being retried in every region. + aborts immediately with the real error rather than being retried in every region. Fallback + applies to **empty resource groups only** — once an RG holds resources their names are + pinned to that region, so any other candidate would fail with `InvalidResourceLocation` and + bury the real error. If all regions fail, the **first** (root cause) error is reported. - **Deterministic naming** — `Get-MhhStableHash` generates a stable suffix per attendee, ensuring names are DNS-safe, globally unique, and consistent across re-runs. The hash is computed **once** and passed to Bicep as `resourceToken`; the template is the single source of truth for every resource name. -- **Idempotent provisioning** — All resources are created once; re-runs skip existing resources. +- **Idempotent provisioning** — Re-runs reuse healthy resources. Because `resourceToken` is + derived from subscription id + RG name — both of which the platform reuses when an event is + torn down and re-deployed — the script first **purges soft-deleted Key Vaults and APIM + services** whose names contain this lab's token, and **recycles Foundry accounts left in a + `Failed` state**. A soft-deleted resource keeps its name reserved, so without this a + re-deployed event fails during preflight with an opaque + `Microsoft.CognitiveServices/accounts ... reported preflight validation errors` before any + resource is touched. Purges are asynchronous, so the script then waits for the names to + actually disappear before deploying. Only this lab's token is matched, so other labs sharing + the subscription are never affected. +- **Fresh Foundry account names** — Foundry is the one resource that purging cannot rescue. An + `AIServices` account with `allowProjectManagement` is backed by a hidden Azure ML (`AmlRp`) + workspace; deleting the account soft-deletes that workspace for ~14 days, purging the + Cognitive Services account does **not** purge it, and Azure exposes no API to list or purge + it. Recreating the account then always lands in `provisioningState: Failed` with + *"Soft-deleted workspace exists. Please purge or recover it."* — which is what took out every + lab on the Sept-10 re-run. The script therefore reuses the existing Foundry accounts only + while they are healthy, and otherwise deploys under a **fresh name** + (`aif-hub-`), recording the suffix as a resource-group tag + (`citadelFoundrySuffix`) so repeat runs stay stable. This is the only reason attendees must + copy the two Foundry account names rather than build them from the token. +- **Transient-failure retry** — A deployment is retried up to three times in the same region + for genuinely transient states (`ServiceLocked` while APIM is still activating, AML + soft-delete lag), re-running the cleanup before each attempt so a retry never trips over the + wreckage of the previous one. - **Managed identities** — No hardcoded keys or connection strings; all auth uses Entra ID + Azure RBAC (DefaultAzureCredential in notebooks). - **Multi-user RBAC** — Every ID in `$AllowedEntraUserIds` is granted the necessary data-plane diff --git a/03-Azure/01-04-AI/06_AI_Governance/labautomation/deploy-lab.ps1 b/03-Azure/01-04-AI/06_AI_Governance/labautomation/deploy-lab.ps1 index 463160d8e..a1875ca99 100644 --- a/03-Azure/01-04-AI/06_AI_Governance/labautomation/deploy-lab.ps1 +++ b/03-Azure/01-04-AI/06_AI_Governance/labautomation/deploy-lab.ps1 @@ -125,40 +125,294 @@ if ($AllowedEntraUserIds.Count -eq 0) { Write-Host "[INFO] Deploying Citadel Agentic Governance Hub + Spoke into RG '$effectiveRG'..." Write-Host "[INFO] Engine: $templateKind, resource-group-scoped." -$deployOutputs = $null -$effectiveLocation = $null +# ===== Purge soft-deleted namesakes left by a previous event ===== +# $resourceToken is derived from subscription id + RG name, and the platform reuses BOTH +# when an event is torn down and a new one is deployed over it. Foundry accounts, Key +# Vaults and APIM services are soft deleted rather than removed, and a soft-deleted resource +# keeps its name reserved — for Foundry, also its globally unique customSubDomainName. +# Re-deploying then fails with 'FlagMustBeSetForRestore' during preflight (Foundry, before +# any resource is touched), 'ConflictError' (Key Vault) or +# 'ServiceAlreadyExistsInSoftDeletedState' (APIM), which is what took out all three labs on +# the Sept-10 re-run. +# +# Only names ending in THIS lab's token are purged, so other labs sharing the subscription +# are never affected. A purge failure is non-fatal: it is reported, and the deployment +# below surfaces the resulting preflight error. +# Wrapped in a function because it must run again before EVERY deployment attempt: a failed +# attempt leaves Foundry accounts in a Failed state and freshly soft-deleted namesakes behind, +# so retrying without cleaning up first fails preflight on the wreckage of the previous try. +function Invoke-LabSoftDeleteCleanup { + # A Foundry account left in a Failed provisioning state by an earlier partial deployment is + # just as blocking: ARM refuses to update it ('AccountIsNotSucceeded'), so the resource group + # would stay permanently un-deployable. Deleting it soft-deletes it, and the purge pass below + # then frees the name for this run to recreate. + try { + $failedAccounts = @(Get-AzResource -ResourceGroupName $effectiveRG ` + -ResourceType 'Microsoft.CognitiveServices/accounts' -ExpandProperties -ErrorAction SilentlyContinue | + Where-Object { $_.Name -and $_.Name.Contains($resourceToken) -and $_.Properties.provisioningState -eq 'Failed' }) + foreach ($account in $failedAccounts) { + Write-Host "[INFO] Foundry account '$($account.Name)' is in a Failed state from an earlier run — recycling it..." + Remove-AzResource -ResourceId $account.ResourceId -Force -ErrorAction Stop | Out-Null + Write-Host "[OK] Deleted '$($account.Name)'; it will be purged and recreated below." + } + } + catch { + Write-Host "[WARN] Could not recycle failed Foundry accounts: $_" + } -foreach ($region in $candidateRegions) { - Write-Host "[INFO] Deploying → RG '$effectiveRG' in '$region' (token '$resourceToken')..." + $softDeletedScopes = @( + @{ Label = 'Foundry account'; ApiVersion = '2023-05-01' + ListPath = "/subscriptions/$SubscriptionId/providers/Microsoft.CognitiveServices/deletedAccounts?api-version=2023-05-01" + # Cognitive Services purges via DELETE on the deleted-account resource itself. + PurgeMethod = 'DELETE'; PurgeSuffix = '' }, + @{ Label = 'Key Vault'; ApiVersion = '2022-07-01' + ListPath = "/subscriptions/$SubscriptionId/providers/Microsoft.KeyVault/deletedVaults?api-version=2022-07-01" + # Key Vault has no DELETE on the deleted vault (that returns HTTP 405) — it exposes a + # dedicated POST .../purge action instead. + PurgeMethod = 'POST'; PurgeSuffix = '/purge' }, + @{ Label = 'APIM service'; ApiVersion = '2022-08-01' + ListPath = "/subscriptions/$SubscriptionId/providers/Microsoft.ApiManagement/deletedservices?api-version=2022-08-01" + # APIM soft-deletes too, failing the deployment with ServiceAlreadyExistsInSoftDeletedState. + PurgeMethod = 'DELETE'; PurgeSuffix = '' } + ) + + # Azure REST calls occasionally fail with transient TLS/socket errors. Without a retry a + # single blip silently skips the rest of the purge pass and the deployment then fails on a + # name that was supposed to have been cleared. + function Invoke-LabAzRest { + param([Parameter(Mandatory)][string]$Method, [Parameter(Mandatory)][string]$Path, [int]$MaxAttempts = 3) + for ($attempt = 1; $attempt -le $MaxAttempts; $attempt++) { + try { + return Invoke-AzRestMethod -Method $Method -Path $Path -ErrorAction Stop + } + catch { + if ($attempt -eq $MaxAttempts) { throw } + $reason = "$($_.Exception.Message)".Split("`n")[0] + Write-Host "[WARN] $Method call failed (attempt $attempt/$MaxAttempts): $reason — retrying..." + Start-Sleep -Seconds (5 * $attempt) + } + } + } + + $purgedAnything = $false + + foreach ($scope in $softDeletedScopes) { + try { + $listResponse = Invoke-LabAzRest -Method GET -Path $scope.ListPath + if ($listResponse.StatusCode -ne 200) { + Write-Host "[WARN] Could not list soft-deleted $($scope.Label)s (HTTP $($listResponse.StatusCode)) — skipping purge check." + continue + } + # The deleted-resource id already encodes location and original RG, so purging by + # id avoids reconstructing a scope that may differ from the current deployment. + $stale = @(($listResponse.Content | ConvertFrom-Json).value | + Where-Object { $_.name -and $_.name.Contains($resourceToken) }) + + foreach ($item in $stale) { + # Isolated per item: one failure must not skip the remaining purges. + try { + Write-Host "[INFO] Purging soft-deleted $($scope.Label) '$($item.name)' from a previous event..." + $purge = Invoke-LabAzRest -Method $scope.PurgeMethod ` + -Path "$($item.id)$($scope.PurgeSuffix)?api-version=$($scope.ApiVersion)" + if (@(200, 202, 204) -contains $purge.StatusCode) { + Write-Host "[OK] Purged '$($item.name)'." + $purgedAnything = $true + } + else { + Write-Host "[WARN] Failed to purge '$($item.name)' (HTTP $($purge.StatusCode)). Deployment will likely" + Write-Host "[WARN] fail preflight because the name is still reserved. Response: $($purge.Content)" + } + } + catch { + Write-Host "[WARN] Failed to purge '$($item.name)': $_" + } + } + } + catch { + Write-Host "[WARN] Soft-deleted $($scope.Label) check skipped: $_" + } + } + + # Purging is eventually consistent, and Key Vault/APIM purges are outright asynchronous. + # Recreating a name before the purge has propagated fails with a bare 'Failed to Create the + # resource. Provisioning state: Failed', which then leaves the account in a Failed state that + # blocks every later run. Wait for the names to actually disappear before deploying. + if ($purgedAnything) { + Write-Host "[INFO] Waiting for purges to propagate before deploying..." + $purgeDeadline = (Get-Date).AddMinutes(5) + do { + Start-Sleep -Seconds 10 + $stillPresent = 0 + foreach ($scope in $softDeletedScopes) { + try { + $check = Invoke-LabAzRest -Method GET -Path $scope.ListPath + if ($check.StatusCode -eq 200) { + $stillPresent += @(($check.Content | ConvertFrom-Json).value | + Where-Object { $_.name -and $_.name.Contains($resourceToken) }).Count + } + } + catch { } + } + } while ($stillPresent -gt 0 -and (Get-Date) -lt $purgeDeadline) + + if ($stillPresent -gt 0) { + Write-Host "[WARN] $stillPresent soft-deleted resource(s) still present after 5 minutes." + Write-Host "[WARN] Continuing anyway — the deployment below will report the conflict." + } + else { + # The name index lags the deleted-resource list by a few seconds. + Start-Sleep -Seconds 20 + Write-Host "[OK] All purges propagated." + } + } +} + +Invoke-LabSoftDeleteCleanup + +# ===== Pick a usable Azure AI Foundry account name ===== +# Deleting an `AIServices` account soft-deletes the hidden AML workspace behind it for ~14 +# days, and that workspace cannot be purged -- Azure exposes no API for it. The account name +# is therefore burned: recreating it always lands in `provisioningState: Failed` with +# "Soft-deleted workspace exists". Since $resourceToken is deterministic and the platform +# reuses subscription + RG name, a re-run would be guaranteed to fail on the old name. +# +# So: reuse the current account only while it is healthy, and otherwise move to a brand-new +# name. The chosen suffix is stored as a resource-group tag so that repeat runs stay stable +# and do not strand a working deployment behind a new set of names. +$foundrySuffixTag = 'citadelFoundrySuffix' + +function Resolve-LabFoundrySuffix { + $suffix = '' try { - $d = New-AzResourceGroupDeployment ` - -ResourceGroupName $effectiveRG ` - -TemplateFile $templateFile ` - -location $region ` - -resourceToken $resourceToken ` - -tags $tags ` - -ErrorAction Stop - $deployOutputs = $d.Outputs - $effectiveLocation = $region - break + $rgTags = (Get-AzResourceGroup -Name $effectiveRG -ErrorAction Stop).Tags + if ($rgTags -and $rgTags[$foundrySuffixTag]) { $suffix = [string]$rgTags[$foundrySuffixTag] } + } + catch { } + + if ($suffix) { + $hub = Get-AzResource -ResourceGroupName $effectiveRG -Name "aif-hub-$resourceToken$suffix" ` + -ResourceType 'Microsoft.CognitiveServices/accounts' -ExpandProperties -ErrorAction SilentlyContinue + if ($hub -and $hub.Properties.provisioningState -eq 'Succeeded') { + Write-Host "[OK] Reusing healthy Foundry accounts 'aif-*-$resourceToken$suffix'." + return $suffix + } + } + + # No healthy account: the previous name (if any) is now unusable, so mint a fresh one. + # An empty RG is treated the same way — the RG may have been deleted and recreated, which + # leaves the soft-deleted workspace behind while removing all visible trace of it. + $new = -join ((1..4) | ForEach-Object { 'abcdefghijklmnopqrstuvwxyz0123456789'[(Get-Random -Maximum 36)] }) + Write-Host "[INFO] No healthy Foundry account found — using fresh account names 'aif-*-$resourceToken$new'." + Write-Host "[INFO] (A previously deleted account keeps its name reserved by an unpurgeable AML workspace.)" + try { + $existingTags = (Get-AzResourceGroup -Name $effectiveRG -ErrorAction Stop).Tags + if (-not $existingTags) { $existingTags = @{} } + $existingTags[$foundrySuffixTag] = $new + Set-AzResourceGroup -Name $effectiveRG -Tag $existingTags -ErrorAction Stop | Out-Null } catch { - # Only capacity/quota/region-availability failures are worth retrying elsewhere. - # A template or permission bug fails identically in every region, so retrying it - # just triples the runtime and buries the real error under the last region's message. - $retryable = $true - if (Get-Command Test-MhhDeploymentFailureRetryable -ErrorAction SilentlyContinue) { - $retryable = [bool](Test-MhhDeploymentFailureRetryable -ErrorRecord $_) + Write-Host "[WARN] Could not record the Foundry suffix on the resource group: $_" + } + return $new +} + +$deployOutputs = $null +$effectiveLocation = $null +$firstFailure = $null + +# Region fallback only makes sense for an EMPTY resource group. Once the RG holds +# resources, their names are already pinned to the region of the first attempt, so every +# later candidate fails with 'InvalidResourceLocation: already exists in location +# ' — which masks the real error and cannot ever succeed. Re-running into a +# non-empty RG (a re-deployed event, or a retry after a partial failure) must therefore +# stay in the region that RG already uses. +$existingRegions = @( + Get-AzResource -ResourceGroupName $effectiveRG -ErrorAction SilentlyContinue | + Select-Object -ExpandProperty Location -ErrorAction SilentlyContinue | + Where-Object { $_ -and $_ -ne 'global' } | + Group-Object | + Sort-Object Count -Descending | + Select-Object -ExpandProperty Name +) +if ($existingRegions.Count -gt 0) { + # Prefer a region the platform actually asked for; otherwise the most common region + # already used in the RG. 'global' resources are ignored — they are not deployable regions. + $pinnedRegion = $candidateRegions | Where-Object { $existingRegions -contains $_ } | Select-Object -First 1 + if (-not $pinnedRegion) { $pinnedRegion = $existingRegions[0] } + if ($candidateRegions.Count -gt 1) { + Write-Host "[INFO] RG '$effectiveRG' already contains resources in '$($existingRegions -join ", ")'." + Write-Host "[INFO] Pinning deployment to '$pinnedRegion' (region fallback applies to empty RGs only)." + } + $candidateRegions = @($pinnedRegion) +} + +foreach ($region in $candidateRegions) { + # Some failures are purely transient and clear on their own: APIM reports 'ServiceLocked' + # while a previous create/purge is still activating, and the AML resource provider behind + # Foundry projects reports 'Soft-deleted workspace exists' for a short window after the + # parent account is purged. Retrying in place beats moving region — a different region + # would hit the same lag, and for a non-empty RG there is no other region to move to. + $transientPatterns = @( + 'ServiceLocked', 'is transitioning', 'Soft-deleted workspace exists', + 'AnotherOperationInProgress', 'operation is in progress', 'RetryableError' + ) + $maxAttempts = 3 + + for ($attempt = 1; $attempt -le $maxAttempts; $attempt++) { + $suffix = if ($attempt -gt 1) { " (attempt $attempt/$maxAttempts)" } else { '' } + if ($attempt -gt 1) { Invoke-LabSoftDeleteCleanup } + $foundryNameSuffix = Resolve-LabFoundrySuffix + Write-Host "[INFO] Deploying → RG '$effectiveRG' in '$region' (token '$resourceToken')$suffix..." + try { + $d = New-AzResourceGroupDeployment ` + -ResourceGroupName $effectiveRG ` + -TemplateFile $templateFile ` + -location $region ` + -resourceToken $resourceToken ` + -foundryNameSuffix $foundryNameSuffix ` + -tags $tags ` + -ErrorAction Stop + $deployOutputs = $d.Outputs + $effectiveLocation = $region + break } - if (-not $retryable) { - throw "Deployment failed in '$region' with a non-retryable error (retrying other regions would fail the same way): $_" + catch { + $message = "$_" + + $isTransient = $false + foreach ($pattern in $transientPatterns) { + if ($message -match $pattern) { $isTransient = $true; break } + } + if ($isTransient -and $attempt -lt $maxAttempts) { + Write-Host "[WARN] Transient Azure state during deployment: $message" + Write-Host "[INFO] This usually resolves itself — waiting 2 minutes and retrying in '$region'..." + Start-Sleep -Seconds 120 + continue + } + + # Only capacity/quota/region-availability failures are worth retrying elsewhere. + # A template or permission bug fails identically in every region, so retrying it + # just triples the runtime and buries the real error under the last region's message. + $retryable = $true + if (Get-Command Test-MhhDeploymentFailureRetryable -ErrorAction SilentlyContinue) { + $retryable = [bool](Test-MhhDeploymentFailureRetryable -ErrorRecord $_) + } + if (-not $retryable) { + throw "Deployment failed in '$region' with a non-retryable error (retrying other regions would fail the same way): $_" + } + # Keep the FIRST failure: it is the real one. Later regions typically fail with a + # downstream symptom, so reporting only the last error hides the actual cause. + if (-not $firstFailure) { $firstFailure = "in '$region': $_" } + Write-Host "[WARN] Deployment failed in '$region': $_ — trying next region." + break } - Write-Host "[WARN] Deployment failed in '$region': $_ — trying next region." } + + if ($deployOutputs) { break } } if (-not $deployOutputs) { - throw "Deployment failed in all candidate regions: $($candidateRegions -join ', ')" + throw "Deployment failed in all candidate regions ($($candidateRegions -join ', ')). First (root cause) error $firstFailure" } Write-Host "[OK] Provisioning complete in '$effectiveLocation' (resource group '$effectiveRG')." diff --git a/03-Azure/01-04-AI/06_AI_Governance/labautomation/infra/resources.bicep b/03-Azure/01-04-AI/06_AI_Governance/labautomation/infra/resources.bicep index 2b2fcf1b8..7fb57796e 100644 --- a/03-Azure/01-04-AI/06_AI_Governance/labautomation/infra/resources.bicep +++ b/03-Azure/01-04-AI/06_AI_Governance/labautomation/infra/resources.bicep @@ -16,6 +16,21 @@ param location string @description('Deterministic suffix for resource naming (DNS-safe, lowercase).') param resourceToken string +@maxLength(6) +@description('''Extra suffix appended to the two Azure AI Foundry account names only. +An `AIServices` account with `allowProjectManagement` is backed by a hidden AML +(`AmlRp`) workspace. Deleting the account soft-deletes that workspace for ~14 days, +and purging the Cognitive Services account does NOT purge it -- Azure exposes no API +to list or purge it at all. The account name is therefore unusable for two weeks after +any delete: recreating it always lands in `provisioningState: Failed` with +"Soft-deleted workspace exists. Please purge or recover it.". +Because `resourceToken` is derived from subscription id + resource group name and the +platform reuses both, every re-run of an event would otherwise be guaranteed to fail. +`labautomation/deploy-lab.ps1` supplies a fresh suffix whenever the previous account is +not healthy, and reuses the existing one when it is. Foundry accounts are the only +resources affected: Key Vault and APIM purge correctly and keep their stable names.''') +param foundryNameSuffix string = '' + // Participant RBAC is applied by labautomation/deploy-lab.ps1 AFTER this deployment, // so that EVERY id in $AllowedEntraUserIds is granted the roles (a template can only // conveniently grant the first one) and so that hub AND spoke resources are both @@ -37,7 +52,7 @@ only after confirming quota with `az cognitiveservices usage list -l `.' param modelCapacity int = 20 // ===== Hub Resource Naming ===== -var hubFoundryAccountName = 'aif-hub-${resourceToken}' +var hubFoundryAccountName = 'aif-hub-${resourceToken}${foundryNameSuffix}' var hubFoundryProjectName = 'citadel-hub-project' var apimName = 'apim-citadel-${resourceToken}' var cosmosAccountName = 'cos-citadel-${resourceToken}' @@ -52,7 +67,7 @@ var apimMIName = 'mi-apim-${resourceToken}' var logicAppMIName = 'mi-logicapp-${resourceToken}' // ===== Spoke Resource Naming ===== -var spokeFoundryAccountName = 'aif-spoke-${resourceToken}' +var spokeFoundryAccountName = 'aif-spoke-${resourceToken}${foundryNameSuffix}' var spokeFoundryProjectName = 'citadel-agents-project' var spokeLaName = 'law-spoke-${resourceToken}' var spokeAiName = 'appi-spoke-${resourceToken}' diff --git a/03-Azure/01-04-AI/06_AI_Governance/labautomation/infra/resources.json b/03-Azure/01-04-AI/06_AI_Governance/labautomation/infra/resources.json index 72d7da8cb..d4046b84b 100644 --- a/03-Azure/01-04-AI/06_AI_Governance/labautomation/infra/resources.json +++ b/03-Azure/01-04-AI/06_AI_Governance/labautomation/infra/resources.json @@ -5,7 +5,7 @@ "_generator": { "name": "bicep", "version": "0.41.2.15936", - "templateHash": "9591618861995214523" + "templateHash": "1413310816187321949" } }, "parameters": { @@ -23,6 +23,14 @@ "description": "Deterministic suffix for resource naming (DNS-safe, lowercase)." } }, + "foundryNameSuffix": { + "type": "string", + "defaultValue": "", + "maxLength": 6, + "metadata": { + "description": "Extra suffix appended to the two Azure AI Foundry account names only.\nAn `AIServices` account with `allowProjectManagement` is backed by a hidden AML\n(`AmlRp`) workspace. Deleting the account soft-deletes that workspace for ~14 days,\nand purging the Cognitive Services account does NOT purge it -- Azure exposes no API\nto list or purge it at all. The account name is therefore unusable for two weeks after\nany delete: recreating it always lands in `provisioningState: Failed` with\n\"Soft-deleted workspace exists. Please purge or recover it.\".\nBecause `resourceToken` is derived from subscription id + resource group name and the\nplatform reuses both, every re-run of an event would otherwise be guaranteed to fail.\n`labautomation/deploy-lab.ps1` supplies a fresh suffix whenever the previous account is\nnot healthy, and reuses the existing one when it is. Foundry accounts are the only\nresources affected: Key Vault and APIM purge correctly and keep their stable names." + } + }, "tags": { "type": "object", "defaultValue": {}, @@ -60,7 +68,7 @@ "$fxv#2": "\n \n (\"piiDeanonymizeContentInput\");\n \n // Pre-process content to replace common encoded symbols\n content = System.Text.RegularExpressions.Regex.Replace(content, @\"\\\\u003c\", \"<\");\n content = System.Text.RegularExpressions.Regex.Replace(content, @\"\\\\u003e\", \">\");\n \n var mappings = JArray.Parse(\n context.Variables.GetValueOrDefault(\"piiMappings\"));\n \n foreach (var mapping in mappings) {\n var original = mapping[\"original\"].ToString();\n var placeholder = mapping[\"placeholder\"].ToString();\n content = content.Replace(placeholder, original);\n }\n return content;\n }\" />\n", "$fxv#3": "\n \n \n (\"piiStateSavingEnabled\") == \"true\")\">\n \n \n \n \n \n \n \n \n \n \n (\"requestedModel\") ?? \"unknown\")\" />\n \n \n @{\n var eventData = new JObject();\n \n // Operation metadata\n eventData[\"id\"] = context.Variables.GetValueOrDefault(\"piiOperationId\");\n eventData[\"timestamp\"] = context.Variables.GetValueOrDefault(\"piiOperationTimestamp\");\n eventData[\"type\"] = \"PII_Processing\";\n eventData[\"targetDeployment\"] = context.Variables.GetValueOrDefault(\"targetDeployment\");\n // Enhanced API and subscription context with content\n eventData[\"context\"] = new JObject {\n [\"api\"] = new JObject {\n [\"id\"] = context.Variables.GetValueOrDefault(\"piiApiId\"),\n [\"name\"] = context.Variables.GetValueOrDefault(\"piiApiName\"),\n [\"operation\"] = new JObject {\n [\"id\"] = context.Operation.Id ?? \"unknown\",\n [\"name\"] = context.Variables.GetValueOrDefault(\"piiOperationName\"),\n [\"method\"] = context.Operation.Method ?? \"unknown\",\n [\"urlTemplate\"] = context.Operation.UrlTemplate ?? \"unknown\"\n },\n [\"content\"] = new JObject {\n [\"originalRequest\"] = context.Variables.GetValueOrDefault(\"piiInputContent\"),\n [\"anonymizedRequest\"] = context.Variables.GetValueOrDefault(\"piiAnonymizedContent\"),\n [\"originalResponse\"] = context.Variables.GetValueOrDefault(\"originalResponse\"),\n [\"deanonymizedResponse\"] = context.Variables.GetValueOrDefault(\"piiDeanonymizedContentOutput\")\n }\n },\n [\"product\"] = new JObject {\n [\"id\"] = context.Variables.GetValueOrDefault(\"piiProductId\"),\n [\"name\"] = context.Variables.GetValueOrDefault(\"piiProductName\")\n },\n [\"subscription\"] = new JObject {\n [\"id\"] = context.Variables.GetValueOrDefault(\"piiSubscriptionId\"),\n [\"name\"] = context.Variables.GetValueOrDefault(\"piiSubscriptionName\")\n }\n };\n \n // Process information\n eventData[\"process\"] = new JObject {\n [\"anonymizationEnabled\"] = context.Variables.GetValueOrDefault(\"piiAnonymizationEnabled\") == \"true\",\n [\"confidenceThreshold\"] = context.Variables.GetValueOrDefault(\"piiConfidenceThreshold\", 0.8),\n [\"entityCategoryExclusions\"] = context.Variables.GetValueOrDefault(\"piiEntityCategoryExclusions\", \"\"),\n [\"deanonymizationPerformed\"] = context.Variables.ContainsKey(\"piiDeanonymizedContentOutput\")\n };\n \n // Results summary\n var mappings = new JArray();\n if (context.Variables.ContainsKey(\"piiMappings\")) {\n try {\n mappings = JArray.Parse(context.Variables.GetValueOrDefault(\"piiMappings\"));\n eventData[\"entityCount\"] = mappings.Count;\n \n // Group entities by category for reporting\n var categories = new Dictionary();\n foreach (var mapping in mappings) {\n var placeholder = mapping[\"placeholder\"].ToString();\n var category = placeholder.TrimStart('<').Split('_')[0];\n \n if (!categories.ContainsKey(category)) {\n categories[category] = 0;\n }\n categories[category]++;\n }\n \n var categoryDetails = new JObject();\n foreach (var category in categories) {\n categoryDetails[category.Key] = category.Value;\n }\n eventData[\"entityCategories\"] = categoryDetails;\n \n // Include mappings for detailed audit (can be removed if too verbose)\n eventData[\"mappings\"] = mappings;\n }\n catch (Exception ex) {\n eventData[\"error\"] = \"Failed to process mappings: \" + ex.Message;\n }\n }\n \n // Content length metrics (not the actual content)\n if (context.Variables.ContainsKey(\"piiInputContent\")) {\n eventData[\"metrics\"] = new JObject {\n [\"inputContentLength\"] = context.Variables.GetValueOrDefault(\"piiInputContent\", \"\").Length\n };\n \n if (context.Variables.ContainsKey(\"piiAnonymizedContent\")) {\n eventData[\"metrics\"][\"anonymizedContentLength\"] = \n context.Variables.GetValueOrDefault(\"piiAnonymizedContent\", \"\").Length;\n }\n \n // Add deanonymization metrics\n if (context.Variables.ContainsKey(\"piiDeanonymizeContentInput\")) {\n eventData[\"metrics\"][\"deanonymizeInputLength\"] = \n context.Variables.GetValueOrDefault(\"piiDeanonymizeContentInput\", \"\").Length;\n }\n \n if (context.Variables.ContainsKey(\"piiDeanonymizedContentOutput\")) {\n eventData[\"metrics\"][\"deanonymizedOutputLength\"] = \n context.Variables.GetValueOrDefault(\"piiDeanonymizedContentOutput\", \"\").Length;\n }\n }\n \n // Add deanonymization flow details\n eventData[\"deanonymization\"] = new JObject {\n [\"performed\"] = context.Variables.ContainsKey(\"piiDeanonymizedContentOutput\"),\n [\"inputSource\"] = context.Variables.ContainsKey(\"piiDeanonymizeContentInput\") ? \"present\" : \"missing\",\n [\"status\"] = context.Variables.ContainsKey(\"piiDeanonymizedContentOutput\") ? \"completed\" : \"not_performed\",\n [\"timestamp\"] = DateTime.UtcNow.ToString(\"o\")\n };\n \n // Track end-to-end flow\n eventData[\"processingFlow\"] = new JObject {\n [\"anonymization\"] = new JObject {\n [\"performed\"] = context.Variables.ContainsKey(\"piiAnonymizedContent\"),\n [\"entityCount\"] = context.Variables.ContainsKey(\"piiMappings\") ? \n JArray.Parse(context.Variables.GetValueOrDefault(\"piiMappings\")).Count : 0\n },\n [\"deanonymization\"] = new JObject {\n [\"performed\"] = context.Variables.ContainsKey(\"piiDeanonymizedContentOutput\"),\n [\"success\"] = context.Variables.ContainsKey(\"piiDeanonymizedContentOutput\")\n }\n };\n \n return eventData.ToString();\n }\n \n \n\n", "$fxv#4": "\n \n \n \n \n \n \n \n application/json\n \n @{\n JArray models = JArray.Parse((string)context.Variables[\"availableFilteredModels\"]);\n return new JObject(new JProperty(\"object\", \"list\"), new JProperty(\"data\", models)).ToString();\n }\n \n \n \n \n \n\n", - "hubFoundryAccountName": "[format('aif-hub-{0}', parameters('resourceToken'))]", + "hubFoundryAccountName": "[format('aif-hub-{0}{1}', parameters('resourceToken'), parameters('foundryNameSuffix'))]", "hubFoundryProjectName": "citadel-hub-project", "apimName": "[format('apim-citadel-{0}', parameters('resourceToken'))]", "cosmosAccountName": "[format('cos-citadel-{0}', parameters('resourceToken'))]", @@ -73,7 +81,7 @@ "logicAppName": "[format('la-usage-ingestion-{0}', parameters('resourceToken'))]", "apimMIName": "[format('mi-apim-{0}', parameters('resourceToken'))]", "logicAppMIName": "[format('mi-logicapp-{0}', parameters('resourceToken'))]", - "spokeFoundryAccountName": "[format('aif-spoke-{0}', parameters('resourceToken'))]", + "spokeFoundryAccountName": "[format('aif-spoke-{0}{1}', parameters('resourceToken'), parameters('foundryNameSuffix'))]", "spokeFoundryProjectName": "citadel-agents-project", "spokeLaName": "[format('law-spoke-{0}', parameters('resourceToken'))]", "spokeAiName": "[format('appi-spoke-{0}', parameters('resourceToken'))]",