From 708848a2ba79567f06c3fa5fdfe83bc71080fb28 Mon Sep 17 00:00:00 2001 From: ajakupov-microsoft Date: Wed, 2 Sep 2026 20:23:33 +0200 Subject: [PATCH 1/3] Pin lab deployment to the RG's existing region and report the root-cause error The region-fallback loop assumed an empty resource group. When a lab is re-deployed into an RG that already holds a previous run, the first attempt pins every resource name to that region, so each subsequent candidate fails with "InvalidResourceLocation: mi-apim- already exists in location ". The retries can never succeed, and the final error reports only the last region's symptom, hiding the real preflight failure. Observed in the Sept-10 event: labuser-0005's RG still contained the previous event's stack, so a CognitiveServices preflight error in swedencentral was reported as "failed in all candidate regions: swedencentral, westeurope, norwayeast" with the actual cause discarded. - Pin to the region the RG already uses when it is non-empty, preferring one the platform asked for. 'global' resources are ignored, since they are not deployable regions; an RG holding only global resources still gets the full fallback list. - Carry the first failure into the final throw, as later regions report downstream symptoms rather than the root cause. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 05a828d3-c050-4705-9340-6c65646dfb58 --- .../labautomation/deploy-lab.ps1 | 32 ++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/03-Azure/01-04-AI/06_AI_Governance/labautomation/deploy-lab.ps1 b/03-Azure/01-04-AI/06_AI_Governance/labautomation/deploy-lab.ps1 index 463160d8e..1e5647b30 100644 --- a/03-Azure/01-04-AI/06_AI_Governance/labautomation/deploy-lab.ps1 +++ b/03-Azure/01-04-AI/06_AI_Governance/labautomation/deploy-lab.ps1 @@ -127,6 +127,33 @@ Write-Host "[INFO] Engine: $templateKind, resource-group-scoped." $deployOutputs = $null $effectiveLocation = $null +$firstFailure = $null + +# Region fallback only makes sense for an EMPTY resource group. Once the RG holds +# resources, their names are already pinned to the region of the first attempt, so every +# later candidate fails with 'InvalidResourceLocation: already exists in location +# ' — which masks the real error and cannot ever succeed. Re-running into a +# non-empty RG (a re-deployed event, or a retry after a partial failure) must therefore +# stay in the region that RG already uses. +$existingRegions = @( + Get-AzResource -ResourceGroupName $effectiveRG -ErrorAction SilentlyContinue | + Select-Object -ExpandProperty Location -ErrorAction SilentlyContinue | + Where-Object { $_ -and $_ -ne 'global' } | + Group-Object | + Sort-Object Count -Descending | + Select-Object -ExpandProperty Name +) +if ($existingRegions.Count -gt 0) { + # Prefer a region the platform actually asked for; otherwise the most common region + # already used in the RG. 'global' resources are ignored — they are not deployable regions. + $pinnedRegion = $candidateRegions | Where-Object { $existingRegions -contains $_ } | Select-Object -First 1 + if (-not $pinnedRegion) { $pinnedRegion = $existingRegions[0] } + if ($candidateRegions.Count -gt 1) { + Write-Host "[INFO] RG '$effectiveRG' already contains resources in '$($existingRegions -join ", ")'." + Write-Host "[INFO] Pinning deployment to '$pinnedRegion' (region fallback applies to empty RGs only)." + } + $candidateRegions = @($pinnedRegion) +} foreach ($region in $candidateRegions) { Write-Host "[INFO] Deploying → RG '$effectiveRG' in '$region' (token '$resourceToken')..." @@ -153,12 +180,15 @@ foreach ($region in $candidateRegions) { if (-not $retryable) { throw "Deployment failed in '$region' with a non-retryable error (retrying other regions would fail the same way): $_" } + # Keep the FIRST failure: it is the real one. Later regions typically fail with a + # downstream symptom, so reporting only the last error hides the actual cause. + if (-not $firstFailure) { $firstFailure = "in '$region': $_" } Write-Host "[WARN] Deployment failed in '$region': $_ — trying next region." } } if (-not $deployOutputs) { - throw "Deployment failed in all candidate regions: $($candidateRegions -join ', ')" + throw "Deployment failed in all candidate regions ($($candidateRegions -join ', ')). First (root cause) error $firstFailure" } Write-Host "[OK] Provisioning complete in '$effectiveLocation' (resource group '$effectiveRG')." From 711babfe2c949d47bcd0215169721733e78ea634 Mon Sep 17 00:00:00 2001 From: ajakupov-microsoft Date: Wed, 2 Sep 2026 21:18:49 +0200 Subject: [PATCH 2/3] Purge soft-deleted Foundry accounts and Key Vaults before deploying resourceToken is derived from subscription id + resource group name, and the platform reuses both when an event is torn down and a new one is deployed over it. Foundry accounts and Key Vaults are soft deleted rather than removed, and a soft-deleted resource keeps its name reserved -- for Foundry, also its globally unique customSubDomainName. Re-deploying therefore fails during preflight with an opaque "Microsoft.CognitiveServices/accounts (2025-04-01-preview) reported preflight validation errors" before any resource is touched. This took out all three labs on the Sept-10 re-run (event 004). The same two labs had deployed cleanly an hour earlier (event 003) when their resource groups were still empty, which is why it read as an environment problem rather than a template one. Purge any soft-deleted Foundry account or Key Vault whose name ends in this lab's token before deploying. Matching on the token means labs sharing a subscription are never touched, and a purge failure stays non-fatal so the deployment still surfaces the underlying preflight error. Also corrects the README's "re-runs skip existing resources" claim, which held only for resources that are not soft deleted. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 05a828d3-c050-4705-9340-6c65646dfb58 --- .../06_AI_Governance/labautomation/README.md | 13 ++++- .../labautomation/deploy-lab.ps1 | 48 +++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/03-Azure/01-04-AI/06_AI_Governance/labautomation/README.md b/03-Azure/01-04-AI/06_AI_Governance/labautomation/README.md index 7e74fc233..2524ab00a 100644 --- a/03-Azure/01-04-AI/06_AI_Governance/labautomation/README.md +++ b/03-Azure/01-04-AI/06_AI_Governance/labautomation/README.md @@ -163,12 +163,23 @@ The script follows MicroHack conventions: - **Region fallback** — Retries across `$PreferredLocation` (honoring platform preference), falling back to [swedencentral, westeurope, norwayeast] if all fail. Retries are gated on `Test-MhhDeploymentFailureRetryable`: a non-retryable failure (template or permission bug) - aborts immediately with the real error rather than being retried in every region. + aborts immediately with the real error rather than being retried in every region. Fallback + applies to **empty resource groups only** — once an RG holds resources their names are + pinned to that region, so any other candidate would fail with `InvalidResourceLocation` and + bury the real error. If all regions fail, the **first** (root cause) error is reported. - **Deterministic naming** — `Get-MhhStableHash` generates a stable suffix per attendee, ensuring names are DNS-safe, globally unique, and consistent across re-runs. The hash is computed **once** and passed to Bicep as `resourceToken`; the template is the single source of truth for every resource name. - **Idempotent provisioning** — All resources are created once; re-runs skip existing resources. + Because `resourceToken` is derived from subscription id + RG name — both of which the + platform reuses when an event is torn down and re-deployed — the script first **purges + soft-deleted Foundry accounts and Key Vaults** whose names end in this lab's token. A + soft-deleted resource keeps its name reserved (for Foundry, also its global + `customSubDomainName`), so without this a re-deployed event fails during preflight with an + opaque `Microsoft.CognitiveServices/accounts ... reported preflight validation errors` + before any resource is touched. Only this lab's token is matched, so other labs sharing the + subscription are never affected. - **Managed identities** — No hardcoded keys or connection strings; all auth uses Entra ID + Azure RBAC (DefaultAzureCredential in notebooks). - **Multi-user RBAC** — Every ID in `$AllowedEntraUserIds` is granted the necessary data-plane diff --git a/03-Azure/01-04-AI/06_AI_Governance/labautomation/deploy-lab.ps1 b/03-Azure/01-04-AI/06_AI_Governance/labautomation/deploy-lab.ps1 index 1e5647b30..bb87a33c4 100644 --- a/03-Azure/01-04-AI/06_AI_Governance/labautomation/deploy-lab.ps1 +++ b/03-Azure/01-04-AI/06_AI_Governance/labautomation/deploy-lab.ps1 @@ -125,6 +125,54 @@ if ($AllowedEntraUserIds.Count -eq 0) { Write-Host "[INFO] Deploying Citadel Agentic Governance Hub + Spoke into RG '$effectiveRG'..." Write-Host "[INFO] Engine: $templateKind, resource-group-scoped." +# ===== Purge soft-deleted namesakes left by a previous event ===== +# $resourceToken is derived from subscription id + RG name, and the platform reuses BOTH +# when an event is torn down and a new one is deployed over it. Foundry accounts and Key +# Vaults are soft deleted rather than removed, and a soft-deleted resource keeps its name +# reserved — for Foundry, also its globally unique customSubDomainName. Re-deploying then +# fails during preflight with an opaque 'Microsoft.CognitiveServices/accounts ... reported +# preflight validation errors' before any resource is touched, which is what took out all +# three labs on the Sept-10 re-run. +# +# Only names ending in THIS lab's token are purged, so other labs sharing the subscription +# are never affected. A purge failure is non-fatal: it is reported, and the deployment +# below surfaces the resulting preflight error. +$softDeletedScopes = @( + @{ Label = 'Foundry account'; ApiVersion = '2023-05-01' + ListPath = "/subscriptions/$SubscriptionId/providers/Microsoft.CognitiveServices/deletedAccounts?api-version=2023-05-01" }, + @{ Label = 'Key Vault'; ApiVersion = '2022-07-01' + ListPath = "/subscriptions/$SubscriptionId/providers/Microsoft.KeyVault/deletedVaults?api-version=2022-07-01" } +) + +foreach ($scope in $softDeletedScopes) { + try { + $listResponse = Invoke-AzRestMethod -Method GET -Path $scope.ListPath -ErrorAction Stop + if ($listResponse.StatusCode -ne 200) { + Write-Host "[WARN] Could not list soft-deleted $($scope.Label)s (HTTP $($listResponse.StatusCode)) — skipping purge check." + continue + } + # The deleted-resource id already encodes location and original RG, so purging by + # id avoids reconstructing a scope that may differ from the current deployment. + $stale = @(($listResponse.Content | ConvertFrom-Json).value | + Where-Object { $_.name -and $_.name.EndsWith($resourceToken) }) + + foreach ($item in $stale) { + Write-Host "[INFO] Purging soft-deleted $($scope.Label) '$($item.name)' from a previous event..." + $purge = Invoke-AzRestMethod -Method DELETE -Path "$($item.id)?api-version=$($scope.ApiVersion)" -ErrorAction Stop + if (@(200, 202, 204) -contains $purge.StatusCode) { + Write-Host "[OK] Purged '$($item.name)'." + } + else { + Write-Host "[WARN] Failed to purge '$($item.name)' (HTTP $($purge.StatusCode)). Deployment will likely" + Write-Host "[WARN] fail preflight because the name is still reserved. Response: $($purge.Content)" + } + } + } + catch { + Write-Host "[WARN] Soft-deleted $($scope.Label) check skipped: $_" + } +} + $deployOutputs = $null $effectiveLocation = $null $firstFailure = $null From b5f46a96ac8adfa231575710d602fd78cda53dc6 Mon Sep 17 00:00:00 2001 From: ajakupov-microsoft Date: Wed, 2 Sep 2026 22:14:31 +0200 Subject: [PATCH 3/3] Fix lab redeployment failures caused by unpurgeable Foundry soft-delete Re-deploying an event into a reused resource group failed for every participant (Sept-10 event 004). Root cause, reproduced and confirmed locally: An `AIServices` account with `allowProjectManagement` is backed by a hidden Azure ML (`AmlRp`) workspace. Deleting the account soft-deletes that workspace for ~14 days. Purging the Cognitive Services account does NOT purge it, and Azure exposes no API to list or purge it (`deletedWorkspaces` 404s on every api-version; `forceToPurge` DELETEs are no-ops). Recreating the account therefore always lands in `provisioningState: Failed` with "Soft-deleted workspace exists", which in turn poisons the resource group permanently. Since `resourceToken` is derived from subscription id + RG name and the platform reuses both, every re-run was guaranteed to fail. Proven by isolation test: purge + recreate under the same name failed three times; a bare account under a fresh name succeeded. - Foundry accounts now take a `foundryNameSuffix`. The script reuses the existing accounts while they are healthy, and otherwise deploys under a fresh name, recording the suffix as the `citadelFoundrySuffix` resource-group tag so re-runs stay stable. - Recycle Foundry accounts stuck in `Failed` state, which ARM otherwise refuses to update ('AccountIsNotSucceeded'). - Wait for purges to propagate before deploying; purging is asynchronous, and recreating a name too early was itself creating the Failed accounts. - Fix the Key Vault purge verb (POST .../purge; DELETE returns 405) and add the missing APIM purge scope ('ServiceAlreadyExistsInSoftDeletedState'). - Retry Azure REST calls: a transient TLS error silently skipped the rest of the purge pass, and per-item isolation stops one failure skipping the others. - Retry a deployment in place up to 3 times for genuinely transient states (APIM 'ServiceLocked', AML soft-delete lag), re-running cleanup before each attempt. - Document the extra Foundry suffix in challenge-01 (attendees now read the two account names directly instead of building them from the token) and in the lab README. Verified end-to-end: full local deployment completes with 6 model deployments, both Foundry projects and all RBAC grants succeeding. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 05a828d3-c050-4705-9340-6c65646dfb58 --- .../challenges/challenge-01.md | 30 +- .../06_AI_Governance/labautomation/README.md | 40 ++- .../labautomation/deploy-lab.ps1 | 292 ++++++++++++++---- .../labautomation/infra/resources.bicep | 19 +- .../labautomation/infra/resources.json | 14 +- 5 files changed, 311 insertions(+), 84 deletions(-) diff --git a/03-Azure/01-04-AI/06_AI_Governance/challenges/challenge-01.md b/03-Azure/01-04-AI/06_AI_Governance/challenges/challenge-01.md index 483196695..87eb402c7 100644 --- a/03-Azure/01-04-AI/06_AI_Governance/challenges/challenge-01.md +++ b/03-Azure/01-04-AI/06_AI_Governance/challenges/challenge-01.md @@ -52,16 +52,27 @@ end of the event. Every resource in your lab is named with the same random token, and the `.env` values are built from it. In the portal, open your resource group (`rg-labuser-00XX`) and find -the AI Foundry account named **`aif-hub-`** — everything after the prefix is your -token. For example `aif-hub-fd79f0b6fee89` means the token is `fd79f0b6fee89`. +the API Management service named **`apim-citadel-`** — everything after the prefix +is your token. For example `apim-citadel-fd79f0b6fee89` means the token is +`fd79f0b6fee89`. From the CLI: ```bash -az cognitiveservices account list -g \ - --query "[?starts_with(name,'aif-hub-')].name" -o tsv +az apim list -g --query "[].name" -o tsv ``` +> [!IMPORTANT] +> The two AI Foundry accounts are the one exception: they carry a few extra random +> characters after the token (for example `aif-hub-fd79f0b6fee89-k3xq`). Azure reserves a +> deleted Foundry account's name for two weeks, so the lab gives them a fresh name on every +> deployment. Copy those two names **exactly as they appear** rather than building them +> from the token: +> +> ```bash +> az cognitiveservices account list -g --query "[].name" -o tsv +> ``` + #### A.3 — Fill in `.env` ```bash @@ -71,7 +82,7 @@ cp .env.template .env Open `.env` and set the values below, replacing `` throughout. The first three come straight from the dashboard / portal Overview blade; the rest are derived from the -token. +token, except the two Foundry account names you copied in A.2. ```bash AZURE_RESOURCE_GROUP=rg-labuser-00XX @@ -80,11 +91,11 @@ AZURE_SUBSCRIPTION_ID= SPOKE_RESOURCE_GROUP=rg-labuser-00XX # same RG — this lab uses one resource group SPOKE_KEY_VAULT_NAME=kv-spoke- -SPOKE_AI_FOUNDRY_ACCOUNT_NAME=aif-spoke- +SPOKE_AI_FOUNDRY_ACCOUNT_NAME= # the full aif-spoke-... name SPOKE_AI_FOUNDRY_PROJECT_NAME=citadel-agents-project SPOKE_ACR_NAME=acrcitadel SPOKE_ACR_LOGIN_SERVER=acrcitadel.azurecr.io -A2A_FOUNDRY_ACCOUNT_NAME=aif-spoke- +A2A_FOUNDRY_ACCOUNT_NAME= # same value as above ``` Challenges 1-6 need only the first four values plus `LLM_BACKEND_CONFIG`. @@ -92,10 +103,11 @@ Challenges 1-6 need only the first four values plus `LLM_BACKEND_CONFIG`. #### A.4 — Fill in `LLM_BACKEND_CONFIG` This one is a single line of JSON describing your gateway's backend and its six models. -Paste the block below as **one line**, with `` replaced in **both** places: +Paste the block below as **one line**, replacing `` in **both** +places with the full `aif-hub-...` name you copied in A.2: ``` -LLM_BACKEND_CONFIG=[{"backendId":"aif-hub-","backendType":"ai-foundry","endpoint":"https://aif-hub-.openai.azure.com","authScheme":"managedIdentity","authType":"managed-identity","priority":1,"weight":100,"supportedModels":[{"name":"gpt-4.1","sku":"GlobalStandard","capacity":20,"modelFormat":"OpenAI","modelVersion":"2025-04-14","retirementDate":"2027-04-14"},{"name":"gpt-5.4-mini","sku":"GlobalStandard","capacity":20,"modelFormat":"OpenAI","modelVersion":"2026-03-17","retirementDate":"2027-09-21"},{"name":"gpt-5.2","sku":"GlobalStandard","capacity":20,"modelFormat":"OpenAI","modelVersion":"2025-12-11","retirementDate":"2027-06-08"},{"name":"text-embedding-3-large","sku":"GlobalStandard","capacity":20,"modelFormat":"OpenAI","modelVersion":"1","retirementDate":"2028-02-09"},{"name":"Mistral-Large-3","sku":"GlobalStandard","capacity":20,"modelFormat":"Mistral AI","modelVersion":"1","retirementDate":"2099-12-31"},{"name":"Phi-4","sku":"GlobalStandard","capacity":1,"modelFormat":"Microsoft","modelVersion":"7","retirementDate":"2099-12-31"}]}] +LLM_BACKEND_CONFIG=[{"backendId":"","backendType":"ai-foundry","endpoint":"https://.openai.azure.com","authScheme":"managedIdentity","authType":"managed-identity","priority":1,"weight":100,"supportedModels":[{"name":"gpt-4.1","sku":"GlobalStandard","capacity":20,"modelFormat":"OpenAI","modelVersion":"2025-04-14","retirementDate":"2027-04-14"},{"name":"gpt-5.4-mini","sku":"GlobalStandard","capacity":20,"modelFormat":"OpenAI","modelVersion":"2026-03-17","retirementDate":"2027-09-21"},{"name":"gpt-5.2","sku":"GlobalStandard","capacity":20,"modelFormat":"OpenAI","modelVersion":"2025-12-11","retirementDate":"2027-06-08"},{"name":"text-embedding-3-large","sku":"GlobalStandard","capacity":20,"modelFormat":"OpenAI","modelVersion":"1","retirementDate":"2028-02-09"},{"name":"Mistral-Large-3","sku":"GlobalStandard","capacity":20,"modelFormat":"Mistral AI","modelVersion":"1","retirementDate":"2099-12-31"},{"name":"Phi-4","sku":"GlobalStandard","capacity":1,"modelFormat":"Microsoft","modelVersion":"7","retirementDate":"2099-12-31"}]}] ``` > [!IMPORTANT] diff --git a/03-Azure/01-04-AI/06_AI_Governance/labautomation/README.md b/03-Azure/01-04-AI/06_AI_Governance/labautomation/README.md index 2524ab00a..4c39f91c0 100644 --- a/03-Azure/01-04-AI/06_AI_Governance/labautomation/README.md +++ b/03-Azure/01-04-AI/06_AI_Governance/labautomation/README.md @@ -14,7 +14,7 @@ Adapted from the Citadel workshop (`azd up`), the lab deploys: | Resource | Purpose | |----------|---------| | **API Management (StandardV2)** | Unified AI gateway; all model calls route through APIM | -| **Azure AI Foundry Account (Hub)** | Primary Foundry account (`aif-hub-`) for governance backend | +| **Azure AI Foundry Account (Hub)** | Primary Foundry account (`aif-hub-`) for governance backend | | **Azure AI Foundry Project (Hub)** | Hosts hub-level agents and access-control demonstrations | | **Log Analytics Workspace** | Hub observability backend | | **Application Insights** | Connected to hub Foundry for agent tracing | @@ -27,7 +27,7 @@ Adapted from the Citadel workshop (`azd up`), the lab deploys: ### Spoke Resources (Folded into Same RG) | Resource | Purpose | |----------|---------| -| **Azure AI Foundry Account (Spoke)** | Sample workload Foundry account (`aif-spoke-`) for attendee agent development | +| **Azure AI Foundry Account (Spoke)** | Sample workload Foundry account (`aif-spoke-`) for attendee agent development | | **Azure AI Foundry Project (Spoke)** | Where attendees deploy their governed agents (`citadel-agents-project`) | | **Log Analytics Workspace (Spoke)** | Spoke observability | | **Application Insights (Spoke)** | Spoke agent tracing | @@ -65,7 +65,7 @@ infix plus the shared `resourceToken`. | Workshop `deploy-spoke-foundry.ps1` step | Equivalent here | |---|---| | Create spoke resource group | *Not applicable* — hub and spoke share the participant's single RG | -| Foundry account (`allowProjectManagement`, system-assigned identity) | `spokeFoundryAccount` → `aif-spoke-` | +| Foundry account (`allowProjectManagement`, system-assigned identity) | `spokeFoundryAccount` → `aif-spoke-` | | Foundry project (system-assigned identity) | `spokeFoundryProject` → `citadel-agents-project` | | Log Analytics workspace | `laSpoke` → `law-spoke-` | | Application Insights | `aiSpoke` → `appi-spoke-` | @@ -171,15 +171,31 @@ The script follows MicroHack conventions: ensuring names are DNS-safe, globally unique, and consistent across re-runs. The hash is computed **once** and passed to Bicep as `resourceToken`; the template is the single source of truth for every resource name. -- **Idempotent provisioning** — All resources are created once; re-runs skip existing resources. - Because `resourceToken` is derived from subscription id + RG name — both of which the - platform reuses when an event is torn down and re-deployed — the script first **purges - soft-deleted Foundry accounts and Key Vaults** whose names end in this lab's token. A - soft-deleted resource keeps its name reserved (for Foundry, also its global - `customSubDomainName`), so without this a re-deployed event fails during preflight with an - opaque `Microsoft.CognitiveServices/accounts ... reported preflight validation errors` - before any resource is touched. Only this lab's token is matched, so other labs sharing the - subscription are never affected. +- **Idempotent provisioning** — Re-runs reuse healthy resources. Because `resourceToken` is + derived from subscription id + RG name — both of which the platform reuses when an event is + torn down and re-deployed — the script first **purges soft-deleted Key Vaults and APIM + services** whose names contain this lab's token, and **recycles Foundry accounts left in a + `Failed` state**. A soft-deleted resource keeps its name reserved, so without this a + re-deployed event fails during preflight with an opaque + `Microsoft.CognitiveServices/accounts ... reported preflight validation errors` before any + resource is touched. Purges are asynchronous, so the script then waits for the names to + actually disappear before deploying. Only this lab's token is matched, so other labs sharing + the subscription are never affected. +- **Fresh Foundry account names** — Foundry is the one resource that purging cannot rescue. An + `AIServices` account with `allowProjectManagement` is backed by a hidden Azure ML (`AmlRp`) + workspace; deleting the account soft-deletes that workspace for ~14 days, purging the + Cognitive Services account does **not** purge it, and Azure exposes no API to list or purge + it. Recreating the account then always lands in `provisioningState: Failed` with + *"Soft-deleted workspace exists. Please purge or recover it."* — which is what took out every + lab on the Sept-10 re-run. The script therefore reuses the existing Foundry accounts only + while they are healthy, and otherwise deploys under a **fresh name** + (`aif-hub-`), recording the suffix as a resource-group tag + (`citadelFoundrySuffix`) so repeat runs stay stable. This is the only reason attendees must + copy the two Foundry account names rather than build them from the token. +- **Transient-failure retry** — A deployment is retried up to three times in the same region + for genuinely transient states (`ServiceLocked` while APIM is still activating, AML + soft-delete lag), re-running the cleanup before each attempt so a retry never trips over the + wreckage of the previous one. - **Managed identities** — No hardcoded keys or connection strings; all auth uses Entra ID + Azure RBAC (DefaultAzureCredential in notebooks). - **Multi-user RBAC** — Every ID in `$AllowedEntraUserIds` is granted the necessary data-plane diff --git a/03-Azure/01-04-AI/06_AI_Governance/labautomation/deploy-lab.ps1 b/03-Azure/01-04-AI/06_AI_Governance/labautomation/deploy-lab.ps1 index bb87a33c4..a1875ca99 100644 --- a/03-Azure/01-04-AI/06_AI_Governance/labautomation/deploy-lab.ps1 +++ b/03-Azure/01-04-AI/06_AI_Governance/labautomation/deploy-lab.ps1 @@ -127,50 +127,193 @@ Write-Host "[INFO] Engine: $templateKind, resource-group-scoped." # ===== Purge soft-deleted namesakes left by a previous event ===== # $resourceToken is derived from subscription id + RG name, and the platform reuses BOTH -# when an event is torn down and a new one is deployed over it. Foundry accounts and Key -# Vaults are soft deleted rather than removed, and a soft-deleted resource keeps its name -# reserved — for Foundry, also its globally unique customSubDomainName. Re-deploying then -# fails during preflight with an opaque 'Microsoft.CognitiveServices/accounts ... reported -# preflight validation errors' before any resource is touched, which is what took out all -# three labs on the Sept-10 re-run. +# when an event is torn down and a new one is deployed over it. Foundry accounts, Key +# Vaults and APIM services are soft deleted rather than removed, and a soft-deleted resource +# keeps its name reserved — for Foundry, also its globally unique customSubDomainName. +# Re-deploying then fails with 'FlagMustBeSetForRestore' during preflight (Foundry, before +# any resource is touched), 'ConflictError' (Key Vault) or +# 'ServiceAlreadyExistsInSoftDeletedState' (APIM), which is what took out all three labs on +# the Sept-10 re-run. # # Only names ending in THIS lab's token are purged, so other labs sharing the subscription # are never affected. A purge failure is non-fatal: it is reported, and the deployment # below surfaces the resulting preflight error. -$softDeletedScopes = @( - @{ Label = 'Foundry account'; ApiVersion = '2023-05-01' - ListPath = "/subscriptions/$SubscriptionId/providers/Microsoft.CognitiveServices/deletedAccounts?api-version=2023-05-01" }, - @{ Label = 'Key Vault'; ApiVersion = '2022-07-01' - ListPath = "/subscriptions/$SubscriptionId/providers/Microsoft.KeyVault/deletedVaults?api-version=2022-07-01" } -) - -foreach ($scope in $softDeletedScopes) { +# Wrapped in a function because it must run again before EVERY deployment attempt: a failed +# attempt leaves Foundry accounts in a Failed state and freshly soft-deleted namesakes behind, +# so retrying without cleaning up first fails preflight on the wreckage of the previous try. +function Invoke-LabSoftDeleteCleanup { + # A Foundry account left in a Failed provisioning state by an earlier partial deployment is + # just as blocking: ARM refuses to update it ('AccountIsNotSucceeded'), so the resource group + # would stay permanently un-deployable. Deleting it soft-deletes it, and the purge pass below + # then frees the name for this run to recreate. try { - $listResponse = Invoke-AzRestMethod -Method GET -Path $scope.ListPath -ErrorAction Stop - if ($listResponse.StatusCode -ne 200) { - Write-Host "[WARN] Could not list soft-deleted $($scope.Label)s (HTTP $($listResponse.StatusCode)) — skipping purge check." - continue + $failedAccounts = @(Get-AzResource -ResourceGroupName $effectiveRG ` + -ResourceType 'Microsoft.CognitiveServices/accounts' -ExpandProperties -ErrorAction SilentlyContinue | + Where-Object { $_.Name -and $_.Name.Contains($resourceToken) -and $_.Properties.provisioningState -eq 'Failed' }) + foreach ($account in $failedAccounts) { + Write-Host "[INFO] Foundry account '$($account.Name)' is in a Failed state from an earlier run — recycling it..." + Remove-AzResource -ResourceId $account.ResourceId -Force -ErrorAction Stop | Out-Null + Write-Host "[OK] Deleted '$($account.Name)'; it will be purged and recreated below." } - # The deleted-resource id already encodes location and original RG, so purging by - # id avoids reconstructing a scope that may differ from the current deployment. - $stale = @(($listResponse.Content | ConvertFrom-Json).value | - Where-Object { $_.name -and $_.name.EndsWith($resourceToken) }) - - foreach ($item in $stale) { - Write-Host "[INFO] Purging soft-deleted $($scope.Label) '$($item.name)' from a previous event..." - $purge = Invoke-AzRestMethod -Method DELETE -Path "$($item.id)?api-version=$($scope.ApiVersion)" -ErrorAction Stop - if (@(200, 202, 204) -contains $purge.StatusCode) { - Write-Host "[OK] Purged '$($item.name)'." + } + catch { + Write-Host "[WARN] Could not recycle failed Foundry accounts: $_" + } + + $softDeletedScopes = @( + @{ Label = 'Foundry account'; ApiVersion = '2023-05-01' + ListPath = "/subscriptions/$SubscriptionId/providers/Microsoft.CognitiveServices/deletedAccounts?api-version=2023-05-01" + # Cognitive Services purges via DELETE on the deleted-account resource itself. + PurgeMethod = 'DELETE'; PurgeSuffix = '' }, + @{ Label = 'Key Vault'; ApiVersion = '2022-07-01' + ListPath = "/subscriptions/$SubscriptionId/providers/Microsoft.KeyVault/deletedVaults?api-version=2022-07-01" + # Key Vault has no DELETE on the deleted vault (that returns HTTP 405) — it exposes a + # dedicated POST .../purge action instead. + PurgeMethod = 'POST'; PurgeSuffix = '/purge' }, + @{ Label = 'APIM service'; ApiVersion = '2022-08-01' + ListPath = "/subscriptions/$SubscriptionId/providers/Microsoft.ApiManagement/deletedservices?api-version=2022-08-01" + # APIM soft-deletes too, failing the deployment with ServiceAlreadyExistsInSoftDeletedState. + PurgeMethod = 'DELETE'; PurgeSuffix = '' } + ) + + # Azure REST calls occasionally fail with transient TLS/socket errors. Without a retry a + # single blip silently skips the rest of the purge pass and the deployment then fails on a + # name that was supposed to have been cleared. + function Invoke-LabAzRest { + param([Parameter(Mandatory)][string]$Method, [Parameter(Mandatory)][string]$Path, [int]$MaxAttempts = 3) + for ($attempt = 1; $attempt -le $MaxAttempts; $attempt++) { + try { + return Invoke-AzRestMethod -Method $Method -Path $Path -ErrorAction Stop } - else { - Write-Host "[WARN] Failed to purge '$($item.name)' (HTTP $($purge.StatusCode)). Deployment will likely" - Write-Host "[WARN] fail preflight because the name is still reserved. Response: $($purge.Content)" + catch { + if ($attempt -eq $MaxAttempts) { throw } + $reason = "$($_.Exception.Message)".Split("`n")[0] + Write-Host "[WARN] $Method call failed (attempt $attempt/$MaxAttempts): $reason — retrying..." + Start-Sleep -Seconds (5 * $attempt) } } } + + $purgedAnything = $false + + foreach ($scope in $softDeletedScopes) { + try { + $listResponse = Invoke-LabAzRest -Method GET -Path $scope.ListPath + if ($listResponse.StatusCode -ne 200) { + Write-Host "[WARN] Could not list soft-deleted $($scope.Label)s (HTTP $($listResponse.StatusCode)) — skipping purge check." + continue + } + # The deleted-resource id already encodes location and original RG, so purging by + # id avoids reconstructing a scope that may differ from the current deployment. + $stale = @(($listResponse.Content | ConvertFrom-Json).value | + Where-Object { $_.name -and $_.name.Contains($resourceToken) }) + + foreach ($item in $stale) { + # Isolated per item: one failure must not skip the remaining purges. + try { + Write-Host "[INFO] Purging soft-deleted $($scope.Label) '$($item.name)' from a previous event..." + $purge = Invoke-LabAzRest -Method $scope.PurgeMethod ` + -Path "$($item.id)$($scope.PurgeSuffix)?api-version=$($scope.ApiVersion)" + if (@(200, 202, 204) -contains $purge.StatusCode) { + Write-Host "[OK] Purged '$($item.name)'." + $purgedAnything = $true + } + else { + Write-Host "[WARN] Failed to purge '$($item.name)' (HTTP $($purge.StatusCode)). Deployment will likely" + Write-Host "[WARN] fail preflight because the name is still reserved. Response: $($purge.Content)" + } + } + catch { + Write-Host "[WARN] Failed to purge '$($item.name)': $_" + } + } + } + catch { + Write-Host "[WARN] Soft-deleted $($scope.Label) check skipped: $_" + } + } + + # Purging is eventually consistent, and Key Vault/APIM purges are outright asynchronous. + # Recreating a name before the purge has propagated fails with a bare 'Failed to Create the + # resource. Provisioning state: Failed', which then leaves the account in a Failed state that + # blocks every later run. Wait for the names to actually disappear before deploying. + if ($purgedAnything) { + Write-Host "[INFO] Waiting for purges to propagate before deploying..." + $purgeDeadline = (Get-Date).AddMinutes(5) + do { + Start-Sleep -Seconds 10 + $stillPresent = 0 + foreach ($scope in $softDeletedScopes) { + try { + $check = Invoke-LabAzRest -Method GET -Path $scope.ListPath + if ($check.StatusCode -eq 200) { + $stillPresent += @(($check.Content | ConvertFrom-Json).value | + Where-Object { $_.name -and $_.name.Contains($resourceToken) }).Count + } + } + catch { } + } + } while ($stillPresent -gt 0 -and (Get-Date) -lt $purgeDeadline) + + if ($stillPresent -gt 0) { + Write-Host "[WARN] $stillPresent soft-deleted resource(s) still present after 5 minutes." + Write-Host "[WARN] Continuing anyway — the deployment below will report the conflict." + } + else { + # The name index lags the deleted-resource list by a few seconds. + Start-Sleep -Seconds 20 + Write-Host "[OK] All purges propagated." + } + } +} + +Invoke-LabSoftDeleteCleanup + +# ===== Pick a usable Azure AI Foundry account name ===== +# Deleting an `AIServices` account soft-deletes the hidden AML workspace behind it for ~14 +# days, and that workspace cannot be purged -- Azure exposes no API for it. The account name +# is therefore burned: recreating it always lands in `provisioningState: Failed` with +# "Soft-deleted workspace exists". Since $resourceToken is deterministic and the platform +# reuses subscription + RG name, a re-run would be guaranteed to fail on the old name. +# +# So: reuse the current account only while it is healthy, and otherwise move to a brand-new +# name. The chosen suffix is stored as a resource-group tag so that repeat runs stay stable +# and do not strand a working deployment behind a new set of names. +$foundrySuffixTag = 'citadelFoundrySuffix' + +function Resolve-LabFoundrySuffix { + $suffix = '' + try { + $rgTags = (Get-AzResourceGroup -Name $effectiveRG -ErrorAction Stop).Tags + if ($rgTags -and $rgTags[$foundrySuffixTag]) { $suffix = [string]$rgTags[$foundrySuffixTag] } + } + catch { } + + if ($suffix) { + $hub = Get-AzResource -ResourceGroupName $effectiveRG -Name "aif-hub-$resourceToken$suffix" ` + -ResourceType 'Microsoft.CognitiveServices/accounts' -ExpandProperties -ErrorAction SilentlyContinue + if ($hub -and $hub.Properties.provisioningState -eq 'Succeeded') { + Write-Host "[OK] Reusing healthy Foundry accounts 'aif-*-$resourceToken$suffix'." + return $suffix + } + } + + # No healthy account: the previous name (if any) is now unusable, so mint a fresh one. + # An empty RG is treated the same way — the RG may have been deleted and recreated, which + # leaves the soft-deleted workspace behind while removing all visible trace of it. + $new = -join ((1..4) | ForEach-Object { 'abcdefghijklmnopqrstuvwxyz0123456789'[(Get-Random -Maximum 36)] }) + Write-Host "[INFO] No healthy Foundry account found — using fresh account names 'aif-*-$resourceToken$new'." + Write-Host "[INFO] (A previously deleted account keeps its name reserved by an unpurgeable AML workspace.)" + try { + $existingTags = (Get-AzResourceGroup -Name $effectiveRG -ErrorAction Stop).Tags + if (-not $existingTags) { $existingTags = @{} } + $existingTags[$foundrySuffixTag] = $new + Set-AzResourceGroup -Name $effectiveRG -Tag $existingTags -ErrorAction Stop | Out-Null + } catch { - Write-Host "[WARN] Soft-deleted $($scope.Label) check skipped: $_" + Write-Host "[WARN] Could not record the Foundry suffix on the resource group: $_" } + return $new } $deployOutputs = $null @@ -204,35 +347,68 @@ if ($existingRegions.Count -gt 0) { } foreach ($region in $candidateRegions) { - Write-Host "[INFO] Deploying → RG '$effectiveRG' in '$region' (token '$resourceToken')..." - try { - $d = New-AzResourceGroupDeployment ` - -ResourceGroupName $effectiveRG ` - -TemplateFile $templateFile ` - -location $region ` - -resourceToken $resourceToken ` - -tags $tags ` - -ErrorAction Stop - $deployOutputs = $d.Outputs - $effectiveLocation = $region - break - } - catch { - # Only capacity/quota/region-availability failures are worth retrying elsewhere. - # A template or permission bug fails identically in every region, so retrying it - # just triples the runtime and buries the real error under the last region's message. - $retryable = $true - if (Get-Command Test-MhhDeploymentFailureRetryable -ErrorAction SilentlyContinue) { - $retryable = [bool](Test-MhhDeploymentFailureRetryable -ErrorRecord $_) + # Some failures are purely transient and clear on their own: APIM reports 'ServiceLocked' + # while a previous create/purge is still activating, and the AML resource provider behind + # Foundry projects reports 'Soft-deleted workspace exists' for a short window after the + # parent account is purged. Retrying in place beats moving region — a different region + # would hit the same lag, and for a non-empty RG there is no other region to move to. + $transientPatterns = @( + 'ServiceLocked', 'is transitioning', 'Soft-deleted workspace exists', + 'AnotherOperationInProgress', 'operation is in progress', 'RetryableError' + ) + $maxAttempts = 3 + + for ($attempt = 1; $attempt -le $maxAttempts; $attempt++) { + $suffix = if ($attempt -gt 1) { " (attempt $attempt/$maxAttempts)" } else { '' } + if ($attempt -gt 1) { Invoke-LabSoftDeleteCleanup } + $foundryNameSuffix = Resolve-LabFoundrySuffix + Write-Host "[INFO] Deploying → RG '$effectiveRG' in '$region' (token '$resourceToken')$suffix..." + try { + $d = New-AzResourceGroupDeployment ` + -ResourceGroupName $effectiveRG ` + -TemplateFile $templateFile ` + -location $region ` + -resourceToken $resourceToken ` + -foundryNameSuffix $foundryNameSuffix ` + -tags $tags ` + -ErrorAction Stop + $deployOutputs = $d.Outputs + $effectiveLocation = $region + break } - if (-not $retryable) { - throw "Deployment failed in '$region' with a non-retryable error (retrying other regions would fail the same way): $_" + catch { + $message = "$_" + + $isTransient = $false + foreach ($pattern in $transientPatterns) { + if ($message -match $pattern) { $isTransient = $true; break } + } + if ($isTransient -and $attempt -lt $maxAttempts) { + Write-Host "[WARN] Transient Azure state during deployment: $message" + Write-Host "[INFO] This usually resolves itself — waiting 2 minutes and retrying in '$region'..." + Start-Sleep -Seconds 120 + continue + } + + # Only capacity/quota/region-availability failures are worth retrying elsewhere. + # A template or permission bug fails identically in every region, so retrying it + # just triples the runtime and buries the real error under the last region's message. + $retryable = $true + if (Get-Command Test-MhhDeploymentFailureRetryable -ErrorAction SilentlyContinue) { + $retryable = [bool](Test-MhhDeploymentFailureRetryable -ErrorRecord $_) + } + if (-not $retryable) { + throw "Deployment failed in '$region' with a non-retryable error (retrying other regions would fail the same way): $_" + } + # Keep the FIRST failure: it is the real one. Later regions typically fail with a + # downstream symptom, so reporting only the last error hides the actual cause. + if (-not $firstFailure) { $firstFailure = "in '$region': $_" } + Write-Host "[WARN] Deployment failed in '$region': $_ — trying next region." + break } - # Keep the FIRST failure: it is the real one. Later regions typically fail with a - # downstream symptom, so reporting only the last error hides the actual cause. - if (-not $firstFailure) { $firstFailure = "in '$region': $_" } - Write-Host "[WARN] Deployment failed in '$region': $_ — trying next region." } + + if ($deployOutputs) { break } } if (-not $deployOutputs) { diff --git a/03-Azure/01-04-AI/06_AI_Governance/labautomation/infra/resources.bicep b/03-Azure/01-04-AI/06_AI_Governance/labautomation/infra/resources.bicep index 2b2fcf1b8..7fb57796e 100644 --- a/03-Azure/01-04-AI/06_AI_Governance/labautomation/infra/resources.bicep +++ b/03-Azure/01-04-AI/06_AI_Governance/labautomation/infra/resources.bicep @@ -16,6 +16,21 @@ param location string @description('Deterministic suffix for resource naming (DNS-safe, lowercase).') param resourceToken string +@maxLength(6) +@description('''Extra suffix appended to the two Azure AI Foundry account names only. +An `AIServices` account with `allowProjectManagement` is backed by a hidden AML +(`AmlRp`) workspace. Deleting the account soft-deletes that workspace for ~14 days, +and purging the Cognitive Services account does NOT purge it -- Azure exposes no API +to list or purge it at all. The account name is therefore unusable for two weeks after +any delete: recreating it always lands in `provisioningState: Failed` with +"Soft-deleted workspace exists. Please purge or recover it.". +Because `resourceToken` is derived from subscription id + resource group name and the +platform reuses both, every re-run of an event would otherwise be guaranteed to fail. +`labautomation/deploy-lab.ps1` supplies a fresh suffix whenever the previous account is +not healthy, and reuses the existing one when it is. Foundry accounts are the only +resources affected: Key Vault and APIM purge correctly and keep their stable names.''') +param foundryNameSuffix string = '' + // Participant RBAC is applied by labautomation/deploy-lab.ps1 AFTER this deployment, // so that EVERY id in $AllowedEntraUserIds is granted the roles (a template can only // conveniently grant the first one) and so that hub AND spoke resources are both @@ -37,7 +52,7 @@ only after confirming quota with `az cognitiveservices usage list -l `.' param modelCapacity int = 20 // ===== Hub Resource Naming ===== -var hubFoundryAccountName = 'aif-hub-${resourceToken}' +var hubFoundryAccountName = 'aif-hub-${resourceToken}${foundryNameSuffix}' var hubFoundryProjectName = 'citadel-hub-project' var apimName = 'apim-citadel-${resourceToken}' var cosmosAccountName = 'cos-citadel-${resourceToken}' @@ -52,7 +67,7 @@ var apimMIName = 'mi-apim-${resourceToken}' var logicAppMIName = 'mi-logicapp-${resourceToken}' // ===== Spoke Resource Naming ===== -var spokeFoundryAccountName = 'aif-spoke-${resourceToken}' +var spokeFoundryAccountName = 'aif-spoke-${resourceToken}${foundryNameSuffix}' var spokeFoundryProjectName = 'citadel-agents-project' var spokeLaName = 'law-spoke-${resourceToken}' var spokeAiName = 'appi-spoke-${resourceToken}' diff --git a/03-Azure/01-04-AI/06_AI_Governance/labautomation/infra/resources.json b/03-Azure/01-04-AI/06_AI_Governance/labautomation/infra/resources.json index 72d7da8cb..d4046b84b 100644 --- a/03-Azure/01-04-AI/06_AI_Governance/labautomation/infra/resources.json +++ b/03-Azure/01-04-AI/06_AI_Governance/labautomation/infra/resources.json @@ -5,7 +5,7 @@ "_generator": { "name": "bicep", "version": "0.41.2.15936", - "templateHash": "9591618861995214523" + "templateHash": "1413310816187321949" } }, "parameters": { @@ -23,6 +23,14 @@ "description": "Deterministic suffix for resource naming (DNS-safe, lowercase)." } }, + "foundryNameSuffix": { + "type": "string", + "defaultValue": "", + "maxLength": 6, + "metadata": { + "description": "Extra suffix appended to the two Azure AI Foundry account names only.\nAn `AIServices` account with `allowProjectManagement` is backed by a hidden AML\n(`AmlRp`) workspace. Deleting the account soft-deletes that workspace for ~14 days,\nand purging the Cognitive Services account does NOT purge it -- Azure exposes no API\nto list or purge it at all. The account name is therefore unusable for two weeks after\nany delete: recreating it always lands in `provisioningState: Failed` with\n\"Soft-deleted workspace exists. Please purge or recover it.\".\nBecause `resourceToken` is derived from subscription id + resource group name and the\nplatform reuses both, every re-run of an event would otherwise be guaranteed to fail.\n`labautomation/deploy-lab.ps1` supplies a fresh suffix whenever the previous account is\nnot healthy, and reuses the existing one when it is. Foundry accounts are the only\nresources affected: Key Vault and APIM purge correctly and keep their stable names." + } + }, "tags": { "type": "object", "defaultValue": {}, @@ -60,7 +68,7 @@ "$fxv#2": "\n \n (\"piiDeanonymizeContentInput\");\n \n // Pre-process content to replace common encoded symbols\n content = System.Text.RegularExpressions.Regex.Replace(content, @\"\\\\u003c\", \"<\");\n content = System.Text.RegularExpressions.Regex.Replace(content, @\"\\\\u003e\", \">\");\n \n var mappings = JArray.Parse(\n context.Variables.GetValueOrDefault(\"piiMappings\"));\n \n foreach (var mapping in mappings) {\n var original = mapping[\"original\"].ToString();\n var placeholder = mapping[\"placeholder\"].ToString();\n content = content.Replace(placeholder, original);\n }\n return content;\n }\" />\n", "$fxv#3": "\n \n \n (\"piiStateSavingEnabled\") == \"true\")\">\n \n \n \n \n \n \n \n \n \n \n (\"requestedModel\") ?? \"unknown\")\" />\n \n \n @{\n var eventData = new JObject();\n \n // Operation metadata\n eventData[\"id\"] = context.Variables.GetValueOrDefault(\"piiOperationId\");\n eventData[\"timestamp\"] = context.Variables.GetValueOrDefault(\"piiOperationTimestamp\");\n eventData[\"type\"] = \"PII_Processing\";\n eventData[\"targetDeployment\"] = context.Variables.GetValueOrDefault(\"targetDeployment\");\n // Enhanced API and subscription context with content\n eventData[\"context\"] = new JObject {\n [\"api\"] = new JObject {\n [\"id\"] = context.Variables.GetValueOrDefault(\"piiApiId\"),\n [\"name\"] = context.Variables.GetValueOrDefault(\"piiApiName\"),\n [\"operation\"] = new JObject {\n [\"id\"] = context.Operation.Id ?? \"unknown\",\n [\"name\"] = context.Variables.GetValueOrDefault(\"piiOperationName\"),\n [\"method\"] = context.Operation.Method ?? \"unknown\",\n [\"urlTemplate\"] = context.Operation.UrlTemplate ?? \"unknown\"\n },\n [\"content\"] = new JObject {\n [\"originalRequest\"] = context.Variables.GetValueOrDefault(\"piiInputContent\"),\n [\"anonymizedRequest\"] = context.Variables.GetValueOrDefault(\"piiAnonymizedContent\"),\n [\"originalResponse\"] = context.Variables.GetValueOrDefault(\"originalResponse\"),\n [\"deanonymizedResponse\"] = context.Variables.GetValueOrDefault(\"piiDeanonymizedContentOutput\")\n }\n },\n [\"product\"] = new JObject {\n [\"id\"] = context.Variables.GetValueOrDefault(\"piiProductId\"),\n [\"name\"] = context.Variables.GetValueOrDefault(\"piiProductName\")\n },\n [\"subscription\"] = new JObject {\n [\"id\"] = context.Variables.GetValueOrDefault(\"piiSubscriptionId\"),\n [\"name\"] = context.Variables.GetValueOrDefault(\"piiSubscriptionName\")\n }\n };\n \n // Process information\n eventData[\"process\"] = new JObject {\n [\"anonymizationEnabled\"] = context.Variables.GetValueOrDefault(\"piiAnonymizationEnabled\") == \"true\",\n [\"confidenceThreshold\"] = context.Variables.GetValueOrDefault(\"piiConfidenceThreshold\", 0.8),\n [\"entityCategoryExclusions\"] = context.Variables.GetValueOrDefault(\"piiEntityCategoryExclusions\", \"\"),\n [\"deanonymizationPerformed\"] = context.Variables.ContainsKey(\"piiDeanonymizedContentOutput\")\n };\n \n // Results summary\n var mappings = new JArray();\n if (context.Variables.ContainsKey(\"piiMappings\")) {\n try {\n mappings = JArray.Parse(context.Variables.GetValueOrDefault(\"piiMappings\"));\n eventData[\"entityCount\"] = mappings.Count;\n \n // Group entities by category for reporting\n var categories = new Dictionary();\n foreach (var mapping in mappings) {\n var placeholder = mapping[\"placeholder\"].ToString();\n var category = placeholder.TrimStart('<').Split('_')[0];\n \n if (!categories.ContainsKey(category)) {\n categories[category] = 0;\n }\n categories[category]++;\n }\n \n var categoryDetails = new JObject();\n foreach (var category in categories) {\n categoryDetails[category.Key] = category.Value;\n }\n eventData[\"entityCategories\"] = categoryDetails;\n \n // Include mappings for detailed audit (can be removed if too verbose)\n eventData[\"mappings\"] = mappings;\n }\n catch (Exception ex) {\n eventData[\"error\"] = \"Failed to process mappings: \" + ex.Message;\n }\n }\n \n // Content length metrics (not the actual content)\n if (context.Variables.ContainsKey(\"piiInputContent\")) {\n eventData[\"metrics\"] = new JObject {\n [\"inputContentLength\"] = context.Variables.GetValueOrDefault(\"piiInputContent\", \"\").Length\n };\n \n if (context.Variables.ContainsKey(\"piiAnonymizedContent\")) {\n eventData[\"metrics\"][\"anonymizedContentLength\"] = \n context.Variables.GetValueOrDefault(\"piiAnonymizedContent\", \"\").Length;\n }\n \n // Add deanonymization metrics\n if (context.Variables.ContainsKey(\"piiDeanonymizeContentInput\")) {\n eventData[\"metrics\"][\"deanonymizeInputLength\"] = \n context.Variables.GetValueOrDefault(\"piiDeanonymizeContentInput\", \"\").Length;\n }\n \n if (context.Variables.ContainsKey(\"piiDeanonymizedContentOutput\")) {\n eventData[\"metrics\"][\"deanonymizedOutputLength\"] = \n context.Variables.GetValueOrDefault(\"piiDeanonymizedContentOutput\", \"\").Length;\n }\n }\n \n // Add deanonymization flow details\n eventData[\"deanonymization\"] = new JObject {\n [\"performed\"] = context.Variables.ContainsKey(\"piiDeanonymizedContentOutput\"),\n [\"inputSource\"] = context.Variables.ContainsKey(\"piiDeanonymizeContentInput\") ? \"present\" : \"missing\",\n [\"status\"] = context.Variables.ContainsKey(\"piiDeanonymizedContentOutput\") ? \"completed\" : \"not_performed\",\n [\"timestamp\"] = DateTime.UtcNow.ToString(\"o\")\n };\n \n // Track end-to-end flow\n eventData[\"processingFlow\"] = new JObject {\n [\"anonymization\"] = new JObject {\n [\"performed\"] = context.Variables.ContainsKey(\"piiAnonymizedContent\"),\n [\"entityCount\"] = context.Variables.ContainsKey(\"piiMappings\") ? \n JArray.Parse(context.Variables.GetValueOrDefault(\"piiMappings\")).Count : 0\n },\n [\"deanonymization\"] = new JObject {\n [\"performed\"] = context.Variables.ContainsKey(\"piiDeanonymizedContentOutput\"),\n [\"success\"] = context.Variables.ContainsKey(\"piiDeanonymizedContentOutput\")\n }\n };\n \n return eventData.ToString();\n }\n \n \n\n", "$fxv#4": "\n \n \n \n \n \n \n \n application/json\n \n @{\n JArray models = JArray.Parse((string)context.Variables[\"availableFilteredModels\"]);\n return new JObject(new JProperty(\"object\", \"list\"), new JProperty(\"data\", models)).ToString();\n }\n \n \n \n \n \n\n", - "hubFoundryAccountName": "[format('aif-hub-{0}', parameters('resourceToken'))]", + "hubFoundryAccountName": "[format('aif-hub-{0}{1}', parameters('resourceToken'), parameters('foundryNameSuffix'))]", "hubFoundryProjectName": "citadel-hub-project", "apimName": "[format('apim-citadel-{0}', parameters('resourceToken'))]", "cosmosAccountName": "[format('cos-citadel-{0}', parameters('resourceToken'))]", @@ -73,7 +81,7 @@ "logicAppName": "[format('la-usage-ingestion-{0}', parameters('resourceToken'))]", "apimMIName": "[format('mi-apim-{0}', parameters('resourceToken'))]", "logicAppMIName": "[format('mi-logicapp-{0}', parameters('resourceToken'))]", - "spokeFoundryAccountName": "[format('aif-spoke-{0}', parameters('resourceToken'))]", + "spokeFoundryAccountName": "[format('aif-spoke-{0}{1}', parameters('resourceToken'), parameters('foundryNameSuffix'))]", "spokeFoundryProjectName": "citadel-agents-project", "spokeLaName": "[format('law-spoke-{0}', parameters('resourceToken'))]", "spokeAiName": "[format('appi-spoke-{0}', parameters('resourceToken'))]",