diff --git a/.azure/deployment-plan.md b/.azure/deployment-plan.md new file mode 100644 index 00000000..5e4d99cc --- /dev/null +++ b/.azure/deployment-plan.md @@ -0,0 +1,413 @@ +# GPT-RAG issue 571 deployment preparation plan + +## Status + +Deployment Blocked + +## Mode + +MODIFY + +## Issue + +- Azure/GPT-RAG#571: establish a practical governance baseline and auditable AI activity trail. +- The user has approved autonomous preparation and implementation. +- This plan changes the existing GPT-RAG AZD/Bicep deployment recipe in place. +- The user has authorized one new Standard validation deployment in + `swedencentral`. No destructive resource operation or resource-group deletion + is authorized. + +## Existing deployment recipe + +- Azure Developer CLI orchestrates the existing deployment through `azure.yaml`. +- The root repository supplies `main.parameters.json`, `manifest.json`, lifecycle hooks, and post-provision configuration. +- `infra/` remains the pinned `Azure/bicep-ptn-aiml-landing-zone` submodule. It will not be edited in place. +- Post-provision configuration uses Azure CLI/SDK authentication, Azure App Configuration label `gpt-rag`, Azure Key Vault, Azure Container Apps, and Azure AI Search REST APIs. + +## Architecture + +- Pin `Azure/gpt-rag-orchestrator` v3.8.0 and `Azure/gpt-rag-ingestion` v2.5.0 in the umbrella manifest and future-minor release surfaces. +- Seed metadata-only audit and provenance settings in App Configuration with all feature gates disabled by default. +- Create `AUDIT_HMAC_KEY` once as a cryptographically random 256-bit Key Vault secret when Key Vault is deployed. Reuse the existing value on later runs so ordinary reprovisioning is stable; rotation is an explicit operator action that creates a new Key Vault version. If Key Vault is explicitly disabled, seed only the disabled feature gates and keep audit events off. +- Publish only an App Configuration Key Vault reference for `AUDIT_HMAC_KEY`; never persist or display the secret value in admin settings, files, outputs, or logs. +- Add the ten optional provenance/governance fields to the RAG Azure AI Search index. +- Replace unconditional index delete/recreate with an in-place `PUT` update for existing indexes. Azure AI Search receives the full desired schema while preserving documents for supported additive changes; incompatible changes fail instead of falling back to deletion. +- Preserve the existing creation path for missing indexes and leave unrelated datasource, skillset, and indexer behavior unchanged. +- Keep the orchestrator v3.8.0 logical contract hash `825db8ef40a81e2c19e5d80d37c565b6b47fc9a6540e9881d35cc12b8fde5aab` and wire hash `066c8f5408610ab839d5121d06ca5bc59e8797e551d5c47c875c5ba52f7e0588` aligned in tests and documentation. + +## Configuration defaults + +### Orchestrator + +- `AUDIT_EVENTS_ENABLED=false` +- `AUDIT_SENSITIVE_CONTENT_ENABLED=false` +- `AUDIT_SENSITIVE_CONTENT_FIELDS=` +- `AUDIT_ACTOR_PSEUDONYM_ENABLED=false` +- `AUDIT_SOURCE_EVENT_LIMIT=25` +- `AUDIT_HMAC_KEY_ID=v1` +- `AUDIT_ADDITIONAL_REDACTED_KEYS=` + +### Ingestion + +- `INGESTION_PROVENANCE_ENABLED=false` +- `INGESTION_REQUIRE_GOVERNANCE_METADATA=false` +- `INGESTION_DEFAULT_CLASSIFICATION=unclassified` +- `INGESTION_DEFAULT_RIGHT_TO_USE=not_asserted` + +## Azure AI Search schema + +The RAG index receives these optional, retrievable fields: + +| Field | Azure AI Search type | Attributes | +| --- | --- | --- | +| `provenance_id` | `Edm.String` | filterable | +| `source_uri_id` | `Edm.String` | filterable | +| `source_version_id` | `Edm.String` | filterable | +| `content_checksum_sha256` | `Edm.String` | filterable | +| `ingested_at` | `Edm.DateTimeOffset` | filterable, sortable | +| `ingest_run_id` | `Edm.String` | filterable | +| `data_classification` | `Edm.String` | filterable, facetable | +| `right_to_use` | `Edm.String` | filterable, facetable | +| `retention_class` | `Edm.String` | filterable, facetable | +| `delete_after` | `Edm.String` | filterable, sortable | + +`delete_after` records policy intent only. This solution does not automatically purge documents based on that field. + +## Security + +- Generate key material with Python's operating-system-backed `secrets` module and require exactly 32 random bytes before encoding. +- Read existing secret metadata/value only to preserve idempotency; do not print or serialize the secret. +- Use the existing App Configuration Key Vault reference content type. +- Do not add the secret to the normal App Configuration plaintext import, AZD outputs, Bicep outputs, logs, changelog, or documentation examples. +- Keep sensitive-content capture disabled independently from metadata audit events. +- Preserve fail-closed component validation and prohibited-field redaction in the pinned runtime releases. +- No credentials, secrets, generated resource names, or personal validation + environment names will be committed. Non-secret Azure scope and principal IDs + may be recorded when required for exact validation proof. + +## Migration and rollback + +- Existing deployments receive disabled defaults, so runtime behavior remains unchanged until operators opt in. +- The Search schema update is additive and performed in place. Existing indexed documents remain valid. Ingestion v2.5.0 emits provenance in audit events but does not populate these index fields; they remain empty unless the indexing pipeline explicitly supplies them. +- No automatic or fallback index deletion is allowed. Unsupported schema mutations fail with an actionable error and leave the existing index intact. +- Upgrade sequence: deploy umbrella configuration with the v3.8.0/v2.5.0 pins, run post-provisioning, verify the Key Vault reference and additive index schema, then enable metadata audit/provenance in a non-production environment. +- Rollback sequence: set `AUDIT_EVENTS_ENABLED=false` and `INGESTION_PROVENANCE_ENABLED=false`, keep sensitive capture disabled, and redeploy the prior component tags if required. +- Additive Search fields are harmless to older component versions and do not need removal during rollback. +- Rotating `AUDIT_HMAC_KEY` is explicit: create a new Key Vault secret version and update `AUDIT_HMAC_KEY_ID` so evidence consumers can distinguish pseudonymization epochs. + +## UX and documentation advisory + +- Steve confirmed the main operator risk is the current unconditional index deletion. The implementation must use in-place updates and must not claim a safe additive migration until tests prove no delete call occurs. +- Provide one umbrella configuration table, link to component guidance, and state that ingestion settings contain no secrets. +- State that audit evidence supports adopter governance work but is not proof of legal or regulatory compliance. +- State that enabling audit events requires the Key Vault secret reference and that `delete_after` does not enforce retention. +- Update the GPT-RAG docs branch in a companion documentation PR. + +## Validation + +- [x] All validation checks pass + - [x] AZD installation and authentication + - [x] `azure.yaml` stable-schema validation + - [x] Bicep compilation + - [x] Bicep lint execution + - [x] Python/config build verification + - [x] Focused and full relevant tests + - [x] Static RBAC verification + - [x] AZD environment/subscription/location validation + - [x] Required pre-provision hooks + - [x] `azd provision --preview --no-prompt` + - [x] AI Landing Zone compiled-template size gate + - [x] AZD package validation + - [x] Azure Policy validation + +- Focused Python unit tests for App Configuration defaults, stable secret creation/reference behavior, contract pins, Search field types, and no-delete index updates. +- Full relevant GPT-RAG configuration test suite. +- Python compile and JSON/schema assertions. +- PowerShell parser and script validation; shell syntax validation when Bash is available. +- `az bicep build --file infra/main.bicep` and the repository template-size gate when applicable. +- Manifest/version/tag assertions against published v3.8.0 and v2.5.0 releases. +- Docker/config validation through existing repository scripts where applicable. +- `azd provision --preview` or the repository's non-deploy preflight only when the current authenticated AZD context is complete. Treat any empty environment resource group that AZD creates before what-if as a preview-tool side effect, not as workload provisioning. +- Preparation and preview remain non-destructive. After region-aware validation + passes, hand off one `azd provision` and `azd deploy` attempt to the + `azure-deploy` recovery workflow. + +## Role Assignment Verification + +- Status: Verified for this change. +- Orchestrator identity: `AppConfigurationDataReader`, + `KeyVaultSecretsUser`, `SearchIndexDataReader`, and the existing model, + Cosmos, ACR, and Storage data-plane roles remain scoped through the landing + zone's per-resource role modules. +- Ingestion identity: `AppConfigurationDataReader` and + `SearchIndexDataContributor` remain present for provenance configuration and + indexed-document writes. +- No new management-plane role, resource-group role, subscription role, or + role-assignment resource was added by this branch. + +## Section 7: Validation Proof + +| Check | Command / proof | Result | +| --- | --- | --- | +| Exact source | `git rev-parse HEAD`; compare local and remote PR branch | Passed at requested and remote PR head `5ea5f7c983660db0c2ece99b9eb4fd6cd1688485` | +| CLI versions | `azd version`; `az version --output json` | Passed, AZD v1.27.1 and Azure CLI v2.79.0 | +| Authentication | `azd auth login --check-status`; `az account show`; `az account get-access-token --resource https://management.azure.com/` | Passed for `paulolacerda@microsoft.com`; default subscription `mcaps-paulolacerda` (`9788a92c-2f71-4629-8173-7ad449cb50e1`), tenant `16b3c013-d300-468d-ac64-7eda0820b6d3`; token tenant matched | +| Deploying principal | `az role assignment list --assignee-object-id 88e31ae8-65b8-46a2-8b26-fad4e6c305f3 --scope /subscriptions/9788a92c-2f71-4629-8173-7ad449cb50e1 --include-inherited` | Passed; principal has inherited `Owner` at subscription scope, sufficient for previewing resource-scoped role assignments | +| AZD schema | Azure AZD `validate_azure_yaml` on root `azure.yaml` | Passed against the official stable schema | +| AZD environment | `azd env new --subscription 9788a92c-2f71-4629-8173-7ad449cb50e1 --location eastus2 --no-prompt`; `azd env set ...`; `azd env get-values` | Passed; isolated local environment used `eastus2`, deploying principal `88e31ae8-65b8-46a2-8b26-fad4e6c305f3`, type `User`, and Standard-mode safe preview values: `NETWORK_ISOLATION=false`, `USE_UAI=false`, `USE_CAPP_API_KEY=false`, `ENABLE_AGENTIC_RETRIEVAL=false`, `RETRIEVAL_BACKEND=foundry_iq`, `DEPLOYMENT_MODE=standalone`, and VM/Bastion/NAT/Firewall deployment flags disabled | +| Required hook | Load `azd env get-values`; `pwsh -NoProfile -File scripts/preProvision.ps1` | Passed. GPT-RAG regional preflight: 0 failures, 2 capacity warnings; required providers and eastus2 locations passed, Container Apps quota passed, both model deployments and quotas passed. Landing-zone preflight: 0 failures, 3 warnings, 1 information item | +| Bicep compile | `az bicep build --file infra/main.bicep` | Passed with existing pinned-submodule warnings only | +| Bicep lint | `az bicep lint --file infra/main.bicep` | Passed with existing pinned-submodule warnings only | +| Template size | `pwsh infra/scripts/Measure-MainJsonSize.ps1 -SkipBuild -WorkingBudgetMB 3.5 -FailThresholdMB 4.7 -ArmHardCeilingMB 5.0` | Passed authoritative gate: 4,872,869 bytes (4.647 MB). Warning above 3.5 MB working budget, below 4.7 MB fail threshold and 5.0 MB ARM ceiling | +| Relevant tests | `python -m unittest config.governance.tests.test_setup config.search.tests.test_governance_schema config.search.tests.test_foundry_iq_templates` | Passed, 131 tests | +| Static checks | `python -m ruff check ...`; `python -m compileall -q ...`; PowerShell AST parse; Git Bash `-n scripts/preProvision.sh scripts/postProvision.sh` | Passed | +| Package | `azd package --no-prompt` | Passed; umbrella package completed successfully | +| Docker | Repository scan | Not applicable: no umbrella Dockerfile or AZD service package | +| Azure Policy | Azure Policy `policy_assignment_list` at subscription scope, followed by ARM what-if | Passed; assignments, including inherited MCAPSGov deny/audit/deploy initiatives, were reviewed and did not block the preview | +| Static RBAC | Review `main.parameters.json`, `infra/main.bicep:3153-3525`, and `infra/constants/roles.json`; assert required role keys and Bicep mappings | Passed. Orchestrator has resource-scoped App Configuration Data Reader, Key Vault Secrets User, and Search Index Data Reader. Ingestion has resource-scoped App Configuration Data Reader, Key Vault Secrets User, and Search Index Data Contributor. The deploying principal receives resource-scoped Key Vault Contributor/Secrets Officer, App Configuration Data Owner, Search Service Contributor, and Search data roles needed by post-provision setup. This PR changes neither `infra/` nor `main.parameters.json` and adds no RBAC assignment | +| Provision preview | `azd provision --preview --no-prompt` | Passed in 1 minute 6 seconds. ARM what-if returned a create-only workload plan with no modify/delete operations and applied no workload resources | +| Preview side effect | `az group show`; `az resource list`; Azure activity log | AZD v1.27.1 created the empty, tagged environment resource group before running what-if at `2026-07-21T12:12:15Z`. The group contains 0 resources. It was not deleted because this validation explicitly prohibited Azure resource or resource-group deletion | + +Validation status is **Validated**. All required formal checks passed at exact PR +head `5ea5f7c983660db0c2ece99b9eb4fd6cd1688485`. The change is ready for the +`azure-deploy` workflow, but deployment remains explicitly out of scope for this +validation. + +## Handoff proof required + +- Feature branch `feature/governance-audit-umbrella-571` was created from + `origin/develop` commit `e6ba89f3f609d900c0dd32c26999cf7fe4e8dca0`. +- Focused and full relevant configuration suites passed: 131 tests. +- Ruff passed for all changed Python implementation and test files. +- Python compilation, JSON/schema parsing, contract SHA-256 assertions, and + PowerShell AST parsing passed. +- Tests prove all ten Search fields and their types/attributes, preservation of + operator-added fields, update-before-cleanup ordering, no index DELETE, + ETag-guarded updates, and `If-None-Match: *` guarded creation. +- Tests prove 32-byte OS-random key generation, stable secret reuse, plaintext + key migration without rotation, preservation of operator-managed Key Vault + references, no plaintext deployment parameter, and fail-closed behavior when + audit is enabled without Key Vault. +- The full documentation site passed `mkdocs build --strict`. +- `az bicep build --file infra/main.bicep` succeeded with existing submodule + warnings. The existing AI Landing Zone v2.3.0 template is 4.647 MB and passed + the authoritative 4.7 MB fail threshold and 5.0 MB ARM ceiling, with a warning + above the 3.5 MB working budget. +- `azd provision --preview --no-prompt` succeeded at exact PR head in eastus2 + and returned a create-only ARM what-if plan with no modify/delete operations. + AZD v1.27.1 created an empty environment resource group before what-if; it has + 0 resources and was left untouched because resource-group deletion was + prohibited. +- Git Bash syntax validation passed for both lifecycle hooks. PowerShell AST + parsing and the Python module invoked by both hooks also passed. +- Docker validation is not applicable because the umbrella repository has no + Dockerfile; the pinned component releases were verified as published, + non-draft, non-prerelease tags. +- No workload resources were deployed, modified, or deleted. AZD preview itself + created one empty tagged environment resource group before ARM what-if; no + deletion was attempted. +- Implementation commits: + `11d65b294cc7a2270e579deddf42c770dc9e1761` and + `5ea5f7c`. +- Implementation PR, unmerged: + `https://github.com/Azure/GPT-RAG/pull/573`. +- Documentation commits: + `2cca8c68f16e0f2ade8fdf4939cf8e9b6335fc5d` and + `8f00c16`. +- Documentation PR, unmerged: + `https://github.com/Azure/GPT-RAG/pull/574`. + +## Section 8: Deployment Proof + +- Deployment source: exact PR evidence commit + `a7c98f187b37360cecd1500301a47dcecfcaac78`. +- Deployment context: the exact validated AZD environment was recreated in + `eastus2` with the validated Standard-mode settings, subscription, tenant, + deploying principal, and principal type. The empty resource group created by + preview was reused after confirming it had only the AZD environment tag, had + no `keep=true` tag, contained no resources, and had no Container Apps + environment or conflicting `azd-service-name` tags. +- Pre-deploy checklist: passed. Authentication, subscription, tenant, + deploying principal, region, Azure Policy assignments, environment values, + repository commit, resource-group location/tags, and AZD recipe were + rechecked before provisioning. +- Provisioning attempts: `azd provision --no-prompt` was attempted three times + on 2026-07-21. The first retry followed a 60-second wait; the final retry + followed a 300-second backoff. Every attempt stopped at Azure AI Search with + `InsufficientResourcesAvailable` for `eastus2`. Azure request IDs: + `d73cd9b4-834a-93c5-fa68-7012f02c5f88`, + `59685ea6-8b58-1525-25e4-89697b8fbb55`, and + `8bc5f789-b35f-848b-fbcb-05bc90fe71d6`. +- Partial result: the resource group was retained with successfully created + platform resources, including Log Analytics, Application Insights, App + Configuration, Key Vault, Container Registry, Storage, Cosmos DB, and a + Container Apps environment. Azure AI Search, Container Apps, component + images, and application endpoints were not created. +- Deployment gate: `azd deploy --no-prompt` was not run because infrastructure + provisioning did not complete. The mandatory ACR `AcrPull` propagation gate, + endpoint/health checks, live application RBAC verification, App + Configuration default verification, Key Vault audit-key semantics/reference + verification, Search schema checks, component-pin checks, safe request and + ingestion flows, KQL audit reconstruction, sensitive-property checks, + root-sentinel/event-name/event-budget checks, and overhead measurement remain + blocked. +- Security and retention: no secret value was read or exposed. Key Vault + metadata verification was unavailable because the data-plane role assignment + had not been provisioned. No resource, resource group, role assignment, or + deployment was deleted; `azd down` and all deletion commands were not run. + The partial validation environment remains running for follow-up. +- Status: **Deployment Blocked** by regional Azure AI Search capacity. A safe + retry is to rerun `azd provision --no-prompt` when `eastus2` capacity is + available, then continue with the required ACR RBAC propagation check, + `azd deploy --no-prompt`, and the post-deployment verification matrix. + +## Section 9: Sweden Central recovery plan + +- Authorization: on 2026-07-21 the user authorized one new, unique Standard + validation AZD environment in `swedencentral`, using subscription + `9788a92c-2f71-4629-8173-7ad449cb50e1`, tenant + `16b3c013-d300-468d-ac64-7eda0820b6d3`, and deploying principal + `88e31ae8-65b8-46a2-8b26-fad4e6c305f3` with principal type `User`. +- Isolation: the recovery environment must be fresh and must not reuse Search + service `srch-c6emckf22jxl4`. That existing service proves prior regional + viability only; it does not guarantee new Search capacity. +- Protected partial environment: do not modify or delete + `rg-gptrag-pr573-7xsurm`. Do not run `azd down` or any deletion command. +- Configuration drift gate: preserve the validated Standard settings, including + `NETWORK_ISOLATION=false`, `USE_UAI=false`, `USE_CAPP_API_KEY=false`, + `ENABLE_AGENTIC_RETRIEVAL=false`, `RETRIEVAL_BACKEND=foundry_iq`, + `DEPLOYMENT_MODE=standalone`, and disabled VM, Bastion, NAT, and Firewall + deployment flags. Only environment/resource names and required regional + values may differ. +- Region-aware gate: rerun the required pre-provision checks for + `swedencentral`, then run one fresh `azd provision --preview --no-prompt`. + Continue only if the plan is create-only and contains no modify/delete + operations. +- Capacity stop condition: run one provision attempt. If Azure AI Search again + returns `InsufficientResourcesAvailable`, stop without retrying another + region and report the blocker. +- Successful deployment gate: after provisioning, verify ACR pull roles have + propagated before `azd deploy --no-prompt`. +- Verification: fully qualified endpoints and health, live RBAC, manifest + component versions, App Configuration safe defaults, Key Vault + `AUDIT-HMAC-KEY` existence and 32-byte semantics without exposing its value, + Key Vault reference wiring, additive Search provenance fields, minimal safe + request and ingestion flows, Application Insights `gptrag.audit.*` custom + events, root sentinel/privacy/event names, and KQL reconstruction. Keep + sensitive capture disabled. +- Evidence: append exact environment, resource group, endpoints, commands, + timestamps, verification results, and blockers to this file; commit and push + the evidence to PR #573. Leave the successful environment active. + +## Section 10: Sweden Central validation proof + +- Validation source: PR head + `b7766419aa03375ef61de9fbe05e90b80f02a0db`. The only committed delta from + implementation head `5ea5f7c983660db0c2ece99b9eb4fd6cd1688485` + is `.azure/deployment-plan.md`; no code, Bicep, parameters, manifest, or + lifecycle hook changed. +- Environment: new local AZD environment `gptrag-pr573-sw-a333b`, target resource + group `rg-gptrag-pr573-sw-a333b`, subscription + `9788a92c-2f71-4629-8173-7ad449cb50e1`, tenant + `16b3c013-d300-468d-ac64-7eda0820b6d3`, region `swedencentral`, deploying + principal `88e31ae8-65b8-46a2-8b26-fad4e6c305f3`, principal type `User`. +- Configuration: `azd env get-values` confirmed the approved Standard settings, + including `NETWORK_ISOLATION=false`, `USE_UAI=false`, + `USE_CAPP_API_KEY=false`, `ENABLE_AGENTIC_RETRIEVAL=false`, + `RETRIEVAL_BACKEND=foundry_iq`, `DEPLOYMENT_MODE=standalone`, and disabled VM, + software, Jumpbox, Bastion, NAT Gateway, and Azure Firewall flags. Foundry, + Search, and Cosmos regional values are all `swedencentral`. +- Authentication and authorization: `az account show`, + `az ad signed-in-user show`, and inherited role lookup confirmed the requested + subscription, tenant, exact principal, and inherited `Owner` at subscription + scope. +- Regional evidence: existing Standard Search service + `srch-c6emckf22jxl4` is running in Sweden Central in the same subscription. + It is not referenced or reused by this environment. +- Policy: Azure Policy assignments were refreshed. Subscription and inherited + MCAPSGov audit, deny, deploy/modify, security-baseline, and MFA assignments + remain applicable; preview found no policy blocker. +- Provider/quota preflight: `scripts/preProvision.ps1` passed with 0 failures, + 2 capacity warnings, and landing-zone preflight passed with 0 failures, + 1 warning, and 1 information item. All required providers and + `swedencentral` locations passed. Container Apps had 50 managed environments + remaining. `gpt-5-nano` had 15,000 quota units remaining for a 100-unit + request; `text-embedding-3-large` had 350 remaining for a 100-unit request. + Search and Cosmos live capacity cannot be guaranteed before provisioning. +- AZD schema: the Azure AZD stable-schema validator passed `azure.yaml`. +- Build/what-if: the first `azd provision --preview --no-prompt` stopped before + Azure access because the fresh clone had not initialized the pinned `infra` + submodule. After `git submodule update --init --recursive` checked out + `1616ddd940b796c32f86e4459b079eebc254de08`, the preview passed in 1 minute + 18 seconds. It compiled the pinned Bicep and returned create-only resources + with no modify/delete operations. +- Preview safety: the preview-created resource group contains 0 resources, has + only the `azd-env-name=gptrag-pr573-sw-a333b` tag, and has no Container Apps + environment or `azd-service-name` conflict. +- Protected environment: read-only checks confirmed + `rg-gptrag-pr573-7xsurm` remains in `eastus2`; no operation targeted, modified, + or deleted it. +- Static RBAC and package/build proof from Section 7 remains applicable because + no implementation, infrastructure, parameter, manifest, or hook file changed. +- Status: **Validated** for one `swedencentral` recovery deployment attempt. Any + modify/delete plan or Azure AI Search `InsufficientResourcesAvailable` result + is a hard stop. No deletion or `azd down` is authorized. + +## Section 11: Sweden Central deployment recovery proof + +- Attempt: one `azd provision --no-prompt` run against validated environment + `gptrag-pr573-sw-a333b` and resource group + `rg-gptrag-pr573-sw-a333b` in `swedencentral`. +- Source: PR evidence head + `b7766419aa03375ef61de9fbe05e90b80f02a0db`, plus this uncommitted evidence + update. No implementation, Bicep, parameter, manifest, or lifecycle-hook file + changed. +- Preflight: repeated during provisioning and passed with 0 failures. Provider, + location, Container Apps quota, model availability, and model quota checks + remained successful. Search and Cosmos live-capacity warnings remained. +- ARM deployment: + `gptrag-pr573-sw-a333b-1784639848`, terminal state `Failed` at + `2026-07-21T13:21:12.160467Z`. +- Capacity blocker: Container Apps managed environment + `cae-n7t6ey-gptrag-pr573-sw-a333b` failed with + `ManagedEnvironmentCapacityHeavyUsageError` / + `AKSCapacityHeavyUsage` because a new managed-environment cluster was + unavailable in `swedencentral`. Azure request ID: + `2757ce4e-9f1a-4e61-8f23-62978db7d9fd`. +- Independent naming blocker: the Cosmos nested deployment name + `umd7bs2my5f6a-sqldb-cosmosdb-n7t6ey-gptrag-pr573-sw-a333b-sdc-001` + was 65 characters and exceeded the ARM deployment-name maximum of 64. +- Stop condition: the requested one-region attempt limit was reached by the + platform-capacity failure. No retry, alternate region, shorter environment, + `azd deploy`, `azd down`, or deletion command was run. +- Partial resources retained in the Sweden Central resource group: + Key Vault `kv-n7t6ey-gptrag-pr573-s`, Log Analytics workspace + `log-n7t6ey-gptrag-pr573-sw-a333b-sdc-001`, App Configuration + `appcs-n7t6ey-gptrag-pr573-sw-a333b-sdc-001`, Container Registry + `crn7t6eygptragpr573swa333bsdc001`, Cosmos DB account + `cosmos-n7t6ey-gptrag-pr573-sw-a333b-sdc-001`, Storage account + `stn7t6eygptragpr573swa33`, Application Insights + `appi-n7t6ey-gptrag-pr573-sw-a333b-sdc-001`, and failed Container Apps + environment `cae-n7t6ey-gptrag-pr573-sw-a333b`. +- Partial platform endpoints: + `https://kv-n7t6ey-gptrag-pr573-s.vault.azure.net/` and + `https://appcs-n7t6ey-gptrag-pr573-sw-a333b-sdc-001.azconfig.io`. + These resources reached management-plane `Succeeded`, but application health + was not testable. +- Not created: Azure AI Search services and Container Apps. No application FQDN + or health endpoint exists. +- Blocked verification: ACR `AcrPull` propagation, `azd deploy`, component + versions in running apps, live application RBAC, App Configuration governance + defaults, Key Vault `AUDIT-HMAC-KEY` existence/32-byte semantics/reference, + additive Search fields, safe request and ingestion flows, Application + Insights `gptrag.audit.*` custom events, root sentinel/privacy/event names, + and KQL reconstruction. +- Security: no Key Vault secret value, connection string, token, payload, or + sensitive event content was read or recorded. Sensitive capture was never + enabled. +- Retention: all Sweden Central partial resources remain active. The eastus2 + group `rg-gptrag-pr573-7xsurm` remains unchanged with 9 resources. No + resource, group, role assignment, or deployment was deleted. +- Status: **Deployment Blocked** by Sweden Central Container Apps platform + capacity, with an additional environment-name length defect that must be + corrected before any future attempt. diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..4c9bffe3 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +contracts/*.json text eol=lf +contracts/*.sha256 text eol=lf +config/search/tests/test_foundry_iq_templates.py -text whitespace=cr-at-eol diff --git a/CHANGELOG.md b/CHANGELOG.md index 42621293..82ca0bf0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,52 @@ # Changelog +## [v3.7.0] - 2026-07-21 + +### User and operator impact + +GPT-RAG `v3.7.0` delivers the governance baseline tracked in [issue #571](https://github.com/Azure/GPT-RAG/issues/571) through [implementation PR #573](https://github.com/Azure/GPT-RAG/pull/573) and [documentation PR #574](https://github.com/Azure/GPT-RAG/pull/574), with [orchestrator `v3.8.0`](https://github.com/Azure/gpt-rag-orchestrator/releases/tag/v3.8.0) and [ingestion `v2.5.0`](https://github.com/Azure/gpt-rag-ingestion/releases/tag/v2.5.0) deployed as one pinned combination. Metadata audit events and ingestion provenance remain disabled by default, sensitive-content capture remains independently disabled, and existing deployments do not change behavior until an operator opts in. + +The shared `audit-event-v1` contract is pinned to logical hash `825db8ef40a81e2c19e5d80d37c565b6b47fc9a6540e9881d35cc12b8fde5aab` and Application Insights wire hash `066c8f5408610ab839d5121d06ca5bc59e8797e551d5c47c875c5ba52f7e0588`. These audit events provide technical evidence for adopter governance and incident response; they do not establish legal or regulatory compliance. + +### Added + +- **Safe governance defaults in App Configuration.** Fresh and upgraded deployments seed missing keys as `AUDIT_EVENTS_ENABLED=false`, `AUDIT_SENSITIVE_CONTENT_ENABLED=false`, `AUDIT_SENSITIVE_CONTENT_FIELDS=''`, `AUDIT_ACTOR_PSEUDONYM_ENABLED=false`, `AUDIT_SOURCE_EVENT_LIMIT=25`, `AUDIT_HMAC_KEY_ID=v1`, `AUDIT_ADDITIONAL_REDACTED_KEYS=''`, `INGESTION_PROVENANCE_ENABLED=false`, `INGESTION_REQUIRE_GOVERNANCE_METADATA=false`, `INGESTION_DEFAULT_CLASSIFICATION=unclassified`, and `INGESTION_DEFAULT_RIGHT_TO_USE=not_asserted`. Reprovisioning preserves operator-managed values unless the corresponding AZD environment value explicitly overrides them. +- **Key Vault-backed audit pseudonymization key.** When `AUDIT_HMAC_KEY` is absent, post-provisioning creates `AUDIT-HMAC-KEY` from 256 cryptographically random bits and registers only its Key Vault reference in App Configuration. Later runs reuse it; an existing operator-managed Key Vault reference is preserved, and a valid legacy plaintext key is migrated without changing its value. The value is not placed in plaintext deployment configuration, output, or logs. Rotation is explicit: create a new Key Vault secret version and advance `AUDIT_HMAC_KEY_ID`. +- **Additive provenance fields in the RAG Search index.** The index adds `provenance_id`, `source_uri_id`, `source_version_id`, `content_checksum_sha256`, `ingested_at`, `ingest_run_id`, `data_classification`, `right_to_use`, `retention_class`, and `delete_after`. Identifier/checksum fields are retrievable and filterable; guaranteed UTC `ingested_at` values use sortable `Edm.DateTimeOffset`; `delete_after` remains a sortable string because ingestion v2.5.0 intentionally passes through operator policy values without imposing a date format; governance classification fields are facetable. +- **Shared contract source.** `contracts/` now carries the byte-pinned logical and Application Insights wire schemas consumed by orchestrator v3.8.0 and ingestion v2.5.0. + +### Changed + +- **Release manifest.** `manifest.json` identifies the umbrella `v3.7.0` combination and pins orchestrator `v3.8.0` plus ingestion `v2.5.0`; UI `v2.3.13` and infra `v2.3.0` remain unchanged. +- **Non-destructive existing-index migration.** Search post-provisioning no longer deletes an index before updating it. Missing fields are merged into the current definition and submitted with an in-place `PUT`; operator-owned fields and existing documents are preserved. An incompatible same-name field fails before any update, and there is no delete/recreate fallback. + +### Security and operational guidance + +Keep `AUDIT_SENSITIVE_CONTENT_ENABLED=false` unless an explicit, reviewed capture policy is in place. Tokens, credentials, authorization material, cookies, connection strings, and configured redacted keys remain prohibited even when sensitive capture is enabled. The audit HMAC secret is intended for orchestrator pseudonymization only; ingestion does not consume it. + +`delete_after` records retention policy intent only. GPT-RAG does not schedule or perform automatic deletion from this field. Operators remain responsible for retention enforcement, confirmed deletion, telemetry access, and export controls. + +Deployments that explicitly disable Key Vault still receive the disabled governance defaults, but no audit HMAC key/reference is created. Audit events must remain disabled in that deployment mode. + +### Migration and rollback + +Upgrade the umbrella configuration and both pinned runtime components together, run post-provisioning to register the Key Vault reference and additive Search fields, verify metadata-only events in a non-production environment, and only then enable `AUDIT_EVENTS_ENABLED` or `INGESTION_PROVENANCE_ENABLED`. + +Rollback by setting both feature gates to `false` and redeploying prior component tags if needed. The additive Search fields can remain in place because older components ignore unknown fields. Post-provisioning never recreates an existing index for this migration. + +### Validation + +| Component | Version | +| --- | --- | +| gpt-rag-ui | v2.3.13 | +| gpt-rag-orchestrator | v3.8.0 | +| gpt-rag-ingestion | v2.5.0 | +| infra / AI Landing Zone | v2.3.0 | + +- GPT-RAG governance and Search suites: 131 tests passed. +- Bicep compilation, template rendering, static RBAC checks, manifest and component-release checks, and create-only Azure previews passed. +- Runtime provisioning could not complete. Azure platform capacity blocked Azure AI Search in `eastus2` and the Container Apps managed environment in `swedencentral`; the Sweden Central attempt also exposed an ARM nested deployment name over the 64-character limit. No runtime endpoints, application health checks, live governance defaults, Search migration result, or Application Insights/KQL evidence were obtained. + ## [v3.6.0] - 2026-07-19 ### User and operator impact diff --git a/config/governance/__init__.py b/config/governance/__init__.py new file mode 100644 index 00000000..9f67bed8 --- /dev/null +++ b/config/governance/__init__.py @@ -0,0 +1 @@ +"""Governance and audit deployment configuration.""" diff --git a/config/governance/setup.py b/config/governance/setup.py new file mode 100644 index 00000000..8ea61236 --- /dev/null +++ b/config/governance/setup.py @@ -0,0 +1,261 @@ +#!/usr/bin/env python3 +"""Seed safe governance defaults and the audit pseudonymization key reference.""" + +from __future__ import annotations + +import base64 +import hmac +import json +import logging +import os +import secrets +from collections.abc import Mapping +from typing import Any + +from azure.appconfiguration import AzureAppConfigurationClient, ConfigurationSetting +from azure.core.exceptions import ResourceNotFoundError +from azure.identity import ( + AzureCliCredential, + ChainedTokenCredential, + ManagedIdentityCredential, +) +from azure.keyvault.secrets import SecretClient + + +LABEL = "gpt-rag" +AUDIT_HMAC_CONFIG_KEY = "AUDIT_HMAC_KEY" +AUDIT_HMAC_SECRET_NAME = "AUDIT-HMAC-KEY" +KEY_VAULT_REFERENCE_CONTENT_TYPE = ( + "application/vnd.microsoft.appconfig.keyvaultref+json;charset=utf-8" +) + +DEFAULT_SETTINGS: Mapping[str, str] = { + "AUDIT_EVENTS_ENABLED": "false", + "AUDIT_SENSITIVE_CONTENT_ENABLED": "false", + "AUDIT_SENSITIVE_CONTENT_FIELDS": "", + "AUDIT_ACTOR_PSEUDONYM_ENABLED": "false", + "AUDIT_SOURCE_EVENT_LIMIT": "25", + "AUDIT_HMAC_KEY_ID": "v1", + "AUDIT_ADDITIONAL_REDACTED_KEYS": "", + "INGESTION_PROVENANCE_ENABLED": "false", + "INGESTION_REQUIRE_GOVERNANCE_METADATA": "false", + "INGESTION_DEFAULT_CLASSIFICATION": "unclassified", + "INGESTION_DEFAULT_RIGHT_TO_USE": "not_asserted", +} + + +def create_credential() -> ChainedTokenCredential: + return ChainedTokenCredential( + AzureCliCredential(process_timeout=30), + ManagedIdentityCredential(process_timeout=30), + ) + + +def generate_audit_hmac_key() -> str: + """Return a Base64URL encoding of exactly 256 random bits.""" + return base64.urlsafe_b64encode(secrets.token_bytes(32)).decode("ascii").rstrip("=") + + +def _decode_audit_hmac_key(value: str) -> bytes: + candidate = value.strip() + if len(candidate) == 64: + try: + decoded = bytes.fromhex(candidate) + except ValueError: + decoded = b"" + if len(decoded) == 32: + return decoded + + padded = candidate + ("=" * (-len(candidate) % 4)) + try: + decoded = base64.urlsafe_b64decode(padded.encode("ascii")) + except (ValueError, UnicodeEncodeError): + decoded = b"" + if len(decoded) != 32: + raise ValueError( + "The existing AUDIT-HMAC-KEY secret is not an encoding of exactly " + "32 bytes. Rotate it explicitly before enabling audit events." + ) + return decoded + + +def ensure_audit_hmac_secret( + secret_client: Any, + initial_value: str | None = None, +) -> bool: + """Create the key once and return whether a new secret version was written.""" + try: + current = secret_client.get_secret(AUDIT_HMAC_SECRET_NAME) + except ResourceNotFoundError: + value = initial_value or generate_audit_hmac_key() + _decode_audit_hmac_key(value) + secret_client.set_secret( + AUDIT_HMAC_SECRET_NAME, + value, + content_type="application/octet-stream", + ) + return True + + current_key = _decode_audit_hmac_key(current.value) + if initial_value is not None: + initial_key = _decode_audit_hmac_key(initial_value) + if not hmac.compare_digest(current_key, initial_key): + raise ValueError( + "The existing AUDIT-HMAC-KEY does not match the operator-managed " + "AUDIT_HMAC_KEY value. Resolve the conflict explicitly." + ) + return False + + +def key_vault_reference(vault_uri: str) -> str: + uri = f"{vault_uri.rstrip('/')}/secrets/{AUDIT_HMAC_SECRET_NAME}" + return json.dumps({"uri": uri}, separators=(",", ":")) + + +def resolve_vault_uri(environ: Mapping[str, str] = os.environ) -> str: + configured_uri = environ.get("KEY_VAULT_URI", "").strip() + if configured_uri: + return configured_uri.rstrip("/") + "/" + + vault_name = environ.get("KEY_VAULT_NAME", "").strip() + if vault_name: + return f"https://{vault_name}.vault.azure.net/" + + raise RuntimeError( + "KEY_VAULT_URI or KEY_VAULT_NAME is required to provision the audit " + "HMAC key reference." + ) + + +def seed_governance_settings( + app_config_client: Any, + environ: Mapping[str, str] = os.environ, +) -> None: + """Seed missing defaults while preserving operator-managed values.""" + for key, default in DEFAULT_SETTINGS.items(): + if key in environ: + value = environ[key] + else: + try: + app_config_client.get_configuration_setting(key=key, label=LABEL) + except ResourceNotFoundError: + value = default + else: + continue + + app_config_client.set_configuration_setting( + ConfigurationSetting( + key=key, + label=LABEL, + value=value, + content_type="text/plain", + ) + ) + + +def get_configuration_setting_or_none(app_config_client: Any, key: str) -> Any: + try: + return app_config_client.get_configuration_setting(key=key, label=LABEL) + except ResourceNotFoundError: + return None + + +def effective_setting( + app_config_client: Any, + key: str, + default: str, + environ: Mapping[str, str] = os.environ, +) -> str: + if key in environ: + return environ[key] + setting = get_configuration_setting_or_none(app_config_client, key) + return default if setting is None else str(setting.value) + + +def is_truthy(value: str) -> bool: + return value.strip().lower() in {"1", "true", "yes", "on"} + + +def apply_governance_configuration( + app_config_client: Any, + secret_client: Any, + vault_uri: str, + environ: Mapping[str, str] = os.environ, +) -> bool: + """Apply plaintext defaults and a Key Vault reference without exposing the key.""" + seed_governance_settings(app_config_client, environ) + current_setting = get_configuration_setting_or_none( + app_config_client, + AUDIT_HMAC_CONFIG_KEY, + ) + desired_reference = key_vault_reference(vault_uri) + + if current_setting is not None and ( + current_setting.content_type or "" + ).startswith("application/vnd.microsoft.appconfig.keyvaultref+json"): + if current_setting.value == desired_reference: + return ensure_audit_hmac_secret(secret_client) + return False + + initial_value = None + if current_setting is not None and current_setting.value: + initial_value = str(current_setting.value) + _decode_audit_hmac_key(initial_value) + + created = ensure_audit_hmac_secret(secret_client, initial_value) + + app_config_client.set_configuration_setting( + ConfigurationSetting( + key=AUDIT_HMAC_CONFIG_KEY, + label=LABEL, + value=desired_reference, + content_type=KEY_VAULT_REFERENCE_CONTENT_TYPE, + ) + ) + return created + + +def main() -> None: + app_config_endpoint = os.environ.get("APP_CONFIG_ENDPOINT", "").strip() + if not app_config_endpoint: + raise RuntimeError("APP_CONFIG_ENDPOINT is required for governance setup.") + + credential = create_credential() + app_config_client = AzureAppConfigurationClient(app_config_endpoint, credential) + deploy_key_vault = os.environ.get("DEPLOY_KEY_VAULT", "true").strip().lower() + if deploy_key_vault in {"0", "false", "no", "off"}: + seed_governance_settings(app_config_client) + audit_enabled = effective_setting( + app_config_client, + "AUDIT_EVENTS_ENABLED", + "false", + ) + if is_truthy(audit_enabled): + raise RuntimeError( + "AUDIT_EVENTS_ENABLED cannot be true when Key Vault is disabled." + ) + logging.warning( + "Governance defaults were seeded, but AUDIT_HMAC_KEY was not " + "provisioned because Key Vault is disabled. Keep audit events disabled." + ) + return + + vault_uri = resolve_vault_uri() + secret_client = SecretClient(vault_url=vault_uri, credential=credential) + created = apply_governance_configuration( + app_config_client, + secret_client, + vault_uri, + ) + + action = "created" if created else "reused" + logging.info( + "Governance configuration applied: safe defaults written, audit HMAC " + "secret %s, and Key Vault reference registered.", + action, + ) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") + main() diff --git a/config/governance/tests/__init__.py b/config/governance/tests/__init__.py new file mode 100644 index 00000000..7aa831ba --- /dev/null +++ b/config/governance/tests/__init__.py @@ -0,0 +1 @@ +"""Tests for governance deployment configuration.""" diff --git a/config/governance/tests/test_setup.py b/config/governance/tests/test_setup.py new file mode 100644 index 00000000..842ddbf8 --- /dev/null +++ b/config/governance/tests/test_setup.py @@ -0,0 +1,271 @@ +import base64 +import hashlib +import json +from pathlib import Path +from types import SimpleNamespace +from unittest import TestCase +from unittest.mock import Mock, patch + +from azure.core.exceptions import ResourceNotFoundError + +from config.governance import setup + + +REPO_ROOT = Path(__file__).resolve().parents[3] +LOGICAL_CONTRACT_HASH = ( + "825db8ef40a81e2c19e5d80d37c565b6b47fc9a6540e9881d35cc12b8fde5aab" +) +WIRE_CONTRACT_HASH = ( + "066c8f5408610ab839d5121d06ca5bc59e8797e551d5c47c875c5ba52f7e0588" +) + + +class GovernanceSetupTests(TestCase): + def test_defaults_keep_audit_and_provenance_disabled(self): + self.assertEqual(setup.DEFAULT_SETTINGS["AUDIT_EVENTS_ENABLED"], "false") + self.assertEqual( + setup.DEFAULT_SETTINGS["AUDIT_SENSITIVE_CONTENT_ENABLED"], "false" + ) + self.assertEqual( + setup.DEFAULT_SETTINGS["INGESTION_PROVENANCE_ENABLED"], "false" + ) + self.assertEqual( + setup.DEFAULT_SETTINGS["INGESTION_REQUIRE_GOVERNANCE_METADATA"], + "false", + ) + self.assertEqual( + setup.DEFAULT_SETTINGS["INGESTION_DEFAULT_CLASSIFICATION"], + "unclassified", + ) + self.assertEqual( + setup.DEFAULT_SETTINGS["INGESTION_DEFAULT_RIGHT_TO_USE"], + "not_asserted", + ) + + def test_generated_key_contains_exactly_256_random_bits(self): + with patch.object(setup.secrets, "token_bytes", return_value=b"x" * 32) as token: + encoded = setup.generate_audit_hmac_key() + + token.assert_called_once_with(32) + decoded = base64.urlsafe_b64decode(encoded + "=" * (-len(encoded) % 4)) + self.assertEqual(decoded, b"x" * 32) + + def test_vault_uri_uses_azd_key_vault_name_output(self): + self.assertEqual( + setup.resolve_vault_uri({"KEY_VAULT_NAME": "kv-example"}), + "https://kv-example.vault.azure.net/", + ) + + def test_existing_valid_key_is_reused_without_rotation(self): + secret_client = Mock() + secret_client.get_secret.return_value = SimpleNamespace(value="ab" * 32) + + created = setup.ensure_audit_hmac_secret(secret_client) + + self.assertFalse(created) + secret_client.set_secret.assert_not_called() + + def test_missing_key_is_created_without_returning_or_logging_its_value(self): + secret_client = Mock() + secret_client.get_secret.side_effect = ResourceNotFoundError("missing") + + created = setup.ensure_audit_hmac_secret(secret_client) + + self.assertTrue(created) + secret_client.set_secret.assert_called_once() + name, value = secret_client.set_secret.call_args.args + self.assertEqual(name, setup.AUDIT_HMAC_SECRET_NAME) + self.assertEqual(len(base64.urlsafe_b64decode(value + "=")), 32) + + def test_invalid_existing_key_fails_instead_of_rotating_silently(self): + secret_client = Mock() + secret_client.get_secret.return_value = SimpleNamespace(value="too-short") + + with self.assertRaisesRegex(ValueError, "exactly 32 bytes"): + setup.ensure_audit_hmac_secret(secret_client) + + secret_client.set_secret.assert_not_called() + + def test_app_configuration_contains_defaults_and_only_a_key_vault_reference(self): + app_config_client = Mock() + app_config_client.get_configuration_setting.side_effect = ResourceNotFoundError( + "missing" + ) + secret_client = Mock() + secret_client.get_secret.return_value = SimpleNamespace(value="ab" * 32) + vault_uri = "https://kv-example.vault.azure.net/" + + setup.apply_governance_configuration( + app_config_client, + secret_client, + vault_uri, + ) + + settings = { + call.args[0].key: call.args[0] + for call in app_config_client.set_configuration_setting.call_args_list + } + self.assertEqual(set(settings), {*setup.DEFAULT_SETTINGS, "AUDIT_HMAC_KEY"}) + reference = settings["AUDIT_HMAC_KEY"] + self.assertEqual( + reference.content_type, + setup.KEY_VAULT_REFERENCE_CONTENT_TYPE, + ) + self.assertEqual( + reference.value, + '{"uri":"https://kv-example.vault.azure.net/secrets/AUDIT-HMAC-KEY"}', + ) + self.assertNotIn("ab" * 32, reference.value) + + def test_existing_operator_key_vault_reference_is_preserved(self): + app_config_client = Mock() + operator_reference = SimpleNamespace( + key="AUDIT_HMAC_KEY", + value='{"uri":"https://operator-kv.vault.azure.net/secrets/audit-key"}', + content_type=setup.KEY_VAULT_REFERENCE_CONTENT_TYPE, + ) + + def get_setting(*, key, label): + if key == "AUDIT_HMAC_KEY": + return operator_reference + raise ResourceNotFoundError("missing") + + app_config_client.get_configuration_setting.side_effect = get_setting + secret_client = Mock() + + created = setup.apply_governance_configuration( + app_config_client, + secret_client, + "https://platform-kv.vault.azure.net/", + ) + + self.assertFalse(created) + secret_client.get_secret.assert_not_called() + written_keys = { + call.args[0].key + for call in app_config_client.set_configuration_setting.call_args_list + } + self.assertNotIn("AUDIT_HMAC_KEY", written_keys) + + def test_plaintext_operator_key_is_migrated_without_rotation(self): + plaintext_key = "cd" * 32 + app_config_client = Mock() + + def get_setting(*, key, label): + if key == "AUDIT_HMAC_KEY": + return SimpleNamespace( + key=key, + value=plaintext_key, + content_type="text/plain", + ) + raise ResourceNotFoundError("missing") + + app_config_client.get_configuration_setting.side_effect = get_setting + secret_client = Mock() + secret_client.get_secret.side_effect = ResourceNotFoundError("missing") + + setup.apply_governance_configuration( + app_config_client, + secret_client, + "https://platform-kv.vault.azure.net/", + ) + + self.assertEqual(secret_client.set_secret.call_args.args[1], plaintext_key) + written = { + call.args[0].key: call.args[0] + for call in app_config_client.set_configuration_setting.call_args_list + } + self.assertEqual( + written["AUDIT_HMAC_KEY"].content_type, + setup.KEY_VAULT_REFERENCE_CONTENT_TYPE, + ) + + def test_key_vault_disabled_rejects_enabled_audit_setting(self): + app_config_client = Mock() + + def get_setting(*, key, label): + if key == "AUDIT_EVENTS_ENABLED": + return SimpleNamespace(value="true") + raise ResourceNotFoundError("missing") + + app_config_client.get_configuration_setting.side_effect = get_setting + with patch.object( + setup, + "AzureAppConfigurationClient", + return_value=app_config_client, + ), patch.dict( + setup.os.environ, + { + "APP_CONFIG_ENDPOINT": "https://config.azconfig.io", + "DEPLOY_KEY_VAULT": "false", + }, + clear=True, + ): + with self.assertRaisesRegex(RuntimeError, "Key Vault is disabled"): + setup.main() + + def test_existing_operator_settings_are_preserved_without_explicit_override(self): + app_config_client = Mock() + + setup.seed_governance_settings(app_config_client) + + app_config_client.get_configuration_setting.assert_called() + app_config_client.set_configuration_setting.assert_not_called() + + def test_explicit_environment_override_updates_existing_setting(self): + app_config_client = Mock() + + setup.seed_governance_settings( + app_config_client, + {"AUDIT_EVENTS_ENABLED": "true"}, + ) + + written = [ + call.args[0] + for call in app_config_client.set_configuration_setting.call_args_list + ] + self.assertEqual( + [(setting.key, setting.value) for setting in written], + [("AUDIT_EVENTS_ENABLED", "true")], + ) + + def test_shared_contract_files_match_v3_8_0_hashes(self): + logical = REPO_ROOT / "contracts" / "audit-event-v1.schema.json" + wire = ( + REPO_ROOT + / "contracts" + / "audit-event-v1.application-insights.schema.json" + ) + + self.assertEqual(hashlib.sha256(logical.read_bytes()).hexdigest(), LOGICAL_CONTRACT_HASH) + self.assertEqual(hashlib.sha256(wire.read_bytes()).hexdigest(), WIRE_CONTRACT_HASH) + + def test_deployment_surfaces_never_publish_the_secret_as_plaintext(self): + parameters = json.loads( + (REPO_ROOT / "main.parameters.json").read_text(encoding="utf-8") + ) + serialized_parameters = json.dumps(parameters) + self.assertNotIn("AUDIT_HMAC_KEY", serialized_parameters) + + for script_name in ("postProvision.ps1", "postProvision.sh"): + script = (REPO_ROOT / "scripts" / script_name).read_text(encoding="utf-8") + self.assertIn("config.governance.setup", script) + + def test_future_minor_manifest_pins_validated_component_pair(self): + manifest = json.loads( + (REPO_ROOT / "manifest.json").read_text(encoding="utf-8") + ) + component_versions = { + component["name"]: component["tag"] + for component in manifest["components"] + } + + self.assertEqual(manifest["tag"], "v3.7.0") + self.assertEqual( + component_versions["gpt-rag-orchestrator"], + "v3.8.0", + ) + self.assertEqual( + component_versions["gpt-rag-ingestion"], + "v2.5.0", + ) diff --git a/config/requirements.txt b/config/requirements.txt index fe56a3b2..f8e865ff 100644 --- a/config/requirements.txt +++ b/config/requirements.txt @@ -1,11 +1,11 @@ # general azure-identity==1.25.3 -azure-appconfiguration==1.8.1 +azure-appconfiguration==1.9.0 # app container apps -azure-mgmt-appcontainers==3.2.0 +azure-mgmt-appcontainers==5.0.0 # aifoundry @@ -14,5 +14,5 @@ azure-mgmt-cognitiveservices==13.6.0 # search requests==2.34.2 -azure-keyvault-secrets==4.9.0 +azure-keyvault-secrets==4.11.0 jinja2==3.1.6 \ No newline at end of file diff --git a/config/search/search.j2 b/config/search/search.j2 index d1920935..392a189e 100644 --- a/config/search/search.j2 +++ b/config/search/search.j2 @@ -56,8 +56,18 @@ { "name": "relatedImages", "type": "Collection(Edm.String)", "searchable": false, "retrievable": true }, { "name": "relatedFiles", "type": "Collection(Edm.String)", "searchable": false, "retrievable": true }, { "name": "source", "type": "Edm.String", "searchable": false, "retrievable": true, "filterable": true }, - { "name": "contentVector", "type": "Collection(Edm.Single)", "searchable": true, "retrievable": true, "dimensions": "{{EMBEDDINGS_VECTOR_DIMENSIONS}}", "vectorSearchProfile": "default" }, - { "name": "captionVector", "type": "Collection(Edm.Single)", "searchable": true, "retrievable": true, "dimensions": "{{EMBEDDINGS_VECTOR_DIMENSIONS}}", "vectorSearchProfile": "default" } + { "name": "provenance_id", "type": "Edm.String", "searchable": false, "retrievable": true, "filterable": true }, + { "name": "source_uri_id", "type": "Edm.String", "searchable": false, "retrievable": true, "filterable": true }, + { "name": "source_version_id", "type": "Edm.String", "searchable": false, "retrievable": true, "filterable": true }, + { "name": "content_checksum_sha256", "type": "Edm.String", "searchable": false, "retrievable": true, "filterable": true }, + { "name": "ingested_at", "type": "Edm.DateTimeOffset", "searchable": false, "retrievable": true, "filterable": true, "sortable": true }, + { "name": "ingest_run_id", "type": "Edm.String", "searchable": false, "retrievable": true, "filterable": true }, + { "name": "data_classification", "type": "Edm.String", "searchable": false, "retrievable": true, "filterable": true, "facetable": true }, + { "name": "right_to_use", "type": "Edm.String", "searchable": false, "retrievable": true, "filterable": true, "facetable": true }, + { "name": "retention_class", "type": "Edm.String", "searchable": false, "retrievable": true, "filterable": true, "facetable": true }, + { "name": "delete_after", "type": "Edm.String", "searchable": false, "retrievable": true, "filterable": true, "sortable": true }, + { "name": "contentVector", "type": "Collection(Edm.Single)", "searchable": true, "retrievable": true, "dimensions": {{EMBEDDINGS_VECTOR_DIMENSIONS}}, "vectorSearchProfile": "default" }, + { "name": "captionVector", "type": "Collection(Edm.Single)", "searchable": true, "retrievable": true, "dimensions": {{EMBEDDINGS_VECTOR_DIMENSIONS}}, "vectorSearchProfile": "default" } ], "corsOptions": { "allowedOrigins": ["*"], @@ -128,7 +138,7 @@ { "name": "question", "type": "Edm.String", "searchable": true, "retrievable": true, "analyzer": "{{SEARCH_ANALYZER_NAME}}" }, { "name": "query", "type": "Edm.String", "searchable": false }, { "name": "reasoning", "type": "Edm.String", "searchable": true }, - { "name": "contentVector", "type": "Collection(Edm.Single)", "searchable": true, "retrievable": true, "dimensions": "{{EMBEDDINGS_VECTOR_DIMENSIONS}}", "vectorSearchProfile": "default" } + { "name": "contentVector", "type": "Collection(Edm.Single)", "searchable": true, "retrievable": true, "dimensions": {{EMBEDDINGS_VECTOR_DIMENSIONS}}, "vectorSearchProfile": "default" } ], "corsOptions": { "allowedOrigins": ["*"], @@ -205,7 +215,7 @@ { "name": "examples", "type": "Collection(Edm.String)" } ] }, - { "name": "contentVector", "type": "Collection(Edm.Single)", "searchable": true, "retrievable": true, "dimensions": "{{EMBEDDINGS_VECTOR_DIMENSIONS}}", "vectorSearchProfile": "default" } + { "name": "contentVector", "type": "Collection(Edm.Single)", "searchable": true, "retrievable": true, "dimensions": {{EMBEDDINGS_VECTOR_DIMENSIONS}}, "vectorSearchProfile": "default" } ], "corsOptions": { "allowedOrigins": ["*"], @@ -276,7 +286,7 @@ { "name": "source_table", "type": "Edm.String", "searchable": true, "filterable": true }, { "name": "data_type", "type": "Edm.String", "searchable": true }, { "name": "source_model", "type": "Edm.String", "searchable": true }, - { "name": "contentVector", "type": "Collection(Edm.Single)", "searchable": true, "retrievable": true, "dimensions": "{{EMBEDDINGS_VECTOR_DIMENSIONS}}", "vectorSearchProfile": "default" } + { "name": "contentVector", "type": "Collection(Edm.Single)", "searchable": true, "retrievable": true, "dimensions": {{EMBEDDINGS_VECTOR_DIMENSIONS}}, "vectorSearchProfile": "default" } ], "corsOptions": { "allowedOrigins": ["*"], diff --git a/config/search/setup.py b/config/search/setup.py index f91066b0..c118240f 100644 --- a/config/search/setup.py +++ b/config/search/setup.py @@ -358,7 +358,7 @@ def render_and_parse_json(template_name_inner: str, ctx: dict) -> Optional[dict] return result, context # ── Azure Search API Call ─────────────────────────────────────────────────-- -def call_search_api(endpoint: str, api_version: str, rtype: str, rname: str, method: str, cred: ChainedTokenCredential, body: Any = None, max_retries: int = 3) -> bool: +def call_search_api(endpoint: str, api_version: str, rtype: str, rname: str, method: str, cred: ChainedTokenCredential, body: Any = None, max_retries: int = 3, if_match: Optional[str] = None, if_none_match: Optional[str] = None) -> bool: """ Call Azure Search REST API with retry logic for authentication failures. """ @@ -367,6 +367,10 @@ def call_search_api(endpoint: str, api_version: str, rtype: str, rname: str, met # Get fresh token on each attempt token = cred.get_token("https://search.azure.com/.default").token headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"} + if if_match: + headers["If-Match"] = if_match + if if_none_match: + headers["If-None-Match"] = if_none_match url = f"{endpoint}/{rtype}/{rname}?api-version={api_version}" resp = getattr(requests, method.lower())(url, headers=headers, json=body) @@ -387,12 +391,135 @@ def call_search_api(endpoint: str, api_version: str, rtype: str, rname: str, met return False else: logging.warning(f"⚠️ Attempt {attempt + 1}/{max_retries} failed for {method.upper()} {rtype}/{rname}: {e}") - logging.warning(f" Retrying in 2 seconds...") + logging.warning(" Retrying in 2 seconds...") time.sleep(2) # Continue to next attempt - cred.get_token() will be called again return False # Should never reach here, but just in case + +def get_search_resource( + endpoint: str, + api_version: str, + resource_type: str, + resource_name: str, + credential: ChainedTokenCredential, +) -> Tuple[Optional[dict], Optional[str]]: + """Return an existing Search resource and ETag, or two ``None`` values.""" + token = credential.get_token("https://search.azure.com/.default").token + response = requests.get( + f"{endpoint}/{resource_type}/{resource_name}?api-version={api_version}", + headers={ + "Authorization": f"Bearer {token}", + "Content-Type": "application/json", + }, + ) + if response.status_code == 404: + return None, None + if response.status_code >= 400: + raise RuntimeError( + f"GET {resource_type}/{resource_name} failed with " + f"HTTP {response.status_code}; no update was attempted." + ) + payload = response.json() + etag = response.headers.get("ETag") or payload.get("@odata.etag") + return strip_odata_metadata(payload), etag + + +_INDEX_FIELD_COMPATIBILITY_PROPERTIES = ( + "type", + "key", + "searchable", + "filterable", + "sortable", + "facetable", + "retrievable", + "analyzer", + "normalizer", + "dimensions", + "vectorSearchProfile", +) + + +def merge_additive_index_schema( + existing_index: dict, + desired_index: dict, +) -> Tuple[dict, list[str]]: + """Preserve the current index and append fields that are absent from it.""" + merged = strip_odata_metadata(existing_index) + existing_fields = { + field["name"]: field for field in merged.get("fields", []) + } + added_fields: list[str] = [] + + for desired_field in desired_index.get("fields", []): + name = desired_field["name"] + current_field = existing_fields.get(name) + if current_field is None: + merged.setdefault("fields", []).append(desired_field) + existing_fields[name] = desired_field + added_fields.append(name) + continue + + incompatible_properties = [] + for property_name in _INDEX_FIELD_COMPATIBILITY_PROPERTIES: + if property_name not in desired_field: + continue + current_value = current_field.get(property_name) + desired_value = desired_field[property_name] + if property_name == "dimensions": + try: + current_value = int(current_value) + desired_value = int(desired_value) + except (TypeError, ValueError): + incompatible_properties.append(property_name) + continue + if current_value != desired_value: + incompatible_properties.append(property_name) + if incompatible_properties: + properties = ", ".join(incompatible_properties) + raise ValueError( + f"Index field '{name}' is incompatible with the desired schema " + f"for properties: {properties}. The existing index was not modified." + ) + + return merged, added_fields + + +def prepare_index_updates( + definitions: dict, + credential: ChainedTokenCredential, + search_endpoint: str, + api_version: str, +) -> list[Tuple[str, Optional[dict], list[str], Optional[str], Optional[str]]]: + """Build every index update before any dependent resource is deleted.""" + updates = [] + for desired_index in definitions.get("indexes", []): + name = desired_index["name"] + existing, etag = get_search_resource( + search_endpoint, + api_version, + "indexes", + name, + credential, + ) + if existing is None: + body = desired_index + added_fields = [ + field["name"] for field in desired_index.get("fields", []) + ] + if_none_match = "*" + else: + body, added_fields = merge_additive_index_schema( + existing, + desired_index, + ) + if not added_fields: + body = None + if_none_match = None + updates.append((name, body, added_fields, etag, if_none_match)) + return updates + # ── Resource Provisioning ─────────────────────────────────────────────────-- def provision_datasources(defs: dict, context: dict, cred: ChainedTokenCredential, ds_to_indexers: dict, search_endpoint: str, api_version: str): logging.info("Creating datasources...") @@ -404,13 +531,59 @@ def provision_datasources(defs: dict, context: dict, cred: ChainedTokenCredentia call_search_api(search_endpoint, api_version, "datasources", name, "delete", cred) call_search_api(search_endpoint, api_version, "datasources", name, "put", cred, body) -def provision_indexes(defs: dict, context: dict, cred: ChainedTokenCredential, search_endpoint: str, api_version: str): - logging.info("Creating indexes...") - for idx in defs.get("indexes", []): - body = idx - name = body["name"] - call_search_api(search_endpoint, api_version, "indexes", name, "delete", cred) - call_search_api(search_endpoint, api_version, "indexes", name, "put", cred, body) +def provision_indexes( + defs: dict, + context: dict, + cred: ChainedTokenCredential, + search_endpoint: str, + api_version: str, + prepared_updates: Optional[ + list[ + Tuple[ + str, + Optional[dict], + list[str], + Optional[str], + Optional[str], + ] + ] + ] = None, +): + logging.info("Creating or additively updating indexes...") + updates = prepared_updates or prepare_index_updates( + defs, + cred, + search_endpoint, + api_version, + ) + for name, body, added_fields, etag, if_none_match in updates: + if body is None: + logging.info( + "✅ Index '%s' already contains the desired fields; skipping update.", + name, + ) + continue + + if not call_search_api( + search_endpoint, + api_version, + "indexes", + name, + "put", + cred, + body, + if_match=etag, + if_none_match=if_none_match, + ): + raise RuntimeError( + f"Additive update failed for index '{name}'. The existing index " + "was not deleted." + ) + logging.info( + "✅ Index '%s' updated in place with %d field(s).", + name, + len(added_fields), + ) def provision_skillsets(defs: dict, context: dict, cred: ChainedTokenCredential, search_endpoint: str, api_version: str): logging.info("Creating skillsets...") @@ -753,12 +926,32 @@ def execute_setup(defs: Optional[dict], context: dict): logging.error("❗️ SEARCH_API_VERSION not found in search.env; skipping Azure Search setup.") return + # Validate and prepare every index update before deleting any dependent + # resources. Unsupported schema changes and transient reads fail without + # disturbing the current knowledge bases, sources, indexers, or documents. + prepared_index_updates = prepare_index_updates( + defs, + cred, + search_endpoint, + api_version, + ) + + # Apply index changes while all dependent resources are still intact. A + # conditional-write failure leaves the current Search topology untouched. + provision_indexes( + defs, + context, + cred, + search_endpoint, + api_version, + prepared_index_updates, + ) + # Step 1: Clean up knowledge base resources in correct order (KB -> KS) cleanup_knowledge_resources(defs, context, cred, search_endpoint) - # Step 2: Provision standard search resources (now indexes can be deleted safely) + # Step 2: Reconcile the remaining standard Search resources. provision_datasources(defs, context, cred, ds_to_indexers, search_endpoint, api_version) - provision_indexes(defs, context, cred, search_endpoint, api_version) provision_skillsets(defs, context, cred, search_endpoint, api_version) provision_indexers(defs, context, cred, search_endpoint, api_version) diff --git a/config/search/tests/test_foundry_iq_templates.py b/config/search/tests/test_foundry_iq_templates.py index db06f082..313afaba 100644 --- a/config/search/tests/test_foundry_iq_templates.py +++ b/config/search/tests/test_foundry_iq_templates.py @@ -17,7 +17,12 @@ TEMPLATE_DIR = Path(__file__).resolve().parents[1] REPO_ROOT = Path(__file__).resolve().parents[3] FIXTURE_DIR = Path(__file__).resolve().parent / "fixtures" -MCP_RUNTIME_CONTRACT_VERSION = "v3.7.0" +MCP_RUNTIME_CONTRACT_MIN_VERSION = "v3.7.0" +MCP_FIXTURE_CONTRACT_VERSION = "v3.7.0" + + +def version_tuple(tag): + return tuple(int(part) for part in tag.removeprefix("v").split(".")) def render_json_template(template_name, context): @@ -645,7 +650,7 @@ def test_web_grounding_enabled_without_domains_emits_empty_lists(self): {"domains": {"allowedDomains": [], "blockedDomains": []}}, ) - + class FoundryIqMcpTemplateTests(unittest.TestCase): """Generic MCP Server knowledge sources are opt-in and default-off. Rendered output must be unchanged when the feature is disabled. @@ -905,11 +910,14 @@ def test_canonical_fixture_shared_with_runtime_validates_and_renders(self): for component in manifest["components"] if component["name"] == "gpt-rag-orchestrator" ) - self.assertEqual(orchestrator["tag"], MCP_RUNTIME_CONTRACT_VERSION) + self.assertGreaterEqual( + version_tuple(orchestrator["tag"]), + version_tuple(MCP_RUNTIME_CONTRACT_MIN_VERSION), + ) fixture_path = ( FIXTURE_DIR - / f"foundry_iq_mcp_canonical_source_{MCP_RUNTIME_CONTRACT_VERSION.removeprefix('v').replace('.', '_')}.json" + / f"foundry_iq_mcp_canonical_source_{MCP_FIXTURE_CONTRACT_VERSION.removeprefix('v').replace('.', '_')}.json" ) source = json.loads(fixture_path.read_text(encoding="utf-8")) context["FOUNDRY_IQ_MCP_TRUSTED_HOSTS"] = "mcp.contoso.com" diff --git a/config/search/tests/test_governance_schema.py b/config/search/tests/test_governance_schema.py new file mode 100644 index 00000000..d7894d8b --- /dev/null +++ b/config/search/tests/test_governance_schema.py @@ -0,0 +1,261 @@ +import json +import unittest +from pathlib import Path +from unittest.mock import Mock, patch + +from jinja2 import Environment, FileSystemLoader, StrictUndefined + +from config.search import setup + + +TEMPLATE_DIR = Path(__file__).resolve().parents[1] +PROVENANCE_FIELDS = { + "provenance_id": ("Edm.String", {"filterable": True}), + "source_uri_id": ("Edm.String", {"filterable": True}), + "source_version_id": ("Edm.String", {"filterable": True}), + "content_checksum_sha256": ("Edm.String", {"filterable": True}), + "ingested_at": ( + "Edm.DateTimeOffset", + {"filterable": True, "sortable": True}, + ), + "ingest_run_id": ("Edm.String", {"filterable": True}), + "data_classification": ( + "Edm.String", + {"filterable": True, "facetable": True}, + ), + "right_to_use": ("Edm.String", {"filterable": True, "facetable": True}), + "retention_class": ( + "Edm.String", + {"filterable": True, "facetable": True}, + ), + "delete_after": ( + "Edm.String", + {"filterable": True, "sortable": True}, + ), +} + + +def render(template_name, context): + environment = Environment( + loader=FileSystemLoader(TEMPLATE_DIR), + undefined=StrictUndefined, + ) + return json.loads(environment.get_template(template_name).render(**context)) + + +def search_context(): + settings = render( + "search.settings.j2", + { + "RESOURCE_TOKEN": "abc123", + "SEARCH_SERVICE_QUERY_ENDPOINT": "https://search.search.windows.net", + "AI_FOUNDRY_ACCOUNT_NAME": "aif-abc123", + "RETRIEVAL_BACKEND": "foundry_iq", + }, + ) + return { + **settings, + "STORAGE_ACCOUNT_RESOURCE_ID": ( + "/subscriptions/s/resourceGroups/rg/providers/" + "Microsoft.Storage/storageAccounts/st" + ), + "EMBEDDING_MODEL_INFO": { + "endpoint": "https://aif-abc123.openai.azure.com/", + "deployment_name": "text-embedding", + "model_name": "text-embedding-3-large", + }, + "GPT_MODEL_INFO": { + "deployment_name": "chat", + "model_name": "gpt-5.2", + "model_format": "OpenAI", + }, + } + + +class GovernanceSearchSchemaTests(unittest.TestCase): + def test_rag_index_contains_provenance_fields_with_safe_types(self): + definitions = render("search.j2", search_context()) + rag_index = definitions["indexes"][0] + fields = {field["name"]: field for field in rag_index["fields"]} + self.assertIsInstance(fields["contentVector"]["dimensions"], int) + + for name, (expected_type, expected_attributes) in PROVENANCE_FIELDS.items(): + with self.subTest(name=name): + self.assertEqual(fields[name]["type"], expected_type) + self.assertTrue(fields[name]["retrievable"]) + self.assertFalse(fields[name]["searchable"]) + for attribute, expected_value in expected_attributes.items(): + self.assertEqual(fields[name][attribute], expected_value) + + def test_additive_merge_preserves_documents_and_custom_fields(self): + existing = { + "@odata.etag": '"etag"', + "name": "rag-index", + "fields": [ + {"name": "id", "type": "Edm.String", "key": True}, + {"name": "operator_custom", "type": "Edm.String"}, + ], + "semantic": {"configurations": [{"name": "operator-owned"}]}, + } + desired = { + "name": "rag-index", + "fields": [ + {"name": "id", "type": "Edm.String", "key": True}, + { + "name": "provenance_id", + "type": "Edm.String", + "filterable": True, + }, + ], + } + + merged, added = setup.merge_additive_index_schema(existing, desired) + + self.assertEqual(added, ["provenance_id"]) + self.assertNotIn("@odata.etag", merged) + self.assertIn( + "operator_custom", + {field["name"] for field in merged["fields"]}, + ) + self.assertEqual( + merged["semantic"], + {"configurations": [{"name": "operator-owned"}]}, + ) + + def test_incompatible_existing_field_fails_without_mutation(self): + existing = { + "name": "rag-index", + "fields": [ + {"name": "delete_after", "type": "Edm.DateTimeOffset"} + ], + } + desired = { + "name": "rag-index", + "fields": [ + {"name": "delete_after", "type": "Edm.String"} + ], + } + + with self.assertRaisesRegex(ValueError, "existing index was not modified"): + setup.merge_additive_index_schema(existing, desired) + + self.assertEqual(existing["fields"][0]["type"], "Edm.DateTimeOffset") + + def test_existing_index_is_updated_with_put_and_never_deleted(self): + credential = Mock() + existing = { + "name": "rag-index", + "fields": [{"name": "id", "type": "Edm.String", "key": True}], + } + desired = { + "name": "rag-index", + "fields": [ + {"name": "id", "type": "Edm.String", "key": True}, + { + "name": "provenance_id", + "type": "Edm.String", + "filterable": True, + }, + ], + } + + with patch.object( + setup, + "get_search_resource", + return_value=(existing, '"etag"'), + ), patch.object(setup, "call_search_api", return_value=True) as call: + setup.provision_indexes( + {"indexes": [desired]}, + {}, + credential, + "https://search.search.windows.net", + "2025-09-01", + ) + + call.assert_called_once() + self.assertEqual(call.call_args.args[4], "put") + self.assertEqual(call.call_args.kwargs["if_match"], '"etag"') + self.assertIsNone(call.call_args.kwargs["if_none_match"]) + self.assertNotIn( + "delete", + [argument for argument in call.call_args.args if isinstance(argument, str)], + ) + + def test_schema_preflight_fails_before_dependent_cleanup(self): + definitions = {"indexes": [{"name": "rag-index", "fields": []}]} + context = { + "SEARCH_SERVICE_QUERY_ENDPOINT": "https://search.search.windows.net", + "SEARCH_API_VERSION": "2025-09-01", + } + + with patch.object( + setup, + "prepare_index_updates", + side_effect=ValueError("incompatible"), + ), patch.object(setup, "cleanup_knowledge_resources") as cleanup: + with self.assertRaisesRegex(ValueError, "incompatible"): + setup.execute_setup(definitions, context) + + cleanup.assert_not_called() + + def test_missing_index_uses_conditional_create(self): + desired = { + "name": "rag-index", + "fields": [{"name": "id", "type": "Edm.String", "key": True}], + } + + with patch.object( + setup, + "get_search_resource", + return_value=(None, None), + ), patch.object(setup, "call_search_api", return_value=True) as call: + setup.provision_indexes( + {"indexes": [desired]}, + {}, + Mock(), + "https://search.search.windows.net", + "2025-09-01", + ) + + self.assertEqual(call.call_args.kwargs["if_none_match"], "*") + self.assertIsNone(call.call_args.kwargs["if_match"]) + + def test_index_update_runs_before_dependent_cleanup(self): + definitions = {"indexes": [{"name": "rag-index", "fields": []}]} + context = { + "SEARCH_SERVICE_QUERY_ENDPOINT": "https://search.search.windows.net", + "SEARCH_API_VERSION": "2025-09-01", + } + events = [] + prepared = [("rag-index", None, [], '"etag"', None)] + + with patch.object( + setup, + "prepare_index_updates", + return_value=prepared, + ), patch.object( + setup, + "provision_indexes", + side_effect=lambda *args: events.append("indexes"), + ), patch.object( + setup, + "cleanup_knowledge_resources", + side_effect=lambda *args: events.append("cleanup"), + ), patch.object(setup, "provision_datasources"), patch.object( + setup, "provision_skillsets" + ), patch.object(setup, "provision_indexers"), patch.object( + setup, "filter_work_iq_sources" + ), patch.object( + setup, "provision_knowledge_sources", return_value=True + ), patch.object( + setup, "enforce_private_execution_for_generated_indexers" + ), patch.object( + setup, "provision_knowledge_bases", return_value=True + ): + setup.execute_setup(definitions, context) + + self.assertEqual(events, ["indexes", "cleanup"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/contracts/README.md b/contracts/README.md new file mode 100644 index 00000000..819db1af --- /dev/null +++ b/contracts/README.md @@ -0,0 +1,13 @@ +# Audit event v1 contract + +These schemas are the shared GPT-RAG audit contract consumed by orchestrator +v3.8.0 and ingestion v2.5.0. + +`audit-event-v1.sha256` pins the exact LF-encoded bytes: + +- Logical schema: `825db8ef40a81e2c19e5d80d37c565b6b47fc9a6540e9881d35cc12b8fde5aab` +- Application Insights wire schema: `066c8f5408610ab839d5121d06ca5bc59e8797e551d5c47c875c5ba52f7e0588` + +Consumers must use `schema_version` when interpreting events and ignore unknown +optional fields. Technical audit evidence supports operator governance work but +does not establish legal or regulatory compliance. diff --git a/contracts/audit-event-v1.application-insights.schema.json b/contracts/audit-event-v1.application-insights.schema.json new file mode 100644 index 00000000..86e90f3d --- /dev/null +++ b/contracts/audit-event-v1.application-insights.schema.json @@ -0,0 +1,98 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/Azure/GPT-RAG/contracts/audit-event-v1.application-insights.schema.json", + "title": "GPT-RAG audit event v1 Application Insights wire shape", + "description": "Azure Monitor represents custom-event property values as strings and drops null custom properties. Parse and type the properties according to audit-event-v1.schema.json, decode the root parent sentinel to logical null, then perform logical contract validation.", + "type": "object", + "additionalProperties": false, + "required": ["name", "properties"], + "properties": { + "name": { + "type": "string", + "enum": [ + "gptrag.audit.request.started", + "gptrag.audit.request.completed", + "gptrag.audit.request.failed", + "gptrag.audit.request.cancelled", + "gptrag.audit.route.selected", + "gptrag.audit.grounding.source.selected", + "gptrag.audit.grounding.source.rejected", + "gptrag.audit.tool.invocation.started", + "gptrag.audit.tool.invocation.completed", + "gptrag.audit.tool.invocation.failed", + "gptrag.audit.tool.invocation.cancelled", + "gptrag.audit.outcome.produced", + "gptrag.audit.outcome.rejected", + "gptrag.audit.audit.emission.failed", + "gptrag.audit.ingestion.run.started", + "gptrag.audit.ingestion.run.completed", + "gptrag.audit.ingestion.run.failed", + "gptrag.audit.ingestion.run.cancelled", + "gptrag.audit.ingestion.document.indexed", + "gptrag.audit.ingestion.document.rejected", + "gptrag.audit.ingestion.document.deleted" + ] + }, + "properties": { + "type": "object", + "maxProperties": 64, + "required": [ + "schema_version", + "event_id", + "event_type", + "event_time_utc", + "correlation_id", + "trace_id", + "span_id", + "parent_event_id", + "service_name", + "service_version", + "environment", + "operation", + "status", + "reason_code", + "capture_mode", + "redaction_applied", + "omitted_fields", + "truncated_fields" + ], + "properties": { + "event_type": { + "type": "string", + "enum": [ + "request.started", + "request.completed", + "request.failed", + "request.cancelled", + "route.selected", + "grounding.source.selected", + "grounding.source.rejected", + "tool.invocation.started", + "tool.invocation.completed", + "tool.invocation.failed", + "tool.invocation.cancelled", + "outcome.produced", + "outcome.rejected", + "audit.emission.failed", + "ingestion.run.started", + "ingestion.run.completed", + "ingestion.run.failed", + "ingestion.run.cancelled", + "ingestion.document.indexed", + "ingestion.document.rejected", + "ingestion.document.deleted" + ] + }, + "parent_event_id": { + "type": "string", + "pattern": "^evt_[0-9a-f]{32}$", + "description": "Wire parent event ID. Exporters must encode logical null as evt_00000000000000000000000000000000 because the pinned Azure Monitor exporter drops null custom properties. Consumers must decode this sentinel back to null and must never join it as an event." + } + }, + "additionalProperties": { + "type": "string", + "maxLength": 16384 + } + } + } +} diff --git a/contracts/audit-event-v1.schema.json b/contracts/audit-event-v1.schema.json new file mode 100644 index 00000000..44c65c1a --- /dev/null +++ b/contracts/audit-event-v1.schema.json @@ -0,0 +1,208 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/Azure/GPT-RAG/contracts/audit-event-v1.schema.json", + "title": "GPT-RAG audit event v1", + "description": "Shared logical audit event contract owned by Azure/GPT-RAG. Readers of major version 1 must ignore unknown optional fields. Logical root events use a null parent_event_id; the Azure Monitor wire sentinel is not a logical event ID.", + "type": "object", + "additionalProperties": true, + "maxProperties": 64, + "required": [ + "schema_version", + "event_id", + "event_type", + "event_time_utc", + "correlation_id", + "trace_id", + "span_id", + "parent_event_id", + "service_name", + "service_version", + "environment", + "operation", + "status", + "reason_code", + "capture_mode", + "redaction_applied", + "omitted_fields", + "truncated_fields" + ], + "properties": { + "schema_version": { "const": 1 }, + "event_id": { + "type": "string", + "pattern": "^evt_[0-9a-f]{32}$", + "not": { "const": "evt_00000000000000000000000000000000" }, + "description": "Logical event ID. The all-zero Azure Monitor wire sentinel is reserved and cannot identify an event." + }, + "event_type": { + "enum": [ + "request.started", + "request.completed", + "request.failed", + "request.cancelled", + "route.selected", + "grounding.source.selected", + "grounding.source.rejected", + "tool.invocation.started", + "tool.invocation.completed", + "tool.invocation.failed", + "tool.invocation.cancelled", + "outcome.produced", + "outcome.rejected", + "audit.emission.failed", + "ingestion.run.started", + "ingestion.run.completed", + "ingestion.run.failed", + "ingestion.run.cancelled", + "ingestion.document.indexed", + "ingestion.document.rejected", + "ingestion.document.deleted" + ] + }, + "event_time_utc": { + "type": "string", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{6}Z$" + }, + "correlation_id": { "type": "string", "pattern": "^req_[0-9a-f]{32}$" }, + "trace_id": { "type": "string", "pattern": "^[0-9a-f]{32}$" }, + "span_id": { "type": "string", "pattern": "^[0-9a-f]{16}$" }, + "parent_event_id": { + "type": ["string", "null"], + "pattern": "^evt_[0-9a-f]{32}$", + "not": { "const": "evt_00000000000000000000000000000000" }, + "description": "Logical parent event ID. Null identifies a root event. Exporters encode null as the reserved all-zero event ID only in the Azure Monitor wire shape." + }, + "service_name": { "type": "string", "maxLength": 512 }, + "service_version": { "type": "string", "maxLength": 512 }, + "environment": { "type": "string", "maxLength": 64 }, + "operation": { "type": "string", "maxLength": 512 }, + "status": { + "enum": [ + "started", + "completed", + "failed", + "cancelled", + "selected", + "rejected", + "produced" + ] + }, + "reason_code": { + "enum": [ + "none", + "request_received", + "request_completed", + "request_failed", + "request_cancelled", + "client_disconnected", + "partial_output", + "strategy_configured", + "direct_model_selected", + "agent_selected", + "source_selected", + "source_rejected", + "source_empty", + "source_limit_reached", + "tool_invoked", + "tool_completed", + "tool_failed", + "tool_cancelled", + "timeout", + "outcome_produced", + "outcome_rejected", + "validation_failed", + "redaction_failure", + "serialization_failure", + "event_too_large", + "attribute_limit_exceeded", + "export_failure", + "unknown" + ] + }, + "capture_mode": { "enum": ["metadata_only", "sensitive_allowlist"] }, + "redaction_applied": { "type": "boolean" }, + "omitted_fields": { + "type": "array", + "maxItems": 32, + "items": { "type": "string", "maxLength": 512 } + }, + "truncated_fields": { + "type": "array", + "maxItems": 32, + "items": { "type": "string", "maxLength": 512 } + }, + "started_at_utc": { + "type": "string", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{6}Z$" + }, + "duration_ms": { "type": "number", "minimum": 0, "maximum": 86400000 }, + "decision_type": { "type": "string", "maxLength": 512 }, + "decision_value": { "type": "string", "maxLength": 512 }, + "source_id": { "type": ["string", "null"], "maxLength": 512 }, + "source_type": { "type": "string", "maxLength": 512 }, + "source_rank": { "type": "integer", "minimum": 0 }, + "tool_name": { "type": "string", "maxLength": 512 }, + "tool_id": { "type": ["string", "null"], "maxLength": 512 }, + "tool_invocation_id": { "type": ["string", "null"], "maxLength": 512 }, + "outcome_type": { "type": "string", "maxLength": 512 }, + "failure_type": { "type": "string", "maxLength": 512 }, + "input_count": { "type": "integer", "minimum": 0 }, + "output_count": { "type": "integer", "minimum": 0 }, + "source_count": { "type": "integer", "minimum": 0 }, + "partial_output": { "type": "boolean" }, + "http_status_code": { "type": "integer", "minimum": 100, "maximum": 599 }, + "transport": { "type": "string", "maxLength": 512 }, + "actor_id": { "type": ["string", "null"], "maxLength": 512 }, + "conversation_id": { "type": ["string", "null"], "maxLength": 512 }, + "question_id": { "type": ["string", "null"], "maxLength": 512 }, + "thread_id": { "type": ["string", "null"], "maxLength": 512 }, + "hmac_key_id": { "type": "string", "maxLength": 512 }, + "timing_source": { + "enum": ["observed", "reconstructed"], + "description": "How the activity start time was obtained. Reconstructed timestamps are approximate." + }, + "audit_events_omitted": { + "type": "integer", + "minimum": 0, + "description": "Number of audit detail events omitted during this request." + }, + "source_events_omitted": { + "type": "integer", + "minimum": 0, + "description": "Number of grounding-source events omitted during this request." + }, + "tool_invocations_omitted": { + "type": "integer", + "minimum": 0, + "description": "Number of complete tool invocation event pairs omitted during this request." + }, + "prompt": { "type": "string", "maxLength": 2048 }, + "response": { "type": "string", "maxLength": 2048 }, + "source_excerpt": { "type": "string", "maxLength": 2048 }, + "tool_arguments": { "type": "string", "maxLength": 2048 }, + "tool_result": { "type": "string", "maxLength": 2048 } + }, + "allOf": [ + { + "if": { + "properties": { + "event_type": { + "enum": [ + "request.completed", + "request.failed", + "request.cancelled", + "tool.invocation.completed", + "tool.invocation.failed", + "tool.invocation.cancelled", + "ingestion.run.completed", + "ingestion.run.failed", + "ingestion.run.cancelled" + ] + } + }, + "required": ["event_type"] + }, + "then": { "required": ["started_at_utc", "duration_ms"] } + } + ] +} diff --git a/contracts/audit-event-v1.sha256 b/contracts/audit-event-v1.sha256 new file mode 100644 index 00000000..a7a40b2c --- /dev/null +++ b/contracts/audit-event-v1.sha256 @@ -0,0 +1,2 @@ +825db8ef40a81e2c19e5d80d37c565b6b47fc9a6540e9881d35cc12b8fde5aab audit-event-v1.schema.json +066c8f5408610ab839d5121d06ca5bc59e8797e551d5c47c875c5ba52f7e0588 audit-event-v1.application-insights.schema.json diff --git a/manifest.json b/manifest.json index f908592b..975f3b4c 100644 --- a/manifest.json +++ b/manifest.json @@ -1,5 +1,5 @@ { - "tag": "v3.6.0", + "tag": "v3.7.0", "repo": "https://github.com/azure/gpt-rag.git", "ailz_tag": "v2.3.0", "components": [ @@ -11,12 +11,12 @@ { "name": "gpt-rag-orchestrator", "repo": "https://github.com/azure/gpt-rag-orchestrator.git", - "tag": "v3.7.0" + "tag": "v3.8.0" }, { "name": "gpt-rag-ingestion", "repo": "https://github.com/azure/gpt-rag-ingestion.git", - "tag": "v2.4.14" + "tag": "v2.5.0" } ] } \ No newline at end of file diff --git a/scripts/postProvision.ps1 b/scripts/postProvision.ps1 index cf512bde..fdc2397a 100755 --- a/scripts/postProvision.ps1 +++ b/scripts/postProvision.ps1 @@ -654,7 +654,20 @@ Invoke-NativeCommand { & python -m pip install -r config/requirements.txt } if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } #------------------------------------------------------------------------------- -# 1) AI Foundry Setup +# 1) Governance and audit configuration +#------------------------------------------------------------------------------- +if (-not $missing.Contains('APP_CONFIG_ENDPOINT')) { + Write-Host "`n🔐 Governance and audit configuration..." + Write-Host "🚀 Running config.governance.setup..." + Invoke-PythonModule -ModuleName 'config.governance.setup' + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + Write-Host "✅ Governance and audit configuration finished." +} else { + Write-Host "⏭️ Skipping governance setup (missing APP_CONFIG_ENDPOINT)." +} + +#------------------------------------------------------------------------------- +# 2) AI Foundry Setup #------------------------------------------------------------------------------- if (-not $missing.Contains('APP_CONFIG_ENDPOINT')) { Write-Host "`n📑 AI Foundry Setup..." @@ -667,7 +680,7 @@ if (-not $missing.Contains('APP_CONFIG_ENDPOINT')) { } #------------------------------------------------------------------------------- -# 2) Container Apps Setup +# 3) Container Apps Setup #------------------------------------------------------------------------------- if (-not $missing.Contains('APP_CONFIG_ENDPOINT')) { Write-Host "`n🔍 ContainerApp setup..." @@ -680,7 +693,7 @@ if (-not $missing.Contains('APP_CONFIG_ENDPOINT')) { } #------------------------------------------------------------------------------- -# 3) AI Search Setup +# 4) AI Search Setup #------------------------------------------------------------------------------- if (-not $missing.Contains('APP_CONFIG_ENDPOINT')) { Write-Host "🔍 AI Search setup..." diff --git a/scripts/postProvision.sh b/scripts/postProvision.sh index 34ea39da..6dc51be0 100755 --- a/scripts/postProvision.sh +++ b/scripts/postProvision.sh @@ -110,7 +110,15 @@ pip install --upgrade pip pip install -r config/requirements.txt ############################################################################### -# 1) AI Foundry Setup +# 1) Governance and audit configuration +############################################################################### +echo +echo "🔐 Governance and audit configuration…" +python -m config.governance.setup +echo "✅ Governance and audit configuration finished." + +############################################################################### +# 2) AI Foundry Setup ############################################################################### echo echo "📑 AI Foundry Setup…" @@ -123,7 +131,7 @@ echo "📑 AI Foundry Setup…" } ############################################################################### -# 2) Container Apps Setup +# 3) Container Apps Setup ############################################################################### echo echo "🔍 ContainerApp setup…" @@ -136,7 +144,7 @@ echo "🔍 ContainerApp setup…" } ############################################################################### -# 3) AI Search Setup +# 4) AI Search Setup ############################################################################### echo echo "🔍 AI Search setup…"