From 4bcb2f1d9e20124543b04e0a6dde7ede945b7cf1 Mon Sep 17 00:00:00 2001 From: John Miller Date: Tue, 26 May 2026 23:44:40 -0400 Subject: [PATCH] Add networking guide and support for existing AI Foundry accounts - Introduced a comprehensive networking guide in `docs/networking.md` detailing network modes: public, managed, and BYO VNet. - Enhanced `ai-project.bicep` to support referencing existing AI Foundry accounts and added parameters for network configuration. - Created new Bicep modules for managing private endpoints and DNS zones, facilitating BYO VNet setups. - Updated `vnet.bicep` to handle both new and existing VNet scenarios, ensuring safe integration with hub-spoke topologies. - Modified `main.bicep` to incorporate network parameters and conditions for creating or referencing existing resources. - Adjusted `main.parameters.json` to include new parameters for existing accounts and network configurations. --- .azdignore | 4 +- README.md | 6 + docs/networking.md | 321 ++++++++++++++++++ infra/core/ai/ai-project.bicep | 142 +++++++- .../networking/private-endpoint-and-dns.bicep | 178 ++++++++++ infra/core/networking/vnet.bicep | 126 +++++++ infra/main.bicep | 130 ++++++- infra/main.parameters.json | 44 ++- 8 files changed, 931 insertions(+), 20 deletions(-) create mode 100644 docs/networking.md create mode 100644 infra/core/networking/private-endpoint-and-dns.bicep create mode 100644 infra/core/networking/vnet.bicep diff --git a/.azdignore b/.azdignore index 0cdfeeb..e542a38 100644 --- a/.azdignore +++ b/.azdignore @@ -6,4 +6,6 @@ CONTRIBUTING.md LICENSE.md README.md SECURITY.md -SUPPORT.md \ No newline at end of file +SUPPORT.md + +docs/ \ No newline at end of file diff --git a/README.md b/README.md index 800b363..221b582 100644 --- a/README.md +++ b/README.md @@ -116,6 +116,12 @@ You'll find agent samples in the [`foundry-samples` repo](https://github.com/mic This template does not use specific models. The model deployments are a parameter of the template. Each model may not be available in all Azure regions. Check for [up-to-date region availability of Microsoft Foundry](https://learn.microsoft.com/en-us/azure/ai-foundry/reference/region-support) and in particular the [Agent Service](https://learn.microsoft.com/en-us/azure/ai-foundry/agents/concepts/model-region-support?tabs=global-standard). +### Networking + +By default the Foundry account is publicly reachable (no VNet). The template also supports a Microsoft-managed network (`managed`) and a bring-your-own VNet posture (`byo-vnet`) with a delegated agent subnet, a private endpoint on the account, and private DNS zones. You can also point the template at an existing Foundry account. + +All of these are opt-in via `azd env set NAME VALUE` between `azd init` and `azd provision`. See [docs/networking.md](docs/networking.md) for the workflow, recipes (managed network, BYO VNet, existing account, hub-spoke DNS reuse), and troubleshooting. + ## Resource Clean-up To prevent incurring unnecessary charges, it's important to clean up your Azure resources after completing your work with the application. diff --git a/docs/networking.md b/docs/networking.md new file mode 100644 index 0000000..74141bb --- /dev/null +++ b/docs/networking.md @@ -0,0 +1,321 @@ +# Networking guide + +This template supports three network postures for the Azure AI Foundry +account it provisions, plus reuse of an existing account. Pick the one +that matches your environment. + +| Mode | Public traffic | Best for | +| --- | --- | --- | +| `none` (default) | Account is publicly reachable from anywhere. | Local dev, demos, getting started fast. | +| `managed` | Microsoft-managed network isolation; no customer VNet. | You want Microsoft to handle isolation without managing your own VNet. | +| `byo-vnet` | Customer-supplied VNet with a delegated agent subnet and a private endpoint on the account. | Enterprise / hub-spoke topologies; you need traffic to stay inside your VNet. | + +You can also point the template at an **existing Foundry account** (with +or without a VNet) and have it provision a new project on top. + +All of these options are opt-in via environment variables. The default +flow (`azd init` -> `azd up`) gives you the public `none` mode, unchanged. + +## How it works + +The template's Bicep params are bound to environment variables in +`infra/main.parameters.json`. To opt into any of the modes below: + +1. Run `azd init -t Azure-Samples/azd-ai-starter-basic` (or + `azd ai agent init`). +2. Run one or more `azd env set NAME VALUE` commands listed in the recipe + for your scenario. +3. Run `azd provision` (or `azd up`). + +If you change network mode after a first deploy, run `azd provision` +again. Going from `none` -> `byo-vnet` adds a VNet, subnets, private +endpoint, and DNS zones; going the other way removes them. + +## Recipe: Microsoft-managed network + +One env var: + +```bash +azd env set FOUNDRY_NETWORK_MODE managed +azd provision +``` + +The account is provisioned with `networkInjections.useMicrosoftManagedNetwork = true`. +No customer VNet is created. Microsoft handles isolation. + +## Recipe: Bring your own VNet (new VNet, default subnets) + +Two env vars: the mode, and your dev machine's public IP so the data +plane (e.g. `azd deploy`, `azd ai agent invoke`) keeps working from your +laptop. + +Linux / macOS / WSL: + +```bash +# 1. Fetch your public IP. Pick any reliable echo service. +MY_IP=$(curl -s https://api.ipify.org) + +# 2. Set env vars. +azd env set FOUNDRY_NETWORK_MODE byo-vnet +azd env set FOUNDRY_CLIENT_IP_ALLOW_LIST "[\"$MY_IP\"]" + +# 3. Provision. +azd provision +``` + +Windows PowerShell: + +```powershell +$myIp = (Invoke-RestMethod https://api.ipify.org) +azd env set FOUNDRY_NETWORK_MODE byo-vnet +azd env set FOUNDRY_CLIENT_IP_ALLOW_LIST "[`"$myIp`"]" +azd provision +``` + +This creates a new VNet `vnet-` in the deployment resource +group with two subnets: + +* `agent-subnet` (192.168.0.0/24) -- delegated to + `Microsoft.App/environments` for the hosted agent runtime. +* `pe-subnet` (192.168.1.0/24) -- holds the private endpoint for the + Foundry account. + +It also creates a private endpoint on the account and three private DNS +zones (`privatelink.services.ai.azure.com`, +`privatelink.openai.azure.com`, `privatelink.cognitiveservices.azure.com`) +linked to the VNet. + +Public access stays Enabled with a Deny ACL + AzureServices bypass + your +IP allow-list, so the Foundry control plane (which lives outside the +VNet) and your laptop both keep working. + +### Customizing the VNet and subnets + +Override any of these only if the defaults collide with another network +you peer with: + +```bash +azd env set FOUNDRY_VNET_NAME my-vnet +azd env set FOUNDRY_VNET_ADDRESS_PREFIX 10.20.0.0/16 +azd env set FOUNDRY_AGENT_SUBNET_NAME agents +azd env set FOUNDRY_AGENT_SUBNET_PREFIX 10.20.0.0/24 +azd env set FOUNDRY_PE_SUBNET_NAME private-endpoints +azd env set FOUNDRY_PE_SUBNET_PREFIX 10.20.1.0/24 +``` + +### Fully private (no public path) + +If your dev machine is inside the VNet (via VPN, ExpressRoute, or a +bastion VM), you can drop public access entirely: + +```bash +azd env set FOUNDRY_DISABLE_PUBLIC_NETWORK_ACCESS true +azd provision +``` + +Do not set this from a laptop that only has internet access -- you will +lose the ability to reach the Foundry control plane from your dev +environment. + +## Recipe: Bring your own VNet (existing VNet) + +Three env vars: mode, VNet ARM ID, IP allow-list. + +```bash +# Linux / macOS / WSL +azd env set FOUNDRY_NETWORK_MODE byo-vnet +azd env set FOUNDRY_VNET_RESOURCE_ID /subscriptions//resourceGroups//providers/Microsoft.Network/virtualNetworks/ +azd env set FOUNDRY_CLIENT_IP_ALLOW_LIST "[\"$(curl -s https://api.ipify.org)\"]" +azd provision +``` + +```powershell +# Windows PowerShell +azd env set FOUNDRY_NETWORK_MODE byo-vnet +azd env set FOUNDRY_VNET_RESOURCE_ID /subscriptions//resourceGroups//providers/Microsoft.Network/virtualNetworks/ +$myIp = (Invoke-RestMethod https://api.ipify.org) +azd env set FOUNDRY_CLIENT_IP_ALLOW_LIST "[`"$myIp`"]" +azd provision +``` + +Cross-RG and cross-subscription references work. The template does NOT +mutate the existing VNet -- it only reads it. The two subnets must +already exist on the VNet: + +* The agent subnet must be **delegated to + `Microsoft.App/environments`** (Bicep cannot retroactively add the + delegation on a referenced subnet). +* The private endpoint subnet must allow private endpoints (no policies + blocking them). + +By default the template looks for subnets named `agent-subnet` and +`pe-subnet`. Override with `FOUNDRY_AGENT_SUBNET_NAME` / +`FOUNDRY_PE_SUBNET_NAME` if your subnets are named differently. + +Region matters: the Foundry account is created in +`AZURE_AI_DEPLOYMENTS_LOCATION` (or `AZURE_LOCATION` if not set). If the +VNet lives in a different region, set `AZURE_AI_DEPLOYMENTS_LOCATION` to +match the VNet's region. + +## Recipe: Allowlist your dev IP + +If `azd ai agent invoke` or `azd deploy` starts failing with 403s or +hangs, your public IP probably rotated. Re-fetch and re-set: + +```bash +# Linux / macOS / WSL +azd env set FOUNDRY_CLIENT_IP_ALLOW_LIST "[\"$(curl -s https://api.ipify.org)\"]" +azd provision +``` + +```powershell +# Windows PowerShell +$myIp = (Invoke-RestMethod https://api.ipify.org) +azd env set FOUNDRY_CLIENT_IP_ALLOW_LIST "[`"$myIp`"]" +azd provision +``` + +The JSON-array value supports multiple IPs and CIDR ranges: + +```bash +azd env set FOUNDRY_CLIENT_IP_ALLOW_LIST '["203.0.113.45","203.0.113.0/24"]' +``` + +You can also update the account's network ACL out of band without +re-running `azd provision`: + +```bash +az cognitiveservices account network-rule add \ + --name \ + --resource-group \ + --ip-address +``` + +The change takes effect immediately. `azd provision` will reconcile back +to whatever `FOUNDRY_CLIENT_IP_ALLOW_LIST` contains the next time it +runs, so keep the env var in sync if you want the change to stick. + +## Recipe: Reuse an existing Foundry account + +Use this when you have a pre-existing Foundry account (created out of +band or by another team) and you want to provision a new project on top +of it. + +```bash +azd env set USE_EXISTING_AI_ACCOUNT true +azd env set AZURE_EXISTING_AI_ACCOUNT_RESOURCE_ID /subscriptions//resourceGroups//providers/Microsoft.CognitiveServices/accounts/ +azd provision +``` + +Constraints: + +* The account must live in the **same resource group** as this + deployment. +* The template creates a new project, the project's capability host, and + any requested model deployments / connections on the existing account. + It does NOT change the account's network configuration. +* If the existing account is in BYO VNet mode, also set + `FOUNDRY_NETWORK_MODE=byo-vnet` and the matching VNet params so the + new project's hosted agent runtime is wired to the same agent subnet. + +This pairs naturally with all the network modes above. + +## Recipe (advanced): Reuse existing private DNS zones (hub-spoke) + +In hub-spoke topologies the private DNS zones for AI services typically +live in a hub subscription / resource group, not the spoke. Point the +template at them so it doesn't try to create duplicates. + +The value is a JSON map from zone FQDN to the resource group that holds +the zone. Empty value for a zone means "create a new one in the spoke +RG" (the default). + +```bash +azd env set FOUNDRY_EXISTING_DNS_ZONES '{"privatelink.services.ai.azure.com":"hub-dns-rg","privatelink.openai.azure.com":"hub-dns-rg","privatelink.cognitiveservices.azure.com":"hub-dns-rg"}' + +# If the zones live in a different subscription: +azd env set FOUNDRY_DNS_ZONES_SUBSCRIPTION_ID + +azd provision +``` + +When you reuse existing zones, the template assumes the hub team has +already linked them to the spoke VNet (the typical hub-spoke pattern). +The template only creates VNet links for zones it creates itself. + +## Verifying + +After `azd provision`, check the env vars set by the template: + +```bash +# Linux / macOS / WSL +azd env get-values | grep -E '^(FOUNDRY_NETWORK_MODE|AZURE_VNET|AZURE_AGENT_SUBNET|AZURE_PE_SUBNET)=' +``` + +```powershell +# Windows PowerShell +azd env get-values | Select-String -Pattern '^(FOUNDRY_NETWORK_MODE|AZURE_VNET|AZURE_AGENT_SUBNET|AZURE_PE_SUBNET)=' +``` + +You should see: + +* `FOUNDRY_NETWORK_MODE` -- the mode you picked. +* `AZURE_VNET_ID` / `AZURE_VNET_NAME` -- the VNet (empty for `none` and + `managed`). +* `AZURE_AGENT_SUBNET_ID` / `AZURE_PE_SUBNET_ID` -- the subnet ARM IDs + (empty for `none` and `managed`). + +Quick DNS sanity check from a peered or in-VNet host: + +```bash +nslookup .services.ai.azure.com +# Should resolve to a 10.x or 192.168.x address (the private endpoint), +# not a public IP. +``` + +## Troubleshooting + +**Symptom: `azd ai agent invoke` returns 403 with "Client IP X.Y.Z.W is +not allowed".** + +Your dev machine's public IP is not in +`FOUNDRY_CLIENT_IP_ALLOW_LIST`. See the allow-list recipe above to +refresh it. + +**Symptom: `azd deploy` hangs or fails with TLS / connection timeout to +the account.** + +The data plane is unreachable. Either your IP isn't allowlisted, OR +`FOUNDRY_DISABLE_PUBLIC_NETWORK_ACCESS` is `true` and you're outside the +VNet. Re-enable public access (`azd env set +FOUNDRY_DISABLE_PUBLIC_NETWORK_ACCESS false && azd provision`) or move +your dev session into the VNet. + +**Symptom: `nslookup` on the privatelink hostname returns a public IP +from inside the VNet.** + +The private DNS zone is not linked to your VNet. If you used the default +flow (template-created zones), the link is automatic. If you're reusing +existing zones via `FOUNDRY_EXISTING_DNS_ZONES`, the hub team must have +linked the zones to your spoke VNet. + +**Symptom: `azd provision` fails with "subnet must be delegated to +Microsoft.App/environments".** + +You pointed `FOUNDRY_VNET_RESOURCE_ID` at an existing VNet whose agent +subnet has no delegation. Add the delegation out of band, then re-run: + +```bash +az network vnet subnet update \ + --vnet-name --name --resource-group \ + --delegations Microsoft.App/environments +``` + +**Symptom: Capability host creation fails on a BYO VNet deploy.** + +The capability host is created with `enablePublicHostingEnvironment = +false` for `byo-vnet` and `managed` modes. If the deploy fails here, +double-check that the agent subnet is delegated correctly and that the +Foundry account's `networkInjections.agent.subnetArmId` matches the +agent subnet's ARM ID (you can verify via the portal: Foundry account +-> Networking -> Network Injection). diff --git a/infra/core/ai/ai-project.bicep b/infra/core/ai/ai-project.bicep index e4ed003..2f9f033 100644 --- a/infra/core/ai/ai-project.bicep +++ b/infra/core/ai/ai-project.bicep @@ -59,6 +59,37 @@ param existingApplicationInsightsResourceId string = '' @description('Optional. Name of an existing Application Insights connection on the Foundry project. If provided, no new App Insights or connection will be created.') param existingAppInsightsConnectionName string = '' +// --------------------------------------------------------------------- +// Existing account parameters (TBD resolution: support BYO Foundry account) +// --------------------------------------------------------------------- + +@description('When true, reference an existing AI Foundry account instead of creating one. A new project (and its model deployments, capability host, and connections) is still created on the existing account. Implied true when existingAiAccountResourceId is non-empty. The account MUST live in the SAME resource group as this deployment.') +param useExistingAiAccount bool = false + +@description('Optional. Full ARM resource ID of an existing AI Foundry (Microsoft.CognitiveServices/accounts, kind=AIServices) account to reuse. The account name is parsed from this ID. The account MUST live in the SAME resource group as this deployment; subscription/RG segments of the ID are accepted for naming consistency but not honored for scope.') +param existingAiAccountResourceId string = '' + +// --------------------------------------------------------------------- +// Network parameters +// --------------------------------------------------------------------- + +@description('Network mode for the AI Foundry account: none (public, default) | managed (Microsoft-managed network) | byo-vnet (customer-delegated agent subnet + private endpoint).') +@allowed([ + 'none' + 'managed' + 'byo-vnet' +]) +param networkMode string = 'none' + +@description('Resource ID of the agent subnet (delegated to Microsoft.App/environments). Required when networkMode is byo-vnet.') +param agentSubnetId string = '' + +@description('Public IPv4 addresses or CIDRs allowed to reach the account data plane while public access is enabled. Used only when networkMode is byo-vnet.') +param clientIpAllowList array = [] + +@description('When true, set publicNetworkAccess to Disabled (fully lock down the account; requires running azd from inside the VNet). Only relevant when networkMode is byo-vnet.') +param disablePublicNetworkAccess bool = false + // Load abbreviations var abbrs = loadJsonContent('../../abbreviations.json') @@ -82,6 +113,71 @@ var searchConnectionName = hasSearchConnection ? filter(additionalDependentResou var bingConnectionName = hasBingConnection ? filter(additionalDependentResources, conn => conn.resource == 'bing_grounding')[0].connectionName : '' var bingCustomConnectionName = hasBingCustomConnection ? filter(additionalDependentResources, conn => conn.resource == 'bing_custom_grounding')[0].connectionName : '' +// --------------------------------------------------------------------- +// Resolved account name + network derivations +// --------------------------------------------------------------------- + +var hasExistingAccountResourceId = !empty(existingAiAccountResourceId) +var resolvedUseExistingAiAccount = useExistingAiAccount || hasExistingAccountResourceId +var accountNameFromId = hasExistingAccountResourceId ? last(split(existingAiAccountResourceId, '/')) : '' +// Single resolved name used by both the new and existing branches so the +// nested children resolve identically. Precedence: ARM-ID-derived name > +// existingAiAccountName param > auto-generated token. +var resolvedAccountName = !empty(accountNameFromId) + ? accountNameFromId + : (!empty(existingAiAccountName) ? existingAiAccountName : 'ai-account-${resourceToken}') + +var isByoVnet = networkMode == 'byo-vnet' +var isManaged = networkMode == 'managed' + +var ipRules = [for ip in clientIpAllowList: { value: ip }] + +// publicNetworkAccess: +// none / managed -> Enabled (managed network handles isolation Foundry-side) +// byo-vnet -> Enabled with Deny ACL + IP allow-list, unless +// disablePublicNetworkAccess=true (fully private; caller +// must reach the account over the private endpoint). +var effectivePublicNetworkAccess = isByoVnet + ? (disablePublicNetworkAccess ? 'Disabled' : 'Enabled') + : 'Enabled' + +// networkAcls: only meaningful for byo-vnet; managed and none use the default Allow. +var effectiveNetworkAcls = isByoVnet + ? { + // Deny by default. AzureServices bypass keeps the Foundry control plane + // reachable. User-supplied IPs in clientIpAllowList let `azd deploy` and + // interactive tooling work from the developer's machine while public + // access stays Enabled. + defaultAction: 'Deny' + virtualNetworkRules: [] + ipRules: ipRules + bypass: 'AzureServices' + } + : { + defaultAction: 'Allow' + virtualNetworkRules: [] + ipRules: [] + } + +// networkInjections: omitted for none; managed uses useMicrosoftManagedNetwork=true; byo-vnet uses the customer subnet. +var effectiveNetworkInjections = isManaged + ? [ + { + scenario: 'agent' + subnetArmId: '' + useMicrosoftManagedNetwork: true + } + ] + : (isByoVnet + ? [ + { + scenario: 'agent' + subnetArmId: agentSubnetId + useMicrosoftManagedNetwork: false + } + ] + : null) + // Enable monitoring via Log Analytics and Application Insights module logAnalytics '../monitor/loganalytics.bicep' = if (shouldCreateAppInsights) { name: 'logAnalytics' @@ -103,10 +199,11 @@ module applicationInsights '../monitor/applicationinsights.bicep' = if (shouldCr } } -// Always create a new AI Account for now (simplified approach) -// TODO: Add support for existing accounts in a future version -resource aiAccount 'Microsoft.CognitiveServices/accounts@2025-06-01' = { - name: !empty(existingAiAccountName) ? existingAiAccountName : 'ai-account-${resourceToken}' +// New AI Account -- created only when not reusing an existing one. All network +// properties (networkAcls, publicNetworkAccess, networkInjections) live here; +// the existing branch never modifies them on the existing account. +resource newAiAccount 'Microsoft.CognitiveServices/accounts@2025-06-01' = if (!resolvedUseExistingAiAccount) { + name: resolvedAccountName location: location tags: tags sku: { @@ -118,16 +215,22 @@ resource aiAccount 'Microsoft.CognitiveServices/accounts@2025-06-01' = { } properties: { allowProjectManagement: true - customSubDomainName: !empty(existingAiAccountName) ? existingAiAccountName : 'ai-account-${resourceToken}' - networkAcls: { - defaultAction: 'Allow' - virtualNetworkRules: [] - ipRules: [] - } - publicNetworkAccess: 'Enabled' + customSubDomainName: resolvedAccountName + networkAcls: effectiveNetworkAcls + publicNetworkAccess: effectivePublicNetworkAccess + networkInjections: effectiveNetworkInjections disableLocalAuth: true } - +} + +// Account reference. Used as parent for the project, model deployments, and +// capability host so the downstream wiring is identical regardless of whether +// the account was just created (newAiAccount) or pre-existed. Children carry +// dependsOn: [newAiAccount] so they wait for creation in the new branch and +// no-op the dependency in the existing branch. +resource aiAccount 'Microsoft.CognitiveServices/accounts@2025-06-01' existing = { + name: resolvedAccountName + @batchSize(1) resource seqDeployments 'deployments' = [ for dep in (deployments??[]): { @@ -136,6 +239,9 @@ resource aiAccount 'Microsoft.CognitiveServices/accounts@2025-06-01' = { model: dep.model } sku: dep.sku + dependsOn: [ + newAiAccount + ] } ] @@ -151,6 +257,7 @@ resource aiAccount 'Microsoft.CognitiveServices/accounts@2025-06-01' = { } dependsOn: [ seqDeployments + newAiAccount ] } @@ -158,10 +265,15 @@ resource aiAccount 'Microsoft.CognitiveServices/accounts@2025-06-01' = { name: 'agents' properties: { capabilityHostKind: 'Agents' - // IMPORTANT: this is required to enable hosted agents deployment - // if no BYO Net is provided - enablePublicHostingEnvironment: true + // For BYO VNet or managed network, the account is reached over the + // private endpoint or Microsoft-managed network -- enablePublicHostingEnvironment + // is false so the hosting environment honors the network isolation. + // Public mode keeps it true so the agent can be invoked from anywhere. + enablePublicHostingEnvironment: !isByoVnet && !isManaged } + dependsOn: [ + newAiAccount + ] } } diff --git a/infra/core/networking/private-endpoint-and-dns.bicep b/infra/core/networking/private-endpoint-and-dns.bicep new file mode 100644 index 0000000..ac4b9f7 --- /dev/null +++ b/infra/core/networking/private-endpoint-and-dns.bicep @@ -0,0 +1,178 @@ +targetScope = 'resourceGroup' + +/* + Private endpoint + 3 private DNS zones for an AI Foundry (Cognitive Services) account. + + - PE (groupIds: ['account']) on the PE subnet of the supplied VNet + - 3 private DNS zones: + * privatelink.services.ai.azure.com + * privatelink.openai.azure.com + * privatelink.cognitiveservices.azure.com + - VNet links and a DNS zone group binding the zones to the PE. + + Supports reusing existing zones owned by a hub / platform team via the + existingDnsZones map (zone name -> resource group). Empty string means + "create a new zone in the current RG". +*/ + +@description('Name of the AI Foundry (Cognitive Services) account in the current resource group scope. The PE is created in this RG; the account itself can live elsewhere if you pass foundryAccountId.') +param foundryAccountName string + +@description('Optional full ARM resource ID of the AI Foundry account. Use when the account lives in a different RG/subscription from the PE. When empty, the account is looked up by name in the current RG.') +param foundryAccountId string = '' + +@description('Location for the private endpoint. Must match the region of the subnet/VNet the PE is attached to. Defaults to the current resource group location.') +param location string = resourceGroup().location + +@description('Name of the VNet containing the PE subnet.') +param vnetName string + +@description('Subscription ID where the VNet lives (defaults to the current subscription).') +param vnetSubscriptionId string = subscription().subscriptionId + +@description('Resource group of the VNet (defaults to the current RG).') +param vnetResourceGroupName string = resourceGroup().name + +@description('Name of the PE subnet within the VNet.') +param peSubnetName string + +@description('Short unique suffix to disambiguate generated resource names.') +param suffix string + +@description('Map of zone FQDN -> resource group of an existing zone to reuse. Empty string means create a new zone in the current RG. When an existing zone is reused, the caller is responsible for ensuring it is linked to the spoke VNet (typical hub-spoke pattern); this module only creates VNet links for zones it creates.') +param existingDnsZones object = { + 'privatelink.services.ai.azure.com': '' + 'privatelink.openai.azure.com': '' + 'privatelink.cognitiveservices.azure.com': '' +} + +@description('Subscription ID where the existing private DNS zones live. Accepts either a bare GUID or a /subscriptions/ path; normalized internally.') +param dnsZonesSubscriptionId string = subscription().subscriptionId + +// Normalize dnsZonesSubscriptionId +var dnsZonesSubIsPath = startsWith(toLower(dnsZonesSubscriptionId), '/subscriptions/') +var resolvedDnsZonesSubscriptionId = dnsZonesSubIsPath ? split(dnsZonesSubscriptionId, '/')[2] : dnsZonesSubscriptionId + +var aiServicesDnsZoneName = 'privatelink.services.ai.azure.com' +var openAiDnsZoneName = 'privatelink.openai.azure.com' +var cognitiveDnsZoneName = 'privatelink.cognitiveservices.azure.com' + +var aiServicesDnsZoneRg = existingDnsZones[aiServicesDnsZoneName] +var openAiDnsZoneRg = existingDnsZones[openAiDnsZoneName] +var cognitiveDnsZoneRg = existingDnsZones[cognitiveDnsZoneName] + +resource vnet 'Microsoft.Network/virtualNetworks@2024-05-01' existing = { + name: vnetName + scope: resourceGroup(vnetSubscriptionId, vnetResourceGroupName) +} + +resource peSubnet 'Microsoft.Network/virtualNetworks/subnets@2024-05-01' existing = { + parent: vnet + name: peSubnetName +} + +resource foundryAccount 'Microsoft.CognitiveServices/accounts@2026-03-01' existing = { + name: foundryAccountName +} + +resource foundryAccountPrivateEndpoint 'Microsoft.Network/privateEndpoints@2024-05-01' = { + name: '${foundryAccountName}-pe-${suffix}' + location: location + properties: { + subnet: { id: peSubnet.id } + privateLinkServiceConnections: [ + { + name: '${foundryAccountName}-pls-${suffix}' + properties: { + privateLinkServiceId: empty(foundryAccountId) ? foundryAccount.id : foundryAccountId + groupIds: ['account'] + } + } + ] + } +} + +// services.ai.azure.com +resource newAiServicesZone 'Microsoft.Network/privateDnsZones@2020-06-01' = if (empty(aiServicesDnsZoneRg)) { + name: aiServicesDnsZoneName + location: 'global' +} +resource existingAiServicesZone 'Microsoft.Network/privateDnsZones@2020-06-01' existing = if (!empty(aiServicesDnsZoneRg)) { + name: aiServicesDnsZoneName + scope: resourceGroup(resolvedDnsZonesSubscriptionId, aiServicesDnsZoneRg) +} +#disable-next-line BCP318 +var aiServicesZoneId = empty(aiServicesDnsZoneRg) ? newAiServicesZone.id : existingAiServicesZone.id + +resource newAiServicesLink 'Microsoft.Network/privateDnsZones/virtualNetworkLinks@2024-06-01' = if (empty(aiServicesDnsZoneRg)) { + parent: newAiServicesZone + location: 'global' + name: 'aiservices-${suffix}-link' + properties: { + virtualNetwork: { id: vnet.id } + registrationEnabled: false + } +} + +// openai.azure.com +resource newOpenAiZone 'Microsoft.Network/privateDnsZones@2020-06-01' = if (empty(openAiDnsZoneRg)) { + name: openAiDnsZoneName + location: 'global' +} +resource existingOpenAiZone 'Microsoft.Network/privateDnsZones@2020-06-01' existing = if (!empty(openAiDnsZoneRg)) { + name: openAiDnsZoneName + scope: resourceGroup(resolvedDnsZonesSubscriptionId, openAiDnsZoneRg) +} +#disable-next-line BCP318 +var openAiZoneId = empty(openAiDnsZoneRg) ? newOpenAiZone.id : existingOpenAiZone.id + +resource newOpenAiLink 'Microsoft.Network/privateDnsZones/virtualNetworkLinks@2024-06-01' = if (empty(openAiDnsZoneRg)) { + parent: newOpenAiZone + location: 'global' + name: 'openai-${suffix}-link' + properties: { + virtualNetwork: { id: vnet.id } + registrationEnabled: false + } +} + +// cognitiveservices.azure.com +resource newCognitiveZone 'Microsoft.Network/privateDnsZones@2020-06-01' = if (empty(cognitiveDnsZoneRg)) { + name: cognitiveDnsZoneName + location: 'global' +} +resource existingCognitiveZone 'Microsoft.Network/privateDnsZones@2020-06-01' existing = if (!empty(cognitiveDnsZoneRg)) { + name: cognitiveDnsZoneName + scope: resourceGroup(resolvedDnsZonesSubscriptionId, cognitiveDnsZoneRg) +} +#disable-next-line BCP318 +var cognitiveZoneId = empty(cognitiveDnsZoneRg) ? newCognitiveZone.id : existingCognitiveZone.id + +resource newCognitiveLink 'Microsoft.Network/privateDnsZones/virtualNetworkLinks@2024-06-01' = if (empty(cognitiveDnsZoneRg)) { + parent: newCognitiveZone + location: 'global' + name: 'cogserv-${suffix}-link' + properties: { + virtualNetwork: { id: vnet.id } + registrationEnabled: false + } +} + +resource foundryAccountDnsZoneGroup 'Microsoft.Network/privateEndpoints/privateDnsZoneGroups@2024-05-01' = { + parent: foundryAccountPrivateEndpoint + name: '${foundryAccountName}-dns-group' + properties: { + privateDnsZoneConfigs: [ + { name: 'aiservices-config', properties: { privateDnsZoneId: aiServicesZoneId } } + { name: 'openai-config', properties: { privateDnsZoneId: openAiZoneId } } + { name: 'cogserv-config', properties: { privateDnsZoneId: cognitiveZoneId } } + ] + } + dependsOn: [ + newAiServicesLink + newOpenAiLink + newCognitiveLink + ] +} + +output privateEndpointId string = foundryAccountPrivateEndpoint.id diff --git a/infra/core/networking/vnet.bicep b/infra/core/networking/vnet.bicep new file mode 100644 index 0000000..4328732 --- /dev/null +++ b/infra/core/networking/vnet.bicep @@ -0,0 +1,126 @@ +targetScope = 'resourceGroup' + +/* + VNet for Foundry BYO VNet (network injection) and BYO VNet Standard modes. + + Creates a new VNet + 2 subnets (agent + PE) when existingVnetResourceId is empty. + Otherwise, references an existing VNet cross-RG / cross-subscription and exposes + subnet IDs without mutating the existing subnets. This is the safer choice for + hub-spoke topologies with platform-managed NSGs/route tables/policies. + + Preconditions when using an existing VNet: + - The agent subnet MUST already be delegated to Microsoft.App/environments. + - The PE subnet must support private endpoints (no policies preventing PEs). + - The Foundry account region MUST match the VNet region (when main.bicep + provisions both, set `aiDeploymentsLocation` to keep them aligned). +*/ + +@description('Location for the new VNet (ignored when an existing VNet is referenced).') +param location string + +@description('Tags for the new VNet (ignored when an existing VNet is referenced).') +param tags object = {} + +@description('Name of the VNet to create. Ignored when existingVnetResourceId is set (name is derived from the resource ID).') +param vnetName string + +@description('Optional. Full ARM resource ID of an existing VNet to reuse. When set, the template will NOT create a new VNet and will reference the existing subnets without modification.') +param existingVnetResourceId string = '' + +@description('Address space for the new VNet. Empty defaults to 192.168.0.0/16. Ignored for existing VNets.') +param vnetAddressPrefix string = '' + +@description('Name of the agent subnet (delegated to Microsoft.App/environments). Must exist on the VNet when existingVnetResourceId is set.') +param agentSubnetName string = 'agent-subnet' + +@description('Address prefix for the new agent subnet. Empty derives 192.168.0.0/24 from the default VNet prefix. Ignored for existing VNets.') +param agentSubnetPrefix string = '' + +@description('Name of the private endpoint subnet. Must exist on the VNet when existingVnetResourceId is set.') +param peSubnetName string = 'pe-subnet' + +@description('Address prefix for the new PE subnet. Empty derives 192.168.1.0/24 from the default VNet prefix. Ignored for existing VNets.') +param peSubnetPrefix string = '' + +// --------------------------------------------------------------------- +// Derived values +// --------------------------------------------------------------------- + +var hasExistingVnet = !empty(existingVnetResourceId) + +var existingVnetParts = split(existingVnetResourceId, '/') +var existingVnetSubscriptionId = hasExistingVnet ? existingVnetParts[2] : subscription().subscriptionId +var existingVnetResourceGroupName = hasExistingVnet ? existingVnetParts[4] : resourceGroup().name +var resolvedVnetName = hasExistingVnet ? last(existingVnetParts) : vnetName + +var defaultVnetAddressPrefix = '192.168.0.0/16' +var resolvedVnetAddressPrefix = empty(vnetAddressPrefix) ? defaultVnetAddressPrefix : vnetAddressPrefix +var resolvedAgentSubnetPrefix = empty(agentSubnetPrefix) ? cidrSubnet(resolvedVnetAddressPrefix, 24, 0) : agentSubnetPrefix +var resolvedPeSubnetPrefix = empty(peSubnetPrefix) ? cidrSubnet(resolvedVnetAddressPrefix, 24, 1) : peSubnetPrefix + +// --------------------------------------------------------------------- +// New VNet +// --------------------------------------------------------------------- + +resource newVnet 'Microsoft.Network/virtualNetworks@2024-05-01' = if (!hasExistingVnet) { + name: vnetName + location: location + tags: tags + properties: { + addressSpace: { + addressPrefixes: [resolvedVnetAddressPrefix] + } + subnets: [ + { + name: agentSubnetName + properties: { + addressPrefix: resolvedAgentSubnetPrefix + delegations: [ + { + name: 'Microsoft.app/environments' + properties: { + serviceName: 'Microsoft.App/environments' + } + } + ] + } + } + { + name: peSubnetName + properties: { + addressPrefix: resolvedPeSubnetPrefix + privateEndpointNetworkPolicies: 'Disabled' + } + } + ] + } +} + +// --------------------------------------------------------------------- +// Existing VNet (referenced for output; not modified) +// --------------------------------------------------------------------- + +resource existingVnet 'Microsoft.Network/virtualNetworks@2024-05-01' existing = if (hasExistingVnet) { + name: resolvedVnetName + scope: resourceGroup(existingVnetSubscriptionId, existingVnetResourceGroupName) +} + +// --------------------------------------------------------------------- +// Outputs +// --------------------------------------------------------------------- + +output vnetName string = resolvedVnetName +#disable-next-line BCP318 +output vnetId string = hasExistingVnet ? existingVnet.id : newVnet.id +output vnetSubscriptionId string = existingVnetSubscriptionId +output vnetResourceGroupName string = existingVnetResourceGroupName +output agentSubnetName string = agentSubnetName +output peSubnetName string = peSubnetName +#disable-next-line BCP318 +output agentSubnetId string = hasExistingVnet + ? resourceId(existingVnetSubscriptionId, existingVnetResourceGroupName, 'Microsoft.Network/virtualNetworks/subnets', resolvedVnetName, agentSubnetName) + : '${newVnet.id}/subnets/${agentSubnetName}' +#disable-next-line BCP318 +output peSubnetId string = hasExistingVnet + ? resourceId(existingVnetSubscriptionId, existingVnetResourceGroupName, 'Microsoft.Network/virtualNetworks/subnets', resolvedVnetName, peSubnetName) + : '${newVnet.id}/subnets/${peSubnetName}' diff --git a/infra/main.bicep b/infra/main.bicep index 02cc18c..d59b4db 100644 --- a/infra/main.bicep +++ b/infra/main.bicep @@ -110,6 +110,61 @@ param existingApplicationInsightsResourceId string = '' @description('Optional. Name of an existing Application Insights connection on the Foundry project. If provided, no new App Insights or connection will be created.') param existingAppInsightsConnectionName string = '' +// ---------- Existing account ---------- + +@description('When true, reference an existing AI Foundry account (created out of band) instead of creating a new one. A new project (and its deployments, capability host, and connections) is still created on that account. Implied true when existingAiAccountResourceId is non-empty. The account MUST live in the SAME resource group as this deployment.') +param useExistingAiAccount bool = false + +@description('Optional. Full ARM resource ID of an existing AI Foundry (Microsoft.CognitiveServices/accounts, kind=AIServices) account to reuse. The account name is parsed from this ID. The account MUST live in the SAME resource group as this deployment.') +param existingAiAccountResourceId string = '' + +// ---------- Network ---------- + +@description('Network mode for the AI Foundry account: none (public, default) | managed (Microsoft-managed network) | byo-vnet (customer-delegated agent subnet + private endpoint on the account).') +@allowed([ + 'none' + 'managed' + 'byo-vnet' +]) +param networkMode string = 'none' + +@description('Optional. Full ARM resource ID of an existing VNet to reuse. The agent subnet must already be delegated to Microsoft.App/environments. Empty creates a new VNet in this resource group. Cross-RG / cross-subscription safe.') +param existingVnetResourceId string = '' + +@description('Name of the new VNet (created when networkMode is byo-vnet and existingVnetResourceId is empty).') +param vnetName string = 'vnet-${environmentName}' + +@description('VNet address prefix. Empty defaults to 192.168.0.0/16. Ignored for existing VNets.') +param vnetAddressPrefix string = '' + +@description('Agent subnet name (delegated to Microsoft.App/environments).') +param agentSubnetName string = 'agent-subnet' + +@description('Agent subnet prefix. Empty derives 192.168.0.0/24 from the default VNet prefix. Ignored for existing VNets.') +param agentSubnetPrefix string = '' + +@description('Private endpoint subnet name.') +param peSubnetName string = 'pe-subnet' + +@description('PE subnet prefix. Empty derives 192.168.1.0/24 from the default VNet prefix. Ignored for existing VNets.') +param peSubnetPrefix string = '' + +@description('JSON array of IPv4 addresses or CIDR ranges allowed to reach the AI Foundry account data plane while public access is enabled (used only when networkMode is byo-vnet).') +param clientIpAllowList array = [] + +@description('When true, set publicNetworkAccess=Disabled on the AI Foundry account. Requires running azd from inside the VNet (or via a private VPN/peer). Only relevant when networkMode is byo-vnet.') +param disablePublicNetworkAccess bool = false + +@description('Map of existing private DNS zone FQDN -> resource group name. Empty value means create a new zone in the current RG. Only consulted when networkMode is byo-vnet.') +param existingDnsZones object = { + 'privatelink.services.ai.azure.com': '' + 'privatelink.openai.azure.com': '' + 'privatelink.cognitiveservices.azure.com': '' +} + +@description('Subscription ID where the existing private DNS zones live. Empty defaults to the current subscription. Accepts a bare GUID or /subscriptions/ path.') +param dnsZonesSubscriptionId string = '' + // Tags that should be applied to all resources. // // Note that 'azd-service-name' tags should be applied separately to service host resources. @@ -126,6 +181,32 @@ resource rg 'Microsoft.Resources/resourceGroups@2021-04-01' = { tags: tags } +// ---------- Network derivations ---------- + +var isByoVnet = networkMode == 'byo-vnet' + +// VNet (only created/referenced when networkMode is byo-vnet). When isByoVnet +// is false, this module is skipped entirely and downstream consumers fall +// back to empty strings. +module vnet 'core/networking/vnet.bicep' = if (isByoVnet) { + scope: rg + name: 'vnet' + params: { + location: aiDeploymentsLocation + tags: tags + vnetName: vnetName + existingVnetResourceId: existingVnetResourceId + vnetAddressPrefix: vnetAddressPrefix + agentSubnetName: agentSubnetName + agentSubnetPrefix: agentSubnetPrefix + peSubnetName: peSubnetName + peSubnetPrefix: peSubnetPrefix + } +} + +#disable-next-line BCP318 +var agentSubnetIdValue = isByoVnet ? vnet.outputs.agentSubnetId : '' + // Build dependent resources array conditionally // Check if ACR already exists in the user-provided array to avoid duplicates // Also skip if user provided an existing container registry endpoint or connection name @@ -138,7 +219,7 @@ var dependentResources = shouldCreateAcr ? union(aiProjectDependentResources, [ } ]) : aiProjectDependentResources -// AI Project module — only when creating new resources +// AI Project module -- only when creating new resources module aiProject 'core/ai/ai-project.bicep' = if (!useExistingAiProject) { scope: rg name: 'ai-project' @@ -149,6 +230,8 @@ module aiProject 'core/ai/ai-project.bicep' = if (!useExistingAiProject) { principalId: principalId principalType: principalType existingAiAccountName: aiFoundryResourceName + useExistingAiAccount: useExistingAiAccount + existingAiAccountResourceId: existingAiAccountResourceId deployments: aiProjectDeployments connections: aiProjectConnections connectionCredentials: aiProjectConnectionCreds @@ -162,10 +245,40 @@ module aiProject 'core/ai/ai-project.bicep' = if (!useExistingAiProject) { existingApplicationInsightsConnectionString: existingApplicationInsightsConnectionString existingApplicationInsightsResourceId: existingApplicationInsightsResourceId existingAppInsightsConnectionName: existingAppInsightsConnectionName + networkMode: networkMode + agentSubnetId: agentSubnetIdValue + clientIpAllowList: clientIpAllowList + disablePublicNetworkAccess: disablePublicNetworkAccess } } -// Existing project module — read-only reference when reusing an existing Foundry project +// Private endpoint + DNS for the AI Foundry account (only when networkMode is +// byo-vnet). Created after the account exists so the PE can target it. +// Skipped for the read-only existing-project path (existing-ai-project.bicep) +// since that flow expects PE/DNS to already be in place. +module accountPeDns 'core/networking/private-endpoint-and-dns.bicep' = if (isByoVnet && !useExistingAiProject) { + scope: rg + name: 'account-pe-dns' + params: { + location: aiDeploymentsLocation + #disable-next-line BCP318 + foundryAccountName: aiProject.outputs.aiServicesAccountName + #disable-next-line BCP318 + foundryAccountId: aiProject.outputs.accountId + #disable-next-line BCP318 + vnetName: isByoVnet ? vnet.outputs.vnetName : '' + #disable-next-line BCP318 + vnetSubscriptionId: isByoVnet ? vnet.outputs.vnetSubscriptionId : subscription().subscriptionId + #disable-next-line BCP318 + vnetResourceGroupName: isByoVnet ? vnet.outputs.vnetResourceGroupName : rg.name + peSubnetName: peSubnetName + suffix: uniqueString(subscription().id, resourceGroupName, location) + existingDnsZones: existingDnsZones + dnsZonesSubscriptionId: empty(dnsZonesSubscriptionId) ? subscription().subscriptionId : dnsZonesSubscriptionId + } +} + +// Existing project module -- read-only reference when reusing an existing Foundry project module existingAiProject 'core/ai/existing-ai-project.bicep' = if (useExistingAiProject) { scope: rg name: 'existing-ai-project' @@ -182,7 +295,7 @@ module existingAiProject 'core/ai/existing-ai-project.bicep' = if (useExistingAi } } -// ACR for existing project — create when hosted agents need a registry but the existing project has none +// ACR for existing project -- create when hosted agents need a registry but the existing project has none var shouldCreateAcrForExistingProject = useExistingAiProject && shouldCreateAcr var acrConnectionName = 'acr-${uniqueString(subscription().id, resourceGroupName, location)}' @@ -242,3 +355,14 @@ output AZURE_STORAGE_ACCOUNT_NAME string = useExistingAiProject ? existingAiProj // Connections output AI_PROJECT_CONNECTION_IDS_JSON string = useExistingAiProject ? string(existingAiProject.outputs.connectionIds) : string(aiProject.outputs.connectionIds) + +// Network +output FOUNDRY_NETWORK_MODE string = networkMode +#disable-next-line BCP318 +output AZURE_VNET_ID string = isByoVnet ? vnet.outputs.vnetId : '' +#disable-next-line BCP318 +output AZURE_VNET_NAME string = isByoVnet ? vnet.outputs.vnetName : '' +#disable-next-line BCP318 +output AZURE_AGENT_SUBNET_ID string = isByoVnet ? vnet.outputs.agentSubnetId : '' +#disable-next-line BCP318 +output AZURE_PE_SUBNET_ID string = isByoVnet ? vnet.outputs.peSubnetId : '' diff --git a/infra/main.parameters.json b/infra/main.parameters.json index af48d8c..17e99dd 100644 --- a/infra/main.parameters.json +++ b/infra/main.parameters.json @@ -70,6 +70,48 @@ }, "existingAppInsightsConnectionName": { "value": "${APPLICATIONINSIGHTS_CONNECTION_NAME=}" + }, + "useExistingAiAccount": { + "value": "${USE_EXISTING_AI_ACCOUNT=false}" + }, + "existingAiAccountResourceId": { + "value": "${AZURE_EXISTING_AI_ACCOUNT_RESOURCE_ID=}" + }, + "networkMode": { + "value": "${FOUNDRY_NETWORK_MODE=none}" + }, + "existingVnetResourceId": { + "value": "${FOUNDRY_VNET_RESOURCE_ID=}" + }, + "vnetName": { + "value": "${FOUNDRY_VNET_NAME=vnet-${AZURE_ENV_NAME}}" + }, + "vnetAddressPrefix": { + "value": "${FOUNDRY_VNET_ADDRESS_PREFIX=}" + }, + "agentSubnetName": { + "value": "${FOUNDRY_AGENT_SUBNET_NAME=agent-subnet}" + }, + "agentSubnetPrefix": { + "value": "${FOUNDRY_AGENT_SUBNET_PREFIX=}" + }, + "peSubnetName": { + "value": "${FOUNDRY_PE_SUBNET_NAME=pe-subnet}" + }, + "peSubnetPrefix": { + "value": "${FOUNDRY_PE_SUBNET_PREFIX=}" + }, + "clientIpAllowList": { + "value": "${FOUNDRY_CLIENT_IP_ALLOW_LIST=[]}" + }, + "disablePublicNetworkAccess": { + "value": "${FOUNDRY_DISABLE_PUBLIC_NETWORK_ACCESS=false}" + }, + "existingDnsZones": { + "value": "${FOUNDRY_EXISTING_DNS_ZONES={}}" + }, + "dnsZonesSubscriptionId": { + "value": "${FOUNDRY_DNS_ZONES_SUBSCRIPTION_ID=}" } - } + } }