From d4243c4530b4963b953b709fcdb6911858950586 Mon Sep 17 00:00:00 2001 From: Marcello Formica Date: Mon, 27 Apr 2026 13:04:26 +0200 Subject: [PATCH 01/10] Add Setup/ Bicep infrastructure files --- Setup/certlc.bicep | 1136 +++++++++++++++++++++++++++++++ Setup/getavailability.bicep | 0 Setup/parameters.dev.bicepparam | 29 + 3 files changed, 1165 insertions(+) create mode 100644 Setup/certlc.bicep create mode 100644 Setup/getavailability.bicep create mode 100644 Setup/parameters.dev.bicepparam diff --git a/Setup/certlc.bicep b/Setup/certlc.bicep new file mode 100644 index 0000000..34ba543 --- /dev/null +++ b/Setup/certlc.bicep @@ -0,0 +1,1136 @@ +/* + +CERTLC - Bicep file for deploying the required resources for the CERTLC solution. + +Validate with: az deployment group validate --resource-group -parameters .\parameters.dev.bicepparam +What-if: az deployment group what-if --resource-group -parameters .\parameters.dev.bicepparam +Deploy with: az deployment group create --resource-group -parameters .\parameters.dev.bicepparam + +*/ + +metadata name = 'CertLC Infrastructure' +metadata description = 'Azure infrastructure deployment for Certificate Lifecycle Management solution with automated certificate enrollment, renewal, and monitoring' + +targetScope = 'resourceGroup' + +@description('The Azure region where resources will be deployed. Defaults to the resource group location.') +param location string = resourceGroup().location + +@description('The resource ID of the subnet for private endpoint connections. Format: /subscriptions/{subscriptionId}/resourceGroups/{rgName}/providers/Microsoft.Network/virtualNetworks/{vnetName}/subnets/{subnetName}') +param peSubnetId string + +@description('The resource ID of the subnet for the function app VNet integration. Must be delegated to Microsoft.App/environments for Flex Consumption plans. Format: /subscriptions/{subscriptionId}/resourceGroups/{rgName}/providers/Microsoft.Network/virtualNetworks/{vnetName}/subnets/{subnetName}') +param fnSubnetId string + +@description('The subscription ID where existing Private DNS Zones are located (for privatelink zones). Format: GUID') +param dnsZonesSubscriptionId string + +@description('The resource group name containing existing Private DNS Zones (e.g., privatelink.blob.core.windows.net, privatelink.vaultcore.azure.net)') +param dnsZonesResourceGroupName string + +@description('The name of the storage account to create. Must be globally unique, 3-24 characters, lowercase letters and numbers only. Used for function app storage and certificate lifecycle queue.') +@minLength(3) +@maxLength(24) +param storageAccountName string + +@description('The name of the function app to create. Must be globally unique, 2-60 characters, alphanumerics and hyphens. Hosts the queue processor and automation triggers.') +@minLength(2) +@maxLength(60) +param functionAppName string + +@description('The name of the Log Analytics workspace for centralized logging and monitoring. Stores diagnostic logs, custom certificate statistics, and application telemetry.') +param logAnalyticsWorkspaceName string + +@description('The name of the Application Insights instance for function app monitoring and performance tracking.') +param applicationInsightsName string + +@description('The name of the Automation Account to create. 6-50 characters, alphanumerics and hyphens. Executes certificate lifecycle runbooks on hybrid workers.') +@minLength(6) +@maxLength(50) +param automationAccountName string + +@description('The name of the hybrid runbook worker group. On-premises workers must be registered to this group to execute certificate operations.') +param hybridWorkerGroupName string + +@description('The name of the runbook to invoke for certificate lifecycle operations. Must match the runbook name deployed to the Automation Account.') +param runbookName string + +@description('The name of the Key Vault to create. Must be globally unique, 3-24 characters, alphanumerics and hyphens. Stores and manages certificates with automated lifecycle tracking.') +@minLength(3) +@maxLength(24) +param keyVaultName string + +@description('The name of the Data Collection Endpoint (DCE) to create. Ingestion endpoint for custom certificate statistics logs sent from automation runbooks.') +param dataCollectionEndpointName string + +@description('The name of the Data Collection Rule (DCR) to create. Defines transformation and routing of certificate statistics to Log Analytics custom table.') +param dataCollectionRuleName string + +@description('The Certificate Authority name for certificate enrollment. Format: CA_SERVER\\\\CA_NAME (e.g., PKI-CA01\\\\ContosoRootCA). Used by runbooks for ADCS operations.') +param automationAccountVarCA string + +@description('The root folder path on hybrid workers where PFX certificates are stored. Format: UNC path or local path (e.g., \\\\\\\\fileserver\\\\certs or C:\\\\\\\\Certificates).') +param automationAccountVarPfxRootFolder string + +@description('The SMTP From email address for certificate expiration notifications (e.g., certlc@contoso.com).') +param automationAccountVarSmtpFrom string + +@description('The SMTP server hostname or IP address for sending email notifications (e.g., smtp.office365.com or smtp.gmail.com).') +param automationAccountVarSmtpServer string + +@description('The SMTP username for authentication to the mail server. Required if the SMTP server requires authentication.') +param automationAccountVarSmtpUser string + +@description('The SMTP password for authentication. Stored encrypted in Automation Account variables.') +@secure() +param automationAccountVarSmtpPassword string + +@description('The start time for the certlcstats schedule. Defaults to 15 minutes from deployment time.') +param scheduleStartTime string = dateTimeAdd(utcNow('u'), 'PT15M') + +/*************/ +/* VARIABLES */ +/*************/ + +// Common tags for all resources +var commonTags = { + solution: 'CertLC' + purpose: 'Certificate Lifecycle Management' +} + +// Azure built-in role definition IDs +var roleDefinitions = { + storageQueueDataReader: '19e7f393-937e-4f77-808e-94535e297925' + storageQueueDataMessageSender: 'c6a89b2d-59bc-44d0-9896-0f6e12d7b80a' + keyVaultCertificatesOfficer: 'a4417e6f-fecd-4de8-b567-7b0420556985' + keyVaultSecretsOfficer: 'b86a8fe4-44ce-4948-aee5-eccb2c155cd7' + reader: 'acdd72a7-3385-48ef-bd42-f606fba81ae7' + monitoringMetricsPublisher: '3913510d-42f4-4e42-8a64-420c390055eb' + storageBlobDataOwner: 'b7e6dc6d-f1e8-4753-8033-0f276bb0955b' + storageQueueDataMessageProcessor: '8a0f0c08-91a1-4084-bc3d-661d67233fed' + storageQueueDataContributor: '974c5e8b-45b9-4653-ba55-5f855dd0fb88' + automationOperator: 'd3881f73-407a-4167-8283-e981cbba0404' +} + +/**********************/ +/* EXISTING RESOURCES */ +/**********************/ + +// References to existing Private DNS Zones in their subscription +resource blobDnsZone 'Microsoft.Network/privateDnsZones@2020-06-01' existing = { + name: 'privatelink.blob.${environment().suffixes.storage}' + scope: resourceGroup(dnsZonesSubscriptionId, dnsZonesResourceGroupName) +} + +resource keyVaultDnsZone 'Microsoft.Network/privateDnsZones@2020-06-01' existing = { + name: 'privatelink.vaultcore.azure.net' + scope: resourceGroup(dnsZonesSubscriptionId, dnsZonesResourceGroupName) +} + +resource queueDnsZone 'Microsoft.Network/privateDnsZones@2020-06-01' existing = { + name: 'privatelink.queue.${environment().suffixes.storage}' + scope: resourceGroup(dnsZonesSubscriptionId, dnsZonesResourceGroupName) +} + +resource webAppDnsZone 'Microsoft.Network/privateDnsZones@2020-06-01' existing = { + name: 'privatelink.azurewebsites.net' + scope: resourceGroup(dnsZonesSubscriptionId, dnsZonesResourceGroupName) +} + +resource automationAccountDnsZone 'Microsoft.Network/privateDnsZones@2020-06-01' existing = { + name: 'privatelink.azure-automation.net' + scope: resourceGroup(dnsZonesSubscriptionId, dnsZonesResourceGroupName) +} + +/*****************/ +/* NEW RESOURCES */ +/*****************/ + +// Storage Account +resource storageAccount 'Microsoft.Storage/storageAccounts@2025-01-01' = { + name: storageAccountName + location: location + sku: { + name: 'Standard_LRS' + } + kind: 'StorageV2' + properties: { + accessTier: 'Hot' + defaultToOAuthAuthentication: true + allowBlobPublicAccess: false + allowSharedKeyAccess: false + minimumTlsVersion: 'TLS1_2' + supportsHttpsTrafficOnly: true + networkAcls: { + bypass: 'AzureServices' + defaultAction: 'Deny' + } + publicNetworkAccess: 'Disabled' + encryption: { + services: { + blob: { + enabled: true + } + queue: { + enabled: true + } + } + } + } + resource blobServices 'blobServices' = { + name: 'default' + properties: {} + } + resource queueServices 'queueServices' = { + name: 'default' + properties: {} + resource queues 'queues' = { + name: 'certlc' + properties: {} + } + } + + tags: commonTags +} + +// Private endpoint for the storage account - blob +resource storageAccountBlobPrivateEndpoint 'Microsoft.Network/privateEndpoints@2024-10-01' = { + name: 'pe-blob-${storageAccountName}' + location: location + properties: { + subnet: { + id: peSubnetId + } + privateLinkServiceConnections: [ + { + name: 'pls-${storageAccountName}' + properties: { + privateLinkServiceId: storageAccount.id + groupIds: [ + 'blob' + ] + } + } + ] + customNetworkInterfaceName: 'nic-pe-${storageAccountName}' + } + tags: commonTags + + resource privateDnsZoneGroup 'privateDnsZoneGroups' = { + name: 'default' + properties: { + privateDnsZoneConfigs: [ + { + name: 'config1' + properties: { + privateDnsZoneId: blobDnsZone.id + } + } + ] + } + } +} + +// Private endpoint for the storage account - queue +resource storageAccountQueuePrivateEndpoint 'Microsoft.Network/privateEndpoints@2024-10-01' = { + name: 'pe-queue-${storageAccountName}' + location: location + properties: { + subnet: { + id: peSubnetId + } + privateLinkServiceConnections: [ + { + name: 'pls-${storageAccountName}' + properties: { + privateLinkServiceId: storageAccount.id + groupIds: [ + 'queue' + ] + } + } + ] + customNetworkInterfaceName: 'nic-pe-queue-${storageAccountName}' + } + tags: commonTags + + resource privateDnsZoneGroup 'privateDnsZoneGroups' = { + name: 'default' + properties: { + privateDnsZoneConfigs: [ + { + name: 'config1' + properties: { + privateDnsZoneId: queueDnsZone.id + } + } + ] + } + } +} + +// Log Analytics Workspace +resource logAnalyticsWorkspace 'Microsoft.OperationalInsights/workspaces@2023-09-01' = { + name: logAnalyticsWorkspaceName + location: location + properties: { + sku: { + name: 'PerGB2018' + } + retentionInDays: 30 + } + tags: commonTags +} + +// Data Collection Endpoint +resource dataCollectionEndpoint 'Microsoft.Insights/dataCollectionEndpoints@2023-03-11' = { + name: dataCollectionEndpointName + location: location + properties: { + networkAcls: { + publicNetworkAccess: 'Enabled' + } + } + tags: commonTags +} + +// Custom Table for Certificate Statistics +resource customTable 'Microsoft.OperationalInsights/workspaces/tables@2022-10-01' = { + name: 'certlc_CL' + parent: logAnalyticsWorkspace + properties: { + retentionInDays: 30 + schema: { + name: 'certlc_CL' + columns: [ + { + name: 'TimeGenerated' + type: 'datetime' + } + { + name: 'Thumbprint' + type: 'string' + } + { + name: 'Name' + type: 'string' + } + { + name: 'Created' + type: 'datetime' + } + { + name: 'Expires' + type: 'datetime' + } + { + name: 'Subject' + type: 'string' + } + { + name: 'Template' + type: 'string' + } + { + name: 'DNSNames' + type: 'string' + } + ] + } + } +} + +// Data Collection Rule for Certificate Statistics +resource dataCollectionRule 'Microsoft.Insights/dataCollectionRules@2023-03-11' = { + name: dataCollectionRuleName + location: location + properties: { + dataCollectionEndpointId: dataCollectionEndpoint.id + streamDeclarations: { + 'Custom-certlc_CL': { + columns: [ + { + name: 'Thumbprint' + type: 'string' + } + { + name: 'Name' + type: 'string' + } + { + name: 'Created' + type: 'datetime' + } + { + name: 'Expires' + type: 'datetime' + } + { + name: 'Subject' + type: 'string' + } + { + name: 'Template' + type: 'string' + } + { + name: 'DNSNames' + type: 'string' + } + ] + } + } + destinations: { + logAnalytics: [ + { + workspaceResourceId: logAnalyticsWorkspace.id + name: 'clv2ws1' + } + ] + } + dataFlows: [ + { + streams: [ + 'Custom-certlc_CL' + ] + destinations: [ + 'clv2ws1' + ] + transformKql: 'source | extend Created = todatetime(Created), Expires = todatetime(Expires) | extend TimeGenerated = now()' + outputStream: 'Custom-certlc_CL' + } + ] + } + dependsOn: [ + customTable // the DCR must be created after the custom table + ] + tags: commonTags +} + +// Application Insights +// IMPORTANT: Deploy AFTER all Log Analytics operations are complete to avoid "Workspace not active" errors +resource applicationInsights 'Microsoft.Insights/components@2020-02-02' = { + name: applicationInsightsName + location: location + kind: 'web' + properties: { + Application_Type: 'web' + WorkspaceResourceId: logAnalyticsWorkspace.id + DisableLocalAuth: true + } + dependsOn: [ + // Force serial deployment: Log Analytics → Custom Table → DCR → Automation Account → Diagnostics → App Insights + // This ensures the workspace backend is fully active before App Insights connects + automationAccountDiagnostics // Wait for diagnostic settings which write to workspace + keyVaultDiagnostics + ] + tags: commonTags +} + +// Flexible Consumption Plan for the function app +resource flexServicePlan 'Microsoft.Web/serverfarms@2024-11-01' = { + name: 'asp-${functionAppName}' + location: location + kind: 'functionapp' + sku: { + tier: 'FlexConsumption' + name: 'FC1' + } + properties: { + reserved: true + } + tags: commonTags +} + +// Function App +resource functionApp 'Microsoft.Web/sites@2024-11-01' = { + name: functionAppName + location: location + kind: 'functionapp,linux' + identity: { + type: 'SystemAssigned' + } + properties: { + serverFarmId: flexServicePlan.id + httpsOnly: true + virtualNetworkSubnetId: fnSubnetId + publicNetworkAccess: 'Disabled' + siteConfig: { + minTlsVersion: '1.2' + } + functionAppConfig: { + deployment: { + storage: { + type: 'blobContainer' + value: '${storageAccount.properties.primaryEndpoints.blob}azure-webjobs-hosts' + authentication: { + type: 'SystemAssignedIdentity' + } + } + } + scaleAndConcurrency: { + maximumInstanceCount: 100 + instanceMemoryMB: 2048 + } + runtime: { + name: 'powerShell' + version: '7.4' + } + } + } + resource appSettings 'config' = { + name: 'appsettings' + properties: { + AutomationAccountName: automationAccount.name + HybridWorkerGroupName: hybridWorkerGroupName + RunbookName: runbookName + ResourceGroupName: resourceGroup().name + AzureWebJobsStorage__credential: 'managedidentity' + AzureWebJobsStorage__blobServiceUri: storageAccount.properties.primaryEndpoints.blob + AzureWebJobsStorage__queueServiceUri: storageAccount.properties.primaryEndpoints.queue + APPLICATIONINSIGHTS_AUTHENTICATION_STRING: 'Authorization=AAD' + APPLICATIONINSIGHTS_CONNECTION_STRING: applicationInsights.properties.ConnectionString + } + } + dependsOn: [ + storageAccountBlobPrivateEndpoint // create the function only after the PEs for the storage account are ready + storageAccountQueuePrivateEndpoint + ] + tags: commonTags +} + +// Private endpoint for the function app +resource functionAppPrivateEndpoint 'Microsoft.Network/privateEndpoints@2024-10-01' = { + name: 'pe-sites-${functionAppName}' + location: location + properties: { + subnet: { + id: peSubnetId + } + privateLinkServiceConnections: [ + { + name: 'pls-${functionAppName}' + properties: { + privateLinkServiceId: functionApp.id + groupIds: [ + 'sites' + ] + } + } + ] + customNetworkInterfaceName: 'nic-pe-${functionAppName}' + } + tags: commonTags + + resource privateDnsZoneGroup 'privateDnsZoneGroups' = { + name: 'default' + properties: { + privateDnsZoneConfigs: [ + { + name: 'config1' + properties: { + privateDnsZoneId: webAppDnsZone.id + } + } + ] + } + } +} + +// Automation Account with its managed identity +resource automationAccount 'Microsoft.Automation/automationAccounts@2024-10-23' = { + name: automationAccountName + location: location + identity: { + type: 'SystemAssigned' + } + properties: { + publicNetworkAccess: false + sku: { + name: 'Basic' + } + } + dependsOn: [ + dataCollectionRule + dataCollectionEndpoint + ] + tags: commonTags + // variables + resource automationAccountVariables 'variables@2024-10-23' = { + name: 'certlc-ca' + properties: { + value: '"${replace(automationAccountVarCA, '\\', '\\\\')}"' + isEncrypted: true + } + } + resource automationAccountVariablesPfxRootFolder 'variables@2024-10-23' = { + name: 'certlc-pfxrootfolder' + properties: { + value: '"${replace(automationAccountVarPfxRootFolder, '\\', '\\\\')}"' + isEncrypted: true + } + } + resource automationAccountVariablesSmtpFrom 'variables@2024-10-23' = { + name: 'certlc-smtpfrom' + properties: { + value: '"${replace(automationAccountVarSmtpFrom, '\\', '\\\\')}"' + isEncrypted: true + } + } + resource automationAccountVariablesSmtpServer 'variables@2024-10-23' = { + name: 'certlc-smtpserver' + properties: { + value: '"${replace(automationAccountVarSmtpServer, '\\', '\\\\')}"' + isEncrypted: true + } + } + resource automationAccountVariablesSmtpUser 'variables@2024-10-23' = { + name: 'certlc-smtpuser' + properties: { + value: '"${replace(automationAccountVarSmtpUser, '\\', '\\\\')}"' + isEncrypted: true + } + } + resource automationAccountVariablesSmtpPassword 'variables@2024-10-23' = { + name: 'certlc-smtppassword' + properties: { + value: '"${replace(automationAccountVarSmtpPassword, '\\', '\\\\')}"' + isEncrypted: true + } + } + resource automationAccountVariablesKeyVault 'variables@2024-10-23' = { + name: 'certlc-stats-keyvault' + properties: { + value: '"${keyVault.name}"' + isEncrypted: true + } + } + resource automationAccountVariablesImmutableId 'variables@2024-10-23' = { + name: 'certlc-stats-immutableid' + properties: { + value: '"${dataCollectionRule.properties.immutableId}"' + isEncrypted: true + } + } + resource automationAccountVariablesStreamName 'variables@2024-10-23' = { + name: 'certlc-stats-streamname' + properties: { + value: '"Custom-certlc_CL"' + isEncrypted: true + } + } + resource automationAccountVariablesIngestionUrl 'variables@2024-10-23' = { + name: 'certlc-stats-ingestionurl' + properties: { + value: '"${dataCollectionEndpoint.properties.logsIngestion.endpoint}"' + isEncrypted: true + } + } + + // Runbook: certlc + resource runbookCertLC 'runbooks@2024-10-23' = { + name: 'certlc' + location: location + properties: { + runbookType: 'PowerShell' + logProgress: false + logVerbose: false + description: 'Certificate lifecycle management runbook for enrollment, renewal, and revocation' + runtimeEnvironment: 'PowerShell-7.2' + } + tags: commonTags + } + + // Runbook: certlcstats + resource runbookCertLCStats 'runbooks@2024-10-23' = { + name: 'certlcstats' + location: location + properties: { + runbookType: 'PowerShell' + logProgress: false + logVerbose: false + description: 'Certificate statistics collection runbook for monitoring and reporting' + runtimeEnvironment: 'PowerShell-7.2' + } + tags: commonTags + } + + // Schedule for certlcstats runbook - runs every hour + // Note: Schedule is created but NOT linked to runbook initially (disabled state) + // To enable: Link the schedule to the runbook in Azure Portal or via Azure CLI + resource scheduleCertLCStats 'schedules@2024-10-23' = { + name: 'schedule-certlcstats-hourly' + properties: { + description: 'Runs certlcstats runbook every hour to collect certificate statistics (manually link to enable)' + startTime: scheduleStartTime + frequency: 'Hour' + interval: 1 + timeZone: 'UTC' + } + } + + // Uncomment to automatically link schedule to runbook (enables automatic execution on hybrid worker group) + // resource jobScheduleCertLCStats 'jobSchedules@2024-10-23' = { + // name: guid(automationAccount.id, 'certlcstats-schedule') + // properties: { + // runbook: { + // name: runbookCertLCStats.name + // } + // schedule: { + // name: scheduleCertLCStats.name + // } + // runOn: hybridWorkerGroupName // Execute on hybrid worker group (not Azure sandbox) + // } + // } +} + +// Hybrid Worker Group +resource hybridWorkerGroup 'Microsoft.Automation/automationAccounts/hybridRunbookWorkerGroups@2023-11-01' = { + name: hybridWorkerGroupName + parent: automationAccount + properties: { + // Hybrid worker group properties - workers will be added separately + } +} + +// Private endpoint for the Automation Account - Webhook +resource automationAccountPrivateEndpoint 'Microsoft.Network/privateEndpoints@2024-10-01' = { + name: 'pe-webhook-${automationAccountName}' + location: location + properties: { + subnet: { + id: peSubnetId + } + privateLinkServiceConnections: [ + { + name: 'pls-${automationAccountName}' + properties: { + privateLinkServiceId: automationAccount.id + groupIds: [ + 'Webhook' + ] + } + } + ] + customNetworkInterfaceName: 'nic-pe-webhook-${automationAccountName}' + } + tags: commonTags + + resource privateDnsZoneGroup 'privateDnsZoneGroups' = { + name: 'default' + properties: { + privateDnsZoneConfigs: [ + { + name: 'config1' + properties: { + privateDnsZoneId: automationAccountDnsZone.id + } + } + ] + } + } +} + +// Private endpoint for the Automation Account - DSCAndHybridWorker +resource automationAccountPrivateEndpointDSCAndHybridWorker 'Microsoft.Network/privateEndpoints@2024-10-01' = { + name: 'pe-dscandhybridworker-${automationAccountName}' + location: location + properties: { + subnet: { + id: peSubnetId + } + privateLinkServiceConnections: [ + { + name: 'pls-dscandhybridworker-${automationAccountName}' + properties: { + privateLinkServiceId: automationAccount.id + groupIds: [ + 'DSCAndHybridWorker' + ] + } + } + ] + customNetworkInterfaceName: 'nic-pe-dscandhybridworker-${automationAccountName}' + } + tags: commonTags + + resource privateDnsZoneGroup 'privateDnsZoneGroups' = { + name: 'default' + properties: { + privateDnsZoneConfigs: [ + { + name: 'config1' + properties: { + privateDnsZoneId: automationAccountDnsZone.id + } + } + ] + } + } +} + +// Diagnostic Settings for Automation Account +resource automationAccountDiagnostics 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { + name: 'diag-${automationAccountName}' + scope: automationAccount + properties: { + workspaceId: logAnalyticsWorkspace.id + logs: [ + { + category: 'JobLogs' + enabled: true + } + { + category: 'JobStreams' + enabled: true + } + ] + metrics: [ + { + category: 'AllMetrics' + enabled: true + } + ] + } +} + +// KeyVault +resource keyVault 'Microsoft.KeyVault/vaults@2025-05-01' = { + name: keyVaultName + location: location + properties: { + sku: { + family: 'A' + name: 'standard' + } + tenantId: subscription().tenantId + enableSoftDelete: true + softDeleteRetentionInDays: 7 + enableRbacAuthorization: true + publicNetworkAccess: 'Disabled' + } + tags: commonTags +} + +// Private endpoint for the KeyVault +resource keyVaultPrivateEndpoint 'Microsoft.Network/privateEndpoints@2024-10-01' = { + name: 'pe-vault-${keyVaultName}' + location: location + properties: { + subnet: { + id: peSubnetId + } + privateLinkServiceConnections: [ + { + name: 'pls-${keyVaultName}' + properties: { + privateLinkServiceId: keyVault.id + groupIds: [ + 'vault' + ] + } + } + ] + customNetworkInterfaceName: 'nic-pe-${keyVaultName}' + } + tags: commonTags + + resource privateDnsZoneGroup 'privateDnsZoneGroups' = { + name: 'default' + properties: { + privateDnsZoneConfigs: [ + { + name: 'config1' + properties: { + privateDnsZoneId: keyVaultDnsZone.id + } + } + ] + } + } +} + +// Diagnostic Settings for Key Vault +resource keyVaultDiagnostics 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { + name: 'diag-${keyVaultName}' + scope: keyVault + properties: { + workspaceId: logAnalyticsWorkspace.id + logs: [ + { + category: 'AuditEvent' + enabled: true + } + { + category: 'AzurePolicyEvaluationDetails' + enabled: true + } + ] + metrics: [ + { + category: 'AllMetrics' + enabled: true + } + ] + } +} + +// Event Grid System Topic for the KeyVault +resource keyVaultEventGridSystemTopic 'Microsoft.EventGrid/systemTopics@2025-02-15' = { + name: 'egst-${keyVaultName}' + location: location + identity: { + type: 'SystemAssigned' + } + properties: { + source: keyVault.id + topicType: 'Microsoft.KeyVault.Vaults' + } + tags: commonTags +} + +// Event Grid subscription for the KeyVault to the queue +// This subscription filters only the CertificateNearExpiry events and sends them to the storage queue +resource keyVaultEventGridSubscription 'Microsoft.EventGrid/systemTopics/eventSubscriptions@2025-02-15' = { + parent: keyVaultEventGridSystemTopic + name: 'egs-${keyVaultEventGridSystemTopic.name}' + properties: { + destination: { + endpointType: 'StorageQueue' + properties: { + resourceId: storageAccount.id + queueName: 'certlc' + queueMessageTimeToLiveInSeconds: 86400 // 1 day + } + } + eventDeliverySchema: 'CloudEventSchemaV1_0' + filter: { + includedEventTypes: [ + 'Microsoft.KeyVault.CertificateNearExpiry' + ] + isSubjectCaseSensitive: false + } + retryPolicy: { + maxDeliveryAttempts: 30 + eventTimeToLiveInMinutes: 1440 // 1 day + } + } +} + +// Azure Monitor Workbook for Certificate Statistics +resource workbookCertLCStats 'Microsoft.Insights/workbooks@2023-06-01' = { + name: guid(resourceGroup().id, 'certlcstats') + location: location + kind: 'shared' + properties: { + displayName: 'certlcstats' + serializedData: '{"version":"Notebook/1.0","items":[],"styleSettings":{},"$schema":"https://github.com/Microsoft/Application-Insights-Workbooks/blob/master/schema/workbook.json"}' + category: 'workbook' + sourceId: logAnalyticsWorkspace.id + } + dependsOn: [ + applicationInsights // Wait for App Insights to ensure workspace is fully active + ] + tags: commonTags +} + +// Role Assignment: Grant the Event Grid System Topic the "Storage Queue Data Reader" role on the Storage Account +// this role allows Event Grid to read messages from the queue +resource eventGridStorageQueueDataReader 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(subscription().id, resourceGroup().id, 'eventGridStorageQueueDataReader') + scope: storageAccount + properties: { + description: 'EventGrid SystemTopic -> Storage Queue Data Reader -> Storage Account' + roleDefinitionId: subscriptionResourceId( + 'Microsoft.Authorization/roleDefinitions', + roleDefinitions.storageQueueDataReader + ) + principalId: keyVaultEventGridSystemTopic.identity.principalId + principalType: 'ServicePrincipal' + } +} + +// Role Assignment: Grant the Event Grid System Topic the "Storage Queue Data Message Sender" role on the Storage Account +// this role allows Event Grid to send messages to the queue +resource eventGridStorageQueueDataMessageSender 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(subscription().id, resourceGroup().id, 'eventGridStorageQueueDataMessageSender') + scope: storageAccount + properties: { + description: 'EventGrid SystemTopic -> Storage Queue Data Message Sender -> Storage Account' + roleDefinitionId: subscriptionResourceId( + 'Microsoft.Authorization/roleDefinitions', + roleDefinitions.storageQueueDataMessageSender + ) + principalId: keyVaultEventGridSystemTopic.identity.principalId + principalType: 'ServicePrincipal' + } +} + +// Role Assignment: Grant the Automation Account the "Key Vault Certificates Officer" role on the KeyVault +// this role allows the automation account to create and manage certificates in the KeyVault +resource automationAccountKeyVaultCertificatesOfficer 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(subscription().id, resourceGroup().id, 'automationAccountKeyVaultCertificatesOfficer') + scope: keyVault + properties: { + description: 'Automation Account -> Key Vault Certificates Officer -> Key Vault' + roleDefinitionId: subscriptionResourceId( + 'Microsoft.Authorization/roleDefinitions', + roleDefinitions.keyVaultCertificatesOfficer + ) + principalId: automationAccount.identity.principalId + principalType: 'ServicePrincipal' + } +} + +// Role Assignment: Grant the Automation Account the "Key Vault Secrets Officer" role on the KeyVault +// this role allows the automation account to create and manage secrets (private keys of the certificates) in the KeyVault +resource automationAccountKeyVaultSecretsOfficer 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(subscription().id, resourceGroup().id, 'automationAccountKeyVaultSecretsOfficer') + scope: keyVault + properties: { + description: 'Automation Account -> Key Vault Secrets Officer -> Key Vault' + roleDefinitionId: subscriptionResourceId( + 'Microsoft.Authorization/roleDefinitions', + roleDefinitions.keyVaultSecretsOfficer + ) + principalId: automationAccount.identity.principalId + principalType: 'ServicePrincipal' + } +} + +// Role Assignment: Grant the Automation Account the "Reader" role on the Automation Account +// This may seem strange, but it is required for the hybrid worker (that uses the automation account's identity) to read the automation account variables +resource automationAccountReader 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(subscription().id, resourceGroup().id, 'automationAccountReader') + scope: automationAccount + properties: { + description: 'Automation Account -> Reader -> Automation Account (self)' + roleDefinitionId: subscriptionResourceId( + 'Microsoft.Authorization/roleDefinitions', + roleDefinitions.reader + ) + principalId: automationAccount.identity.principalId + principalType: 'ServicePrincipal' + } +} + +// Role Assignment: Grant the Automation Account the "Monitoring Metrics Publisher" role on the DCR +// (this is to allow the automation account to write custom logs to the DCR) +resource automationAccountMonitoringMetricsPublisher 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(subscription().id, resourceGroup().id, 'automationAccountMonitoringMetricsPublisher') + scope: dataCollectionRule + properties: { + description: 'Automation Account -> Monitoring Metrics Publisher -> DCR' + roleDefinitionId: subscriptionResourceId( + 'Microsoft.Authorization/roleDefinitions', + roleDefinitions.monitoringMetricsPublisher + ) + principalId: automationAccount.identity.principalId + principalType: 'ServicePrincipal' + } +} + +// Role Assignment: Grant the Function App the "Storage Blob Data Owner" role on the Storage Account +resource functionAppStorageBlobDataOwner 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(subscription().id, resourceGroup().id, 'functionAppStorageBlobDataOwner') + scope: storageAccount + properties: { + description: 'Function App -> Storage Blob Data Owner -> Storage Account' + roleDefinitionId: subscriptionResourceId( + 'Microsoft.Authorization/roleDefinitions', + roleDefinitions.storageBlobDataOwner + ) + principalId: functionApp.identity.principalId + principalType: 'ServicePrincipal' + } +} + +// Role Assignment: Grant the Function App the "Storage Queue Data Message Processor" role on the Storage Account +resource functionAppStorageQueueDataMessageProcessor 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(subscription().id, resourceGroup().id, 'functionAppStorageQueueDataMessageProcessor') + scope: storageAccount + properties: { + description: 'Function App -> Storage Queue Data Message Processor -> Storage Account' + roleDefinitionId: subscriptionResourceId( + 'Microsoft.Authorization/roleDefinitions', + roleDefinitions.storageQueueDataMessageProcessor + ) + principalId: functionApp.identity.principalId + principalType: 'ServicePrincipal' + } +} + +// Role Assignment: Grant the Function App the "Storage Queue Data Contributor" role on the Storage Account +resource functionAppStorageQueueDataContributor 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(subscription().id, resourceGroup().id, 'functionAppStorageQueueDataContributor') + scope: storageAccount + properties: { + description: 'Function App -> Storage Queue Data Contributor -> Storage Account' + roleDefinitionId: subscriptionResourceId( + 'Microsoft.Authorization/roleDefinitions', + roleDefinitions.storageQueueDataContributor + ) + principalId: functionApp.identity.principalId + principalType: 'ServicePrincipal' + } +} + +// Role Assignment: Grant the Function App the "Reader" role on the Automation Account +// (this is to allow the function app to read automation account information and trigger runbooks) +resource functionAppAutomationAccountReader 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(subscription().id, resourceGroup().id, 'functionAppAutomationAccountReader') + scope: automationAccount + properties: { + description: 'Function App -> Reader -> Automation Account' + roleDefinitionId: subscriptionResourceId( + 'Microsoft.Authorization/roleDefinitions', + roleDefinitions.reader + ) + principalId: functionApp.identity.principalId + principalType: 'ServicePrincipal' + } +} + +// Role Assignment: Grant the Function App the "Automation Operator" role on the Automation Account +// (this is to allow the function app to start runbook jobs) +resource functionAppAutomationOperator 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(subscription().id, resourceGroup().id, 'functionAppAutomationOperator') + scope: automationAccount + properties: { + description: 'Function App -> Automation Operator -> Automation Account' + roleDefinitionId: subscriptionResourceId( + 'Microsoft.Authorization/roleDefinitions', + roleDefinitions.automationOperator + ) + principalId: functionApp.identity.principalId + principalType: 'ServicePrincipal' + } +} + +// Role Assignment: Grant the Function App the "Monitoring Metrics Publisher" role on the Application Insights instance +// (this is to instrument the function app with App Insights) +resource functionAppMonitoringMetricsPublisher 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(subscription().id, resourceGroup().id, 'functionAppMonitoringMetricsPublisher') + scope: applicationInsights + properties: { + description: 'Function App -> Monitoring Metrics Publisher -> Application Insights' + roleDefinitionId: subscriptionResourceId( + 'Microsoft.Authorization/roleDefinitions', + roleDefinitions.monitoringMetricsPublisher + ) + principalId: functionApp.identity.principalId + principalType: 'ServicePrincipal' + } +} + +// Output all resource IDs and important properties +output storageAccountId string = storageAccount.id +output storageAccountQueueUri string = storageAccount.properties.primaryEndpoints.queue +output automationAccountId string = automationAccount.id +output keyVaultId string = keyVault.id +output functionAppId string = functionApp.id +output logAnalyticsWorkspaceId string = logAnalyticsWorkspace.id +output applicationInsightsId string = applicationInsights.id +output dceIngestionEndpoint string = dataCollectionEndpoint.properties.logsIngestion.endpoint +@secure() +output dataCollectionRuleImmutableId string = dataCollectionRule.properties.immutableId diff --git a/Setup/getavailability.bicep b/Setup/getavailability.bicep new file mode 100644 index 0000000..e69de29 diff --git a/Setup/parameters.dev.bicepparam b/Setup/parameters.dev.bicepparam new file mode 100644 index 0000000..c9c3898 --- /dev/null +++ b/Setup/parameters.dev.bicepparam @@ -0,0 +1,29 @@ +using './getavailability.bicep' + +// Network Configuration +param peSubnetId = '/subscriptions/9068a229-f092-400e-8093-87e8e7d26ae1/resourceGroups/rg-alz-net-workloads-itn-001/providers/Microsoft.Network/virtualNetworks/vnet-alz-workloads-itn-001/subnets/snet-alz-pe-workloads-itn-001' +param fnSubnetId = '/subscriptions/9068a229-f092-400e-8093-87e8e7d26ae1/resourceGroups/rg-alz-net-workloads-itn-001/providers/Microsoft.Network/virtualNetworks/vnet-alz-workloads-itn-001/subnets/snet-alz-fn-workloads-itn-001' + +// DNS Configuration +param dnsZonesSubscriptionId = 'c4e6c176-bf9c-4e8c-87b2-ebdceea7085f' +param dnsZonesResourceGroupName = 'rg-alz-dns-hub-itn-001' + +// Resource Names +param storageAccountName = 'flazstcertlcitn001' +param functionAppName = 'flazfn-certlc-itn-001' +param logAnalyticsWorkspaceName = 'log-certlc-itn-001' +param applicationInsightsName = 'appi-certlc-itn-001' +param automationAccountName = 'aa-certlc-itn-001' +param hybridWorkerGroupName = 'hwg-certlc-itn-001' +param runbookName = "certlc" +param keyVaultName = 'flazkv-certlc-itn-001' +param dataCollectionEndpointName = 'dce-certlc-itn-001' +param dataCollectionRuleName = 'dcr-certlc-itn-001' + +// Automation account variables +param automationAccountVarCA = 'flazdc03.lab.formicalab.casa\\SubCA' // Name of the CA to use (for the automation account variable) +param automationAccountVarPfxRootFolder = 'C:\\PFX_Repo' // Name of the folder to use (for the automation account variable) +param automationAccountVarSmtpFrom = 'certlc@formicalab.casa' // SMTP From address to use (for the automation account variable) +param automationAccountVarSmtpServer = 'mail.smtp2go.com' // SMTP Server to use (for the automation account variable) +param automationAccountVarSmtpUser = 'certlc' // SMTP User to use (for the automation account variable) +param automationAccountVarSmtpPassword = '' // SMTP Password to use (for the automation account variable) From 7e50cd37967585937757eb19ac44a439aa5a142b Mon Sep 17 00:00:00 2001 From: Marcello Formica Date: Mon, 27 Apr 2026 13:32:53 +0200 Subject: [PATCH 02/10] feat: add optional Log Analytics ingestion to PowerShell script - Add -DceEndpoint and -DcrImmutableId parameters to get-availability.ps1 - Add Send-ToLogAnalytics function (gzip, 900KB batching) - Add ingestion block: builds resource + summary payloads, sends to custom tables - Add subscription-scoped Bicep (getavailability.bicep + resources module) creates: Log Analytics workspace, 2 custom tables, DCE, DCR - Update parameters.dev.bicepparam for new Bicep - Rewrite Setup/README.md for Get-Availability infrastructure - Create csharp/README.md with C#-specific content from root README - Update root README.md: add new params, ingestion section, link to C# README --- README.md | 106 ++++------ Setup/PLAN-log-analytics-ingestion.md | 289 ++++++++++++++++++++++++++ Setup/README.md | 108 ++++++++++ Setup/getavailability-resources.bicep | 202 ++++++++++++++++++ Setup/getavailability.bicep | 65 ++++++ Setup/parameters.dev.bicepparam | 32 +-- csharp/README.md | 77 +++++++ get-availability.ps1 | 208 ++++++++++++++++++ 8 files changed, 991 insertions(+), 96 deletions(-) create mode 100644 Setup/PLAN-log-analytics-ingestion.md create mode 100644 Setup/README.md create mode 100644 Setup/getavailability-resources.bicep create mode 100644 csharp/README.md diff --git a/README.md b/README.md index fa4e957..e67e6a9 100644 --- a/README.md +++ b/README.md @@ -5,12 +5,12 @@ Reports month-scoped availability for Azure Virtual Machines, Azure SQL Databases, Azure Storage Accounts, and Azure Web Apps across one or more Azure subscriptions. -Two equivalent implementations are provided: +Two implementations are provided: | Version | Path | Runtime | Notes | |---|---|---|---| -| **C#** | `csharp/GetAvailability/` | .NET 10 Native AOT (~15 MB standalone binary, no runtime required) | Fastest; recommended for production use | -| **PowerShell** | `get-availability.ps1` | PowerShell 7+ with `Az.Accounts` and `Az.ResourceGraph` modules | No build step; convenient for ad-hoc use | +| **C#** | [`csharp/`](csharp/README.md) | .NET 10 Native AOT (~15 MB standalone binary, no runtime required) | Fastest; recommended for production use | +| **PowerShell** | `get-availability.ps1` | PowerShell 7+ with `Az.Accounts` and `Az.ResourceGraph` modules | No build step; convenient for ad-hoc use; supports Log Analytics ingestion | Both versions share the same pipeline, classification rules, output format, and invariants. @@ -28,17 +28,6 @@ The relationship `Suspect = Faults + Excused + Unresolved` always holds. ### Prerequisites -**C# version:** - -| Requirement | Detail | -|---|---| -| .NET SDK | 10.0 or later (build from source only) | -| Azure auth | `az login` or any method supported by `DefaultAzureCredential` | - -The published binary (`GetAvailability.exe`) requires no .NET runtime — it is a Native AOT self-contained executable. - -**PowerShell version:** - | Requirement | Detail | |---|---| | PowerShell | 7.0 or later (`pwsh`) | @@ -46,26 +35,11 @@ The published binary (`GetAvailability.exe`) requires no .NET runtime — it is | Az.ResourceGraph | `Install-Module Az.ResourceGraph` | | Azure auth | `Connect-AzAccount` (used by both Az modules and for ARM token acquisition) | -If Azure authentication fails, the tool prints the SDK/module exception message directly. For Azure CLI-based auth (C#), re-run `az login`; for PowerShell, re-run `Connect-AzAccount`. - -### Parameters +If Azure authentication fails, the tool prints the module exception message directly. Re-run `Connect-AzAccount` to fix. -**C# (`GetAvailability.exe`):** +For the C# version prerequisites and usage, see the [C# README](csharp/README.md). -| Option | Short | Default | Description | -|---|---|---|---| -| `--subscriptions` | `-s` | *(required)* | One or more Azure subscription display names | -| `--month` | `-m` | *(required)* | Observation month in UTC, format `YYYYMM` | -| `--kinds` | `-k` | `vm sql storage webapp` | Resource kinds to process | -| `--resource` | `-r` | *(all)* | Filter to a single resource name | -| `--parallelism` | `-p` | *(auto)* | Max concurrent API calls (scales to CPU cores) | -| `--activity-grace-minutes` | `-g` | `10` | Post-operation grace window for Activity Log lifecycle events | -| `--batch` | `-b` | off | Use the regional Metrics Batch API instead of per-resource calls | -| `--batch-size` | | `10` | Max resources per batch call (1–50); implies `--batch` | -| `--workspace` | `-w` | *(none)* | Log Analytics workspace ID (GUID). Fetches Activity Log via bulk KQL; Resource Health uses hybrid approach (KQL for older + REST for last ~30 days) | -| `--version` | `-v` | | Print version and exit | - -**PowerShell (`get-availability.ps1`):** +### Parameters | Parameter | Default | Description | |---|---|---| @@ -78,41 +52,16 @@ If Azure authentication fails, the tool prints the SDK/module exception message | `-Batch` | off | Use the regional Metrics Batch API instead of per-resource calls | | `-BatchSize` | `10` | Max resources per batch call (1–50); implies `-Batch` | | `-Workspace` | *(none)* | Log Analytics workspace ID (GUID). Fetches Activity Log lifecycle events from the workspace via a single bulk KQL query (faster for large estates). Resource Health uses a hybrid approach: KQL transitions cover the period beyond the REST API's ~30-day retention, while REST API transitions (curated, with corrected causes) are authoritative for the last ~30 days. Provides complete Resource Health coverage across the full observation window. | +| `-DceEndpoint` | *(none)* | Data Collection Endpoint ingestion URL. When provided together with `-DcrImmutableId`, results are sent to Log Analytics custom tables via the Azure Monitor Ingestion API. | +| `-DcrImmutableId` | *(none)* | Data Collection Rule immutable ID. Required together with `-DceEndpoint` to enable Log Analytics ingestion. | | `-Version` | | Print version and exit | +For C# parameters, see the [C# README](csharp/README.md). + The observation window is a UTC calendar month: past months use the full calendar month, the current month is reported month-to-date. The requested month cannot start more than 90 days before the current UTC time. Metrics and Activity Log support that 90-day lookback; Health History is applied only for its overlap with the ~30-day REST API retention window. When `-Workspace` / `--workspace` is used, Health History coverage extends to the full observation period via a hybrid approach (Log Analytics for older transitions + REST API for the last ~30 days). ### Examples -**C#:** - -```bash -# Build the Native AOT binary (one-time) -cd csharp/GetAvailability -dotnet publish -c Release -r win-x64 # output in bin/Release/net10.0/win-x64/publish/ - -# Single subscription -./GetAvailability --subscriptions Contoso-Production --month 202603 - -# Multiple subscriptions, filtered by kind -./GetAvailability --subscriptions Contoso-Development Contoso-Production --month 202603 --kinds vm sql - -# Single resource (SQL database by server/database name) -./GetAvailability --subscriptions Contoso-Development --month 202603 --resource sqlserver01/sqldb01 - -# Batch API with custom batch size -./GetAvailability --subscriptions Contoso-Production Contoso-Development --month 202603 --batch-size 20 - -# Use Log Analytics for Activity Log + Resource Health (faster, extended retention) -./GetAvailability --subscriptions Contoso-Production --month 202603 --workspace b233a4b7-3c43-433c-ac60-1f6ff217ddd4 - -# Run directly without publishing -cd csharp/GetAvailability -dotnet run -- --subscriptions Contoso-Production --month 202603 -``` - -**PowerShell:** - ```powershell # Single subscription ./get-availability.ps1 -Subscriptions 'Contoso-Production' -Month 202603 @@ -129,10 +78,17 @@ dotnet run -- --subscriptions Contoso-Production --month 202603 # Use Log Analytics for Activity Log + Resource Health (faster, extended retention) ./get-availability.ps1 -Subscriptions 'Contoso-Production' -Month 202603 -Workspace 'b233a4b7-3c43-433c-ac60-1f6ff217ddd4' +# Send results to Log Analytics custom tables +./get-availability.ps1 -Subscriptions 'Contoso-Production' -Month 202603 ` + -DceEndpoint 'https://dce-getavail-itn-001.italynorth-1.ingest.monitor.azure.com' ` + -DcrImmutableId 'dcr-00000000000000000000000000000000' + # Pipe results to CSV ./get-availability.ps1 -Subscriptions 'Contoso-Production' -Month 202603 | Export-Csv availability.csv ``` +For C# examples, see the [C# README](csharp/README.md). + ### Output The header line shows the observation window and total minutes: @@ -287,15 +243,27 @@ AvailabilityPct = 40,066 / 40,125 × 100 = 99.85390% ## Implementation notes -These notes cover performance and implementation details specific to each version. +These notes cover performance and implementation details specific to the PowerShell version. For C# implementation notes, see the [C# README](csharp/README.md). -- **`Parallel.ForEachAsync`** (C#) / **`ForEach-Object -Parallel`** (PowerShell) for concurrent metric, Activity Log, and Resource Health queries with configurable parallelism. -- **Shared `HttpClient`** with connection pooling (PowerShell) — avoids per-request TCP/TLS overhead; streams JSON responses directly into `System.Text.Json` without intermediate string allocation. Used for both per-resource metrics and gap investigation paths. -- **Compiled metric processor** (PowerShell) — the ~44k-datapoint-per-resource JSON processing loop is compiled as C# via `Add-Type` and runs at native .NET speed. -- **Compiled gap processor** (PowerShell) — `ExpandToTickSet` (interval → `HashSet`) and `ClassifyGaps` (minute-by-minute classification) are also compiled via `Add-Type`. -- **Idempotent `Add-Type` guards** (PowerShell) — each compiled C# block (`MetricProcessor`, `GapProcessor`) is independently guarded by a `PSTypeName` check so the script can be re-run within the same session. +- **`ForEach-Object -Parallel`** for concurrent metric, Activity Log, and Resource Health queries with configurable parallelism. +- **Shared `HttpClient`** with connection pooling — avoids per-request TCP/TLS overhead; streams JSON responses directly into `System.Text.Json` without intermediate string allocation. Used for both per-resource metrics and gap investigation paths. +- **Compiled metric processor** — the ~44k-datapoint-per-resource JSON processing loop is compiled as C# via `Add-Type` and runs at native .NET speed. +- **Compiled gap processor** — `ExpandToTickSet` (interval → `HashSet`) and `ClassifyGaps` (minute-by-minute classification) are also compiled via `Add-Type`. +- **Idempotent `Add-Type` guards** — each compiled C# block (`MetricProcessor`, `GapProcessor`) is independently guarded by a `PSTypeName` check so the script can be re-run within the same session. - **HashSet-based interval containment** — suspect-minute classification pre-expands intervals into `HashSet` tick sets for O(1) lookups instead of linear scans. - **O(1) JSON property access** — `TryGetProperty` hash lookup instead of `EnumerateObject` linear scan (~44k calls per resource per month). - **Ticks-based metric keying** — `long` instead of `DateTime` for zero-allocation per data point. -- **`System.Text.Json`** for efficient JSON parsing — avoids large PSObject trees in PowerShell and enables AOT-safe parsing in C#. -- **Native AOT** (C#) — ~15 MB standalone binary, no .NET runtime required. +- **`System.Text.Json`** for efficient JSON parsing — avoids large PSObject trees. + +## Log Analytics Ingestion (Optional) + +When `-DceEndpoint` and `-DcrImmutableId` are provided, the script sends results to two Log Analytics custom tables via the [Azure Monitor Ingestion API](https://learn.microsoft.com/azure/azure-monitor/logs/logs-ingestion-api-overview): + +| Table | Content | +|---|---| +| `GetAvailResources_CL` | Per-resource detail (one row per resource per run) | +| `GetAvailSummary_CL` | Aggregated summaries (per Kind+Location, per subscription, overall) | + +Authentication uses `Get-AzAccessToken -ResourceUrl 'https://monitor.azure.com'`, which works identically for interactive sessions (`Connect-AzAccount`) and Azure Function managed identities. Payloads are gzip-compressed and batched at 900 KB to stay within API limits. + +The infrastructure (Log Analytics workspace, custom tables, DCE, DCR) is deployed via the Bicep templates in the [`Setup/`](Setup/README.md) directory. diff --git a/Setup/PLAN-log-analytics-ingestion.md b/Setup/PLAN-log-analytics-ingestion.md new file mode 100644 index 0000000..fcf51b2 --- /dev/null +++ b/Setup/PLAN-log-analytics-ingestion.md @@ -0,0 +1,289 @@ +# Plan: Ingest Get-Availability Results into Log Analytics Custom Tables + +## Goal + +After each run of Get-Availability, push the per-resource detail rows **and** the +aggregated summary rows into custom tables in a Log Analytics workspace so that +results can be queried with KQL, visualized in Workbooks, and trended over time. + +--- + +## 1. Custom Tables + +### Table A — `GetAvailResources_CL` (per-resource detail) + +One row per resource per run. Carries the full availability breakdown. + +| Column | Type | Source / Notes | +|----------------------------|------------|----------------------------------------------------------| +| `TimeGenerated` | datetime | Injected by DCR transform: `now()` | +| `RunId` | string | GUID generated once per execution (correlates all rows) | +| `Month` | string | Observation month, e.g. `"202604"` | +| `PeriodStart` | datetime | UTC start of observation window | +| `PeriodEnd` | datetime | UTC end of observation window | +| `IsMonthToDate` | boolean | `true` if the run was mid-month | +| `SubscriptionName` | string | Azure subscription display name | +| `ResourceName` | string | Resource name | +| `ResourceId` | string | Full ARM resource ID | +| `ResourceGroup` | string | Resource group name | +| `Kind` | string | `VirtualMachine`, `AzureSqlDatabase`, `StorageAccount`, `WebApp` | +| `Location` | string | Azure region | +| `EligibleMinutes` | int | Minutes eligible for availability measurement | +| `AvailableMinutes` | real | Actual available minutes (may be fractional due to degraded datapoints) | +| `SuspectMinutes` | int | Total suspect minutes from metric scan | +| `ConfirmedDowntimeMinutes` | int | Platform-fault minutes confirmed by Resource Health | +| `ExcusedMinutes` | int | Minutes excused from eligibility (lifecycle, customer, metric issues, zero-tx) | +| `UnexplainedSuspectMinutes`| int | Suspect minutes remaining after all classification | +| `AvailabilityPct` | real | Availability percentage (5 decimal places); -1 for N/A resources | + +### Table B — `GetAvailSummary_CL` (aggregated summaries) + +One row per aggregation group per run. Stores the subscription-level and +cross-subscription roll-ups. + +| Column | Type | Source / Notes | +|---------------------|------------|-------------------------------------------------------------------| +| `TimeGenerated` | datetime | Injected by DCR transform: `now()` | +| `RunId` | string | Same GUID as the detail rows (for correlation) | +| `Month` | string | Observation month, e.g. `"202604"` | +| `PeriodStart` | datetime | UTC start of observation window | +| `PeriodEnd` | datetime | UTC end of observation window | +| `IsMonthToDate` | boolean | `true` if mid-month | +| `SummaryLevel` | string | `KindLocation` / `SubscriptionTotal` / `Overall` | +| `SubscriptionName` | string | Subscription name (empty for `Overall` rows) | +| `Kind` | string | Resource kind (empty for `SubscriptionTotal` and `Overall` rows) | +| `Location` | string | Azure region (empty for `SubscriptionTotal` and `Overall` rows) | +| `ResourceCount` | int | Number of resources in this group | +| `EligibleMinutes` | real | Sum of eligible minutes across resources in the group | +| `AvailableMinutes` | real | Sum of available minutes across resources in the group | +| `AvailabilityPct` | real | Aggregate availability percentage for the group | + +--- + +## 2. Data Collection Endpoint (DCE) + +A single DCE is created to provide the ingestion URL. Both tables will share +this endpoint. Network access will be set to `Enabled` (can be locked down +later with Private Link if needed). + +**Resource:** `Microsoft.Insights/dataCollectionEndpoints` + +--- + +## 3. Data Collection Rule (DCR) + +A single DCR declares **two custom streams** — one per table — and routes each +stream to the corresponding custom table in the workspace. + +| Stream | Target Table | +|-----------------------------------|--------------------------| +| `Custom-GetAvailResources_CL` | `GetAvailResources_CL` | +| `Custom-GetAvailSummary_CL` | `GetAvailSummary_CL` | + +Each stream has a `transformKql` that injects `TimeGenerated = now()` and +performs any necessary type coercion (e.g. `todatetime()` on the period +timestamps). + +--- + +## 4. Bicep File: `getavailability.bicep` + +The Bicep file uses `targetScope = 'subscription'` so it can create its own +dedicated resource group. It receives parameters for the resource group name, +Log Analytics workspace name, region, DCE name, and DCR name. + +It creates (in order): + +1. Resource group (dedicated to Get-Availability telemetry) +2. Log Analytics workspace (inside the new resource group) +3. Custom table `GetAvailResources_CL` (child of the workspace) +4. Custom table `GetAvailSummary_CL` (child of the workspace) +5. Data Collection Endpoint +6. Data Collection Rule (with `dependsOn` on both tables and the DCE) + +Because the scope is `subscription`, the resource group is created via a +top-level `resource` declaration, and all other resources are deployed via a +Bicep module (or nested `module` with `scope: resourceGroup(...)`) targeting +the newly created resource group. + +Deployment command changes accordingly: +```powershell +# Subscription-scoped deployment (no --resource-group flag) +az deployment sub create --location --parameters .\parameters.dev.bicepparam +``` + +The Bicep file outputs: +- Resource group name +- Log Analytics workspace ID +- DCE ingestion endpoint URL +- DCR immutable ID +- DCR stream names (for the caller / script to use when posting data) + +--- + +## 5. Implementation Steps (for later) + +- [ ] **Step 1:** Write `getavailability.bicep` with resource group + Log Analytics workspace + custom tables + DCE + DCR + - `targetScope = 'subscription'` + - Create the resource group first, then deploy all other resources into it + (using a Bicep module scoped to the new resource group) +- [ ] **Step 2:** Update `parameters.dev.bicepparam`: + - Change the `using` directive from `'./certlc.bicep'` → `'./getavailability.bicep'` + - Remove **all** CertLC-specific parameters: + - `peSubnetId`, `fnSubnetId` (no private endpoints or function apps) + - `dnsZonesSubscriptionId`, `dnsZonesResourceGroupName` (no private DNS) + - `storageAccountName`, `functionAppName`, `applicationInsightsName` (not used) + - `automationAccountName`, `hybridWorkerGroupName`, `runbookName` (not used) + - `keyVaultName` (not used) + - All `automationAccountVar*` parameters (not used) + - Add **all** parameters required by `getavailability.bicep`: + - `resourceGroupName` — name of the dedicated resource group to create + - `location` — Azure region for all resources + - `logAnalyticsWorkspaceName` — name of the Log Analytics workspace to create + - `dataCollectionEndpointName` — name for the DCE + - `dataCollectionRuleName` — name for the DCR + - The resulting file should be minimal, e.g.: + ```bicepparam + using './getavailability.bicep' + param resourceGroupName = 'rg-getavail-itn-001' + param location = 'italynorth' + param logAnalyticsWorkspaceName = 'log-getavail-itn-001' + param dataCollectionEndpointName = 'dce-getavail-itn-001' + param dataCollectionRuleName = 'dcr-getavail-itn-001' + ``` +- [ ] **Step 3:** Add new **optional** parameters to `get-availability.ps1`: + - `-DceEndpoint [string]` — DCE logs ingestion URL (from Bicep output `dceIngestionEndpoint`) + - `-DcrImmutableId [string]` — DCR immutable ID (from Bicep output `dataCollectionRuleImmutableId`) + - Both are optional; ingestion happens **only** when both are provided + - When omitted, the script behaves exactly as today (console output only) + - Add validation: if one is supplied without the other, throw an error + - Use a `$sendToLogAnalytics = $DceEndpoint -and $DcrImmutableId` flag to guard all + ingestion code paths — zero overhead when ingestion is not requested +- [ ] **Step 4:** Add a helper function `Send-ToLogAnalytics` in `get-availability.ps1`: + - Uses the **Azure Monitor Ingestion** REST API + (`POST https://{dce-endpoint}/dataCollectionRules/{dcr-immutableId}/streams/{streamName}?api-version=2023-01-01`) + - **Authentication — dual execution context:** + The script runs in two environments and must acquire an Azure Monitor token in both: + 1. **Interactive** (`az login` / `Connect-AzAccount`): use `Get-AzAccessToken` + with `-ResourceUrl 'https://monitor.azure.com'`, same pattern as the existing + ARM token acquisition in Step 2 + 2. **Azure Function** (managed identity): the `Az.Accounts` module is available in + the PowerShell worker; `Connect-AzAccount -Identity` is typically run at function + startup (or by the host), so `Get-AzAccessToken` works identically — no code change + needed for this path + Implementation: acquire the monitor token once near the top of the ingestion block + (right after the existing ARM token), using the same `Get-AzAccessToken` + + `SecureString` handling pattern already in the script: + ```powershell + $rawMonitor = (Get-AzAccessToken -ResourceUrl 'https://monitor.azure.com').Token + $monitorToken = ($rawMonitor -is [securestring]) ` + ? ($rawMonitor | ConvertFrom-SecureString -AsPlainText) : [string]$rawMonitor + $rawMonitor = $null + ``` + - Accepts: endpoint URL, DCR immutable ID, stream name, bearer token, and an array of + PSObjects (the payload) + - Serializes the array to JSON with `ConvertTo-Json -Depth 5 -Compress` + - Sets `Content-Type: application/json` and `Content-Encoding: gzip` (gzip the body for efficiency) + - Handles the Ingestion API's 1 MB per call limit: if the payload exceeds ~900 KB, split into batches + - Returns nothing on success (204); throws on failure with status code and body +- [ ] **Step 5:** Build and send the **per-resource detail** payload after Step 8 (output): + - Generate a `$runId = [guid]::NewGuid().ToString()` once per execution + - Map each `$eligByRes` entry to a hashtable matching the `Custom-GetAvailResources_CL` stream schema: + ``` + @{ + RunId = $runId + Month = $normalizedMonth + PeriodStart = $utcStart.ToString('o') + PeriodEnd = $utcEnd.ToString('o') + IsMonthToDate = $isMonthToDate + SubscriptionName = $elig.SubscriptionName + ResourceName = $elig.Name + ResourceId = $elig.ResourceId + ResourceGroup = $elig.ResourceGroupName + Kind = $elig.Kind + Location = $elig.Location + EligibleMinutes = $elig.EligibleMinutes + AvailableMinutes = $elig.AvailableMinutes + SuspectMinutes = $elig.SuspectMinutes + ConfirmedDowntimeMinutes = $elig.ConfirmedDowntimeMinutes + ExcusedMinutes = $elig.ExcusedMinutes + UnexplainedSuspectMinutes= $elig.UnexplainedSuspectMinutes + AvailabilityPct = if ($elig.AvailabilityPct -eq 'N/A') { -1 } else { [double]$elig.AvailabilityPct } + } + ``` + - Call `Send-ToLogAnalytics` with stream name `Custom-GetAvailResources_CL` +- [ ] **Step 6:** Build and send the **summary** payload: + - Reuse the same grouping logic already in `Write-SubscriptionSummaries` to produce rows: + - One row per Kind+Location per subscription (`SummaryLevel = 'KindLocation'`) + - One row per subscription total (`SummaryLevel = 'SubscriptionTotal'`) + - One overall row across all subscriptions (`SummaryLevel = 'Overall'`, only if >1 subscription) + - Each row is a hashtable matching the `Custom-GetAvailSummary_CL` stream schema + - Call `Send-ToLogAnalytics` with stream name `Custom-GetAvailSummary_CL` +- [ ] **Step 7:** Add RBAC: the caller identity needs **Monitoring Metrics Publisher** role on the DCR + - Document this in the script help text and in the Bicep file comments + - The Bicep file should optionally accept a principal ID to assign the role (or leave it as a manual step) +- [ ] **Step 8:** Rewrite `Setup/README.md` for Get-Availability (replace CertLC content): + - **Remove entirely** all CertLC-specific content: + - Title, description, and solution overview references to CertLC + - Prerequisites: VNet/subnet requirements, Private DNS Zones, Hybrid Worker VM + - RBAC section: Owner role, Private DNS Zone Contributor, all CertLC role assignment tables + - Resources Created: all 16 CertLC resources (Storage Account, Function App, Automation Account, + Key Vault, Event Grid, Private Endpoints, DNS Zone Groups, Workbook, etc.) + - Parameters table: all CertLC parameters (peSubnetId, fnSubnetId, dnsZones*, storage*, function*, + automation*, keyVault*, automationAccountVar*, scheduleStartTime) + - Post-Deployment Steps: hybrid worker registration, runbook upload, certlcstats schedule, + function app deployment, CA permissions, workbook customization, end-to-end testing + - Security Notes section (CertLC-specific) + - Manual Configuration (on-premises CA) section + - Files section listing `certlc.bicep` + - **Replace with** Get-Availability infrastructure content: + - Title: "Get-Availability Setup" (or similar) + - Purpose: deploys custom Log Analytics tables, DCE, and DCR for ingesting + Get-Availability script results + - Prerequisites: Contributor role on the subscription (to create the resource + group and resources), Monitoring Metrics Publisher on the DCR for the caller identity + - Deployment commands: subscription-scoped (`az deployment sub create --location + --parameters .\parameters.dev.bicepparam`) — no `--resource-group` flag + - Resources Created: 6 resources — 1 resource group, 1 Log Analytics workspace, + 2 custom tables, 1 DCE, 1 DCR + - Parameters table: `resourceGroupName`, `location`, `logAnalyticsWorkspaceName`, + `dataCollectionEndpointName`, `dataCollectionRuleName` + - Outputs: DCE ingestion endpoint, DCR immutable ID + - Post-Deployment: how to use `-DceEndpoint` / `-DcrImmutableId` with the script + - Files section listing `getavailability.bicep`, `parameters.dev.bicepparam`, and this README + +--- + +## Design Decisions & Rationale + +- **Two tables** instead of one: the per-resource table has ~17 columns with + detailed investigation fields that don't apply to summaries. The summary + table has `SummaryLevel`, `ResourceCount` etc. that don't apply to individual + resources. Separate tables keep KQL queries cleaner and avoid wide sparse rows. +- **RunId + Month** as correlation keys: allows querying "latest run for month + X" or "all runs for month X" (useful when mid-month runs are repeated). +- **AvailabilityPct as real (-1 for N/A):** avoids a string column that would + complicate numeric KQL queries. -1 signals excluded resources. +- **Single DCR with two streams:** reduces resource count and keeps routing + in one place. The Ingestion API supports specifying the stream name per call. +- **`TimeGenerated` via DCR transform:** standard Log Analytics pattern; + the script doesn't need to supply it. +- **PowerShell-only implementation:** the ingestion feature targets the + PowerShell script (`get-availability.ps1`). The C# version is not updated + for this feature. +- **Strictly optional ingestion:** when `-DceEndpoint` and `-DcrImmutableId` are + omitted, zero ingestion code runs. No token is acquired, no payloads are built, + no REST calls are made. Console output is always produced regardless. +- **Dual execution context (interactive + Azure Function):** the script already + depends on `Az.Accounts`. Both `az login` (interactive) and managed-identity + (Azure Function) contexts expose `Get-AzAccessToken`, so the same code path + acquires the `https://monitor.azure.com` bearer token in both environments. + No conditional logic or separate auth path is needed. +- **REST API over SDK:** using the Azure Monitor Ingestion REST API directly + (with `Invoke-RestMethod`) avoids adding a PowerShell module dependency. + Authentication reuses the existing `Az.Accounts` session via `Get-AzAccessToken`. +- **Subscription-scoped Bicep with dedicated resource group:** the Bicep creates + its own resource group so the deployment is self-contained — no pre-existing + resource group or workspace is needed. The deployer only needs Contributor + on the subscription. This also keeps Get-Availability telemetry resources + isolated from other workloads. diff --git a/Setup/README.md b/Setup/README.md new file mode 100644 index 0000000..b6525d6 --- /dev/null +++ b/Setup/README.md @@ -0,0 +1,108 @@ +# Get-Availability Setup + +This directory contains the Bicep infrastructure-as-code templates for deploying the Log Analytics ingestion infrastructure used by the Get-Availability script. + +For a complete solution overview, pipeline description, and usage, see the [main README](../README.md). + +## What it deploys + +The Bicep template creates a self-contained resource group with: + +| # | Resource | Purpose | +|---|----------|---------| +| 1 | **Resource Group** | Dedicated container for all Get-Availability telemetry resources | +| 2 | **Log Analytics Workspace** | Stores availability data in custom tables; enables KQL queries and Workbooks | +| 3 | **Custom Table `GetAvailResources_CL`** | Per-resource availability detail (one row per resource per run) | +| 4 | **Custom Table `GetAvailSummary_CL`** | Aggregated summaries (per Kind+Location, per subscription, overall) | +| 5 | **Data Collection Endpoint (DCE)** | Ingestion URL for the Azure Monitor Ingestion API | +| 6 | **Data Collection Rule (DCR)** | Routes two custom streams to the corresponding tables with `TimeGenerated` injection | + +## Prerequisites + +- **Azure CLI** with Bicep support (`az bicep version`) +- **Contributor** role on the target subscription (to create the resource group and all resources) +- After deployment, the identity running the script needs **Monitoring Metrics Publisher** role on the DCR + +## Parameters + +Configured in `parameters.dev.bicepparam`: + +| Parameter | Description | Example | +|-----------|-------------|---------| +| `resourceGroupName` | Name of the dedicated resource group to create | `rg-getavail-itn-001` | +| `location` | Azure region for all resources | `italynorth` | +| `logAnalyticsWorkspaceName` | Name of the Log Analytics workspace | `log-getavail-itn-001` | +| `dataCollectionEndpointName` | Name of the Data Collection Endpoint | `dce-getavail-itn-001` | +| `dataCollectionRuleName` | Name of the Data Collection Rule | `dcr-getavail-itn-001` | + +## Deployment + +This is a **subscription-scoped** deployment (no `--resource-group` flag): + +```powershell +# Validate +az deployment sub validate --location italynorth --parameters .\parameters.dev.bicepparam + +# What-if (dry run) +az deployment sub what-if --location italynorth --parameters .\parameters.dev.bicepparam + +# Deploy +az deployment sub create --location italynorth --parameters .\parameters.dev.bicepparam +``` + +## Outputs + +After deployment, retrieve the values needed by the script: + +| Output | Description | +|--------|-------------| +| `resourceGroupName` | Created resource group name | +| `logAnalyticsWorkspaceId` | Workspace resource ID | +| `dceIngestionEndpoint` | DCE ingestion URL (pass to `-DceEndpoint`) | +| `dataCollectionRuleImmutableId` | DCR immutable ID (pass to `-DcrImmutableId`) | + +```powershell +# Retrieve outputs +$outputs = (az deployment sub show --name getavailability --query properties.outputs -o json | ConvertFrom-Json) +$outputs.dceIngestionEndpoint.value +$outputs.dataCollectionRuleImmutableId.value +``` + +## Post-Deployment: RBAC + +Grant the caller identity **Monitoring Metrics Publisher** on the DCR: + +```powershell +az role assignment create ` + --assignee ` + --role "Monitoring Metrics Publisher" ` + --scope "/subscriptions//resourceGroups//providers/Microsoft.Insights/dataCollectionRules/" +``` + +## Using with the script + +Once deployed, pass the DCE endpoint and DCR immutable ID to the script: + +```powershell +# Interactive +./get-availability.ps1 -Subscriptions 'MySub' -Month 202604 ` + -DceEndpoint 'https://dce-getavail-itn-001.italynorth-1.ingest.monitor.azure.com' ` + -DcrImmutableId 'dcr-00000000000000000000000000000000' + +# From an Azure Function (managed identity — same parameters, auth is automatic) +./get-availability.ps1 -Subscriptions 'MySub' -Month 202604 ` + -DceEndpoint $env:DCE_ENDPOINT ` + -DcrImmutableId $env:DCR_IMMUTABLE_ID +``` + +When `-DceEndpoint` and `-DcrImmutableId` are omitted, the script produces console output only (no ingestion). + +## Files + +| File | Description | +|------|-------------| +| `getavailability.bicep` | Main Bicep template (subscription-scoped, creates resource group) | +| `getavailability-resources.bicep` | Resource module (workspace, tables, DCE, DCR) | +| `parameters.dev.bicepparam` | Parameter file for dev environment | +| `certlc.bicep` | CertLC infrastructure (separate solution, not related to Get-Availability) | +| `PLAN-log-analytics-ingestion.md` | Implementation plan for the ingestion feature | \ No newline at end of file diff --git a/Setup/getavailability-resources.bicep b/Setup/getavailability-resources.bicep new file mode 100644 index 0000000..a86d07d --- /dev/null +++ b/Setup/getavailability-resources.bicep @@ -0,0 +1,202 @@ +/* + +Get-Availability — Resource module deployed into the dedicated resource group. + +Creates: Log Analytics workspace, two custom tables (GetAvailResources_CL, +GetAvailSummary_CL), Data Collection Endpoint, and Data Collection Rule. + +This file is invoked as a module from getavailability.bicep and should not +be deployed directly. + +*/ + +// ── Parameters ─────────────────────────────────────────────────────────────── + +param location string +param logAnalyticsWorkspaceName string +param dataCollectionEndpointName string +param dataCollectionRuleName string + +// ── Variables ──────────────────────────────────────────────────────────────── + +var commonTags = { + solution: 'Get-Availability' +} + +// ── Log Analytics Workspace ────────────────────────────────────────────────── + +resource logAnalyticsWorkspace 'Microsoft.OperationalInsights/workspaces@2023-09-01' = { + name: logAnalyticsWorkspaceName + location: location + properties: { + sku: { + name: 'PerGB2018' + } + retentionInDays: 30 + } + tags: commonTags +} + +// ── Custom Table: GetAvailResources_CL (per-resource detail) ───────────────── + +resource resourcesTable 'Microsoft.OperationalInsights/workspaces/tables@2022-10-01' = { + name: 'GetAvailResources_CL' + parent: logAnalyticsWorkspace + properties: { + retentionInDays: 30 + schema: { + name: 'GetAvailResources_CL' + columns: [ + { name: 'TimeGenerated', type: 'dateTime' } + { name: 'RunId', type: 'string' } + { name: 'Month', type: 'string' } + { name: 'PeriodStart', type: 'dateTime' } + { name: 'PeriodEnd', type: 'dateTime' } + { name: 'IsMonthToDate', type: 'boolean' } + { name: 'SubscriptionName', type: 'string' } + { name: 'ResourceName', type: 'string' } + { name: 'ResourceId', type: 'string' } + { name: 'ResourceGroup', type: 'string' } + { name: 'Kind', type: 'string' } + { name: 'Location', type: 'string' } + { name: 'EligibleMinutes', type: 'int' } + { name: 'AvailableMinutes', type: 'real' } + { name: 'SuspectMinutes', type: 'int' } + { name: 'ConfirmedDowntimeMinutes', type: 'int' } + { name: 'ExcusedMinutes', type: 'int' } + { name: 'UnexplainedSuspectMinutes', type: 'int' } + { name: 'AvailabilityPct', type: 'real' } + ] + } + } +} + +// ── Custom Table: GetAvailSummary_CL (aggregated summaries) ────────────────── + +resource summaryTable 'Microsoft.OperationalInsights/workspaces/tables@2022-10-01' = { + name: 'GetAvailSummary_CL' + parent: logAnalyticsWorkspace + properties: { + retentionInDays: 30 + schema: { + name: 'GetAvailSummary_CL' + columns: [ + { name: 'TimeGenerated', type: 'dateTime' } + { name: 'RunId', type: 'string' } + { name: 'Month', type: 'string' } + { name: 'PeriodStart', type: 'dateTime' } + { name: 'PeriodEnd', type: 'dateTime' } + { name: 'IsMonthToDate', type: 'boolean' } + { name: 'SummaryLevel', type: 'string' } + { name: 'SubscriptionName', type: 'string' } + { name: 'Kind', type: 'string' } + { name: 'Location', type: 'string' } + { name: 'ResourceCount', type: 'int' } + { name: 'EligibleMinutes', type: 'real' } + { name: 'AvailableMinutes', type: 'real' } + { name: 'AvailabilityPct', type: 'real' } + ] + } + } +} + +// ── Data Collection Endpoint ───────────────────────────────────────────────── + +resource dataCollectionEndpoint 'Microsoft.Insights/dataCollectionEndpoints@2023-03-11' = { + name: dataCollectionEndpointName + location: location + properties: { + networkAcls: { + publicNetworkAccess: 'Enabled' + } + } + tags: commonTags +} + +// ── Data Collection Rule (two streams, one per table) ──────────────────────── + +resource dataCollectionRule 'Microsoft.Insights/dataCollectionRules@2023-03-11' = { + name: dataCollectionRuleName + location: location + properties: { + dataCollectionEndpointId: dataCollectionEndpoint.id + + // Stream declarations — columns sent by the script (no TimeGenerated; injected by transform) + streamDeclarations: { + 'Custom-GetAvailResources_CL': { + columns: [ + { name: 'RunId', type: 'string' } + { name: 'Month', type: 'string' } + { name: 'PeriodStart', type: 'string' } + { name: 'PeriodEnd', type: 'string' } + { name: 'IsMonthToDate', type: 'boolean' } + { name: 'SubscriptionName', type: 'string' } + { name: 'ResourceName', type: 'string' } + { name: 'ResourceId', type: 'string' } + { name: 'ResourceGroup', type: 'string' } + { name: 'Kind', type: 'string' } + { name: 'Location', type: 'string' } + { name: 'EligibleMinutes', type: 'int' } + { name: 'AvailableMinutes', type: 'real' } + { name: 'SuspectMinutes', type: 'int' } + { name: 'ConfirmedDowntimeMinutes', type: 'int' } + { name: 'ExcusedMinutes', type: 'int' } + { name: 'UnexplainedSuspectMinutes', type: 'int' } + { name: 'AvailabilityPct', type: 'real' } + ] + } + 'Custom-GetAvailSummary_CL': { + columns: [ + { name: 'RunId', type: 'string' } + { name: 'Month', type: 'string' } + { name: 'PeriodStart', type: 'string' } + { name: 'PeriodEnd', type: 'string' } + { name: 'IsMonthToDate', type: 'boolean' } + { name: 'SummaryLevel', type: 'string' } + { name: 'SubscriptionName', type: 'string' } + { name: 'Kind', type: 'string' } + { name: 'Location', type: 'string' } + { name: 'ResourceCount', type: 'int' } + { name: 'EligibleMinutes', type: 'real' } + { name: 'AvailableMinutes', type: 'real' } + { name: 'AvailabilityPct', type: 'real' } + ] + } + } + + destinations: { + logAnalytics: [ + { + workspaceResourceId: logAnalyticsWorkspace.id + name: 'workspace' + } + ] + } + + dataFlows: [ + { + streams: [ 'Custom-GetAvailResources_CL' ] + destinations: [ 'workspace' ] + transformKql: 'source | extend TimeGenerated = now(), PeriodStart = todatetime(PeriodStart), PeriodEnd = todatetime(PeriodEnd)' + outputStream: 'Custom-GetAvailResources_CL' + } + { + streams: [ 'Custom-GetAvailSummary_CL' ] + destinations: [ 'workspace' ] + transformKql: 'source | extend TimeGenerated = now(), PeriodStart = todatetime(PeriodStart), PeriodEnd = todatetime(PeriodEnd)' + outputStream: 'Custom-GetAvailSummary_CL' + } + ] + } + dependsOn: [ + resourcesTable + summaryTable + ] + tags: commonTags +} + +// ── Outputs ────────────────────────────────────────────────────────────────── + +output logAnalyticsWorkspaceId string = logAnalyticsWorkspace.id +output dceIngestionEndpoint string = dataCollectionEndpoint.properties.logsIngestion.endpoint +output dataCollectionRuleImmutableId string = dataCollectionRule.properties.immutableId diff --git a/Setup/getavailability.bicep b/Setup/getavailability.bicep index e69de29..26d8cbc 100644 --- a/Setup/getavailability.bicep +++ b/Setup/getavailability.bicep @@ -0,0 +1,65 @@ +/* + +Get-Availability — Bicep template for Log Analytics ingestion infrastructure. + +Creates a dedicated resource group containing a Log Analytics workspace, +two custom tables, a Data Collection Endpoint (DCE), and a Data Collection +Rule (DCR) for ingesting Get-Availability script results. + +Validate: az deployment sub validate --location --parameters .\parameters.dev.bicepparam +What-if: az deployment sub what-if --location --parameters .\parameters.dev.bicepparam +Deploy: az deployment sub create --location --parameters .\parameters.dev.bicepparam + +*/ + +metadata name = 'Get-Availability Infrastructure' +metadata description = 'Log Analytics workspace, custom tables, DCE, and DCR for Get-Availability telemetry ingestion' + +targetScope = 'subscription' + +// ── Parameters ─────────────────────────────────────────────────────────────── + +@description('Name of the dedicated resource group to create.') +param resourceGroupName string + +@description('Azure region for all resources.') +param location string + +@description('Name of the Log Analytics workspace to create.') +param logAnalyticsWorkspaceName string + +@description('Name of the Data Collection Endpoint.') +param dataCollectionEndpointName string + +@description('Name of the Data Collection Rule.') +param dataCollectionRuleName string + +// ── Resource Group ─────────────────────────────────────────────────────────── + +resource rg 'Microsoft.Resources/resourceGroups@2024-11-01' = { + name: resourceGroupName + location: location + tags: { + solution: 'Get-Availability' + } +} + +// ── Module: all resources inside the new resource group ────────────────────── + +module resources 'getavailability-resources.bicep' = { + name: 'getavailability-resources' + scope: rg + params: { + location: location + logAnalyticsWorkspaceName: logAnalyticsWorkspaceName + dataCollectionEndpointName: dataCollectionEndpointName + dataCollectionRuleName: dataCollectionRuleName + } +} + +// ── Outputs ────────────────────────────────────────────────────────────────── + +output resourceGroupName string = rg.name +output logAnalyticsWorkspaceId string = resources.outputs.logAnalyticsWorkspaceId +output dceIngestionEndpoint string = resources.outputs.dceIngestionEndpoint +output dataCollectionRuleImmutableId string = resources.outputs.dataCollectionRuleImmutableId diff --git a/Setup/parameters.dev.bicepparam b/Setup/parameters.dev.bicepparam index c9c3898..52ce1f8 100644 --- a/Setup/parameters.dev.bicepparam +++ b/Setup/parameters.dev.bicepparam @@ -1,29 +1,7 @@ using './getavailability.bicep' -// Network Configuration -param peSubnetId = '/subscriptions/9068a229-f092-400e-8093-87e8e7d26ae1/resourceGroups/rg-alz-net-workloads-itn-001/providers/Microsoft.Network/virtualNetworks/vnet-alz-workloads-itn-001/subnets/snet-alz-pe-workloads-itn-001' -param fnSubnetId = '/subscriptions/9068a229-f092-400e-8093-87e8e7d26ae1/resourceGroups/rg-alz-net-workloads-itn-001/providers/Microsoft.Network/virtualNetworks/vnet-alz-workloads-itn-001/subnets/snet-alz-fn-workloads-itn-001' - -// DNS Configuration -param dnsZonesSubscriptionId = 'c4e6c176-bf9c-4e8c-87b2-ebdceea7085f' -param dnsZonesResourceGroupName = 'rg-alz-dns-hub-itn-001' - -// Resource Names -param storageAccountName = 'flazstcertlcitn001' -param functionAppName = 'flazfn-certlc-itn-001' -param logAnalyticsWorkspaceName = 'log-certlc-itn-001' -param applicationInsightsName = 'appi-certlc-itn-001' -param automationAccountName = 'aa-certlc-itn-001' -param hybridWorkerGroupName = 'hwg-certlc-itn-001' -param runbookName = "certlc" -param keyVaultName = 'flazkv-certlc-itn-001' -param dataCollectionEndpointName = 'dce-certlc-itn-001' -param dataCollectionRuleName = 'dcr-certlc-itn-001' - -// Automation account variables -param automationAccountVarCA = 'flazdc03.lab.formicalab.casa\\SubCA' // Name of the CA to use (for the automation account variable) -param automationAccountVarPfxRootFolder = 'C:\\PFX_Repo' // Name of the folder to use (for the automation account variable) -param automationAccountVarSmtpFrom = 'certlc@formicalab.casa' // SMTP From address to use (for the automation account variable) -param automationAccountVarSmtpServer = 'mail.smtp2go.com' // SMTP Server to use (for the automation account variable) -param automationAccountVarSmtpUser = 'certlc' // SMTP User to use (for the automation account variable) -param automationAccountVarSmtpPassword = '' // SMTP Password to use (for the automation account variable) +param resourceGroupName = 'rg-getavail-itn-001' +param location = 'italynorth' +param logAnalyticsWorkspaceName = 'log-getavail-itn-001' +param dataCollectionEndpointName = 'dce-getavail-itn-001' +param dataCollectionRuleName = 'dcr-getavail-itn-001' diff --git a/csharp/README.md b/csharp/README.md new file mode 100644 index 0000000..d951bf6 --- /dev/null +++ b/csharp/README.md @@ -0,0 +1,77 @@ +# Get-Availability — C# Version + +Native AOT implementation of Get-Availability using .NET 10. Produces a ~15 MB standalone binary with no runtime dependency. + +For the full pipeline description, classification rules, output format, and invariants shared with the PowerShell version, see the [main README](../README.md). + +## Prerequisites + +| Requirement | Detail | +|---|---| +| .NET SDK | 10.0 or later (build from source only) | +| Azure auth | `az login` or any method supported by `DefaultAzureCredential` | + +The published binary (`GetAvailability.exe`) requires no .NET runtime — it is a Native AOT self-contained executable. + +If Azure authentication fails, the tool prints the SDK exception message directly. Re-run `az login` to fix. + +## Parameters + +| Option | Short | Default | Description | +|---|---|---|---| +| `--subscriptions` | `-s` | *(required)* | One or more Azure subscription display names | +| `--month` | `-m` | *(required)* | Observation month in UTC, format `YYYYMM` | +| `--kinds` | `-k` | `vm sql storage webapp` | Resource kinds to process | +| `--resource` | `-r` | *(all)* | Filter to a single resource name | +| `--parallelism` | `-p` | *(auto)* | Max concurrent API calls (scales to CPU cores) | +| `--activity-grace-minutes` | `-g` | `10` | Post-operation grace window for Activity Log lifecycle events | +| `--batch` | `-b` | off | Use the regional Metrics Batch API instead of per-resource calls | +| `--batch-size` | | `10` | Max resources per batch call (1–50); implies `--batch` | +| `--workspace` | `-w` | *(none)* | Log Analytics workspace ID (GUID). Fetches Activity Log via bulk KQL; Resource Health uses hybrid approach (KQL for older + REST for last ~30 days) | +| `--version` | `-v` | | Print version and exit | + +## Build + +```bash +cd csharp/GetAvailability + +# Debug (JIT, for development) +dotnet build + +# Release Native AOT binary +dotnet publish -c Release -r win-x64 # output in bin/Release/net10.0/win-x64/publish/ +``` + +## Examples + +```bash +# Single subscription +./GetAvailability --subscriptions Contoso-Production --month 202603 + +# Multiple subscriptions, filtered by kind +./GetAvailability --subscriptions Contoso-Development Contoso-Production --month 202603 --kinds vm sql + +# Single resource (SQL database by server/database name) +./GetAvailability --subscriptions Contoso-Development --month 202603 --resource sqlserver01/sqldb01 + +# Batch API with custom batch size +./GetAvailability --subscriptions Contoso-Production Contoso-Development --month 202603 --batch-size 20 + +# Use Log Analytics for Activity Log + Resource Health (faster, extended retention) +./GetAvailability --subscriptions Contoso-Production --month 202603 --workspace b233a4b7-3c43-433c-ac60-1f6ff217ddd4 + +# Run directly without publishing +cd csharp/GetAvailability +dotnet run -- --subscriptions Contoso-Production --month 202603 +``` + +## Implementation Notes + +- **`Parallel.ForEachAsync`** for concurrent metric, Activity Log, and Resource Health queries with configurable parallelism. +- **`System.Text.Json`** for AOT-safe, efficient JSON parsing. +- **Native AOT** — ~15 MB standalone binary, no .NET runtime required. +- **O(1) JSON property access** — `TryGetProperty` hash lookup instead of `EnumerateObject` linear scan (~44k calls per resource per month). +- **Ticks-based metric keying** — `long` instead of `DateTime` for zero-allocation per data point. +- **HashSet-based interval containment** — suspect-minute classification pre-expands intervals into `HashSet` tick sets for O(1) lookups instead of linear scans. + +> **Note:** The C# version does not currently support the Log Analytics ingestion feature (`-DceEndpoint`/`-DcrImmutableId`). Use the PowerShell version for that capability. diff --git a/get-availability.ps1 b/get-availability.ps1 index 15eb0f7..ff3516d 100644 --- a/get-availability.ps1 +++ b/get-availability.ps1 @@ -78,6 +78,18 @@ across the full observation window. Requires the workspace to receive Activity Log diagnostic settings from the target subscriptions. +.PARAMETER DceEndpoint + Data Collection Endpoint ingestion URL. When both -DceEndpoint and + -DcrImmutableId are provided, results are sent to Log Analytics custom + tables in addition to the normal console output. Obtain the URL from + the Bicep deployment output 'dceIngestionEndpoint'. Works in both + interactive (az login / Connect-AzAccount) and Azure Function + (managed identity) execution contexts. + +.PARAMETER DcrImmutableId + Data Collection Rule immutable ID. Required together with -DceEndpoint. + Obtain from the Bicep deployment output 'dataCollectionRuleImmutableId'. + .EXAMPLE ./get-availability.ps1 -Subscriptions 'MySubscription' -Month 202506 @@ -89,6 +101,9 @@ .EXAMPLE ./get-availability.ps1 -Subscriptions 'MySub' -Month 202506 -Workspace 'b233a4b7-3c43-433c-ac60-1f6ff217ddd4' + +.EXAMPLE + ./get-availability.ps1 -Subscriptions 'MySub' -Month 202506 -DceEndpoint 'https://dce-getavail-itn-001.italynorth-1.ingest.monitor.azure.com' -DcrImmutableId 'dcr-00000000000000000000000000000000' #> [CmdletBinding(DefaultParameterSetName = 'Run')] @@ -127,6 +142,14 @@ param( [ValidatePattern('^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$')] [string]$Workspace, + [Parameter(ParameterSetName = 'Run')] + [ValidateNotNullOrEmpty()] + [string]$DceEndpoint, + + [Parameter(ParameterSetName = 'Run')] + [ValidateNotNullOrEmpty()] + [string]$DcrImmutableId, + [Parameter(Mandatory, ParameterSetName = 'ShowVersion')] [switch]$Version ) @@ -141,6 +164,78 @@ if ($Version) { return } +# Validate paired ingestion parameters +$sendToLogAnalytics = $false +if ($DceEndpoint -and $DcrImmutableId) { + $sendToLogAnalytics = $true +} elseif ($DceEndpoint -or $DcrImmutableId) { + throw '-DceEndpoint and -DcrImmutableId must both be provided together.' +} + +# ── Log Analytics Ingestion ─────────────────────────────────────────────────── + +## Sends an array of objects to a Log Analytics custom table via the Azure +## Monitor Ingestion REST API. Handles gzip compression and batching for +## the 1 MB per-call payload limit. +function Send-ToLogAnalytics { + param( + [string]$Endpoint, + [string]$RuleId, + [string]$StreamName, + [string]$Token, + [object[]]$Payload + ) + + $uri = "$Endpoint/dataCollectionRules/$RuleId/streams/${StreamName}?api-version=2023-01-01" + + # Split into batches if payload is large (target < 900 KB compressed) + $batchSize = $Payload.Count + $batches = @() + if ($Payload.Count -gt 0) { + # Start with a single batch; split only if the compressed size exceeds the limit + $json = $Payload | ConvertTo-Json -Depth 5 -Compress -AsArray + $jsonBytes = [System.Text.Encoding]::UTF8.GetBytes($json) + $ms = [System.IO.MemoryStream]::new() + $gz = [System.IO.Compression.GZipStream]::new($ms, [System.IO.Compression.CompressionLevel]::Optimal) + $gz.Write($jsonBytes, 0, $jsonBytes.Length) + $gz.Dispose() + $compressed = $ms.ToArray() + $ms.Dispose() + + if ($compressed.Length -lt 900KB) { + $batches = @(, $compressed) + } else { + # Split into smaller chunks and compress each + $chunkSize = [math]::Max(1, [math]::Floor($Payload.Count / [math]::Ceiling($compressed.Length / 900KB))) + for ($i = 0; $i -lt $Payload.Count; $i += $chunkSize) { + $chunk = @($Payload[$i..([math]::Min($i + $chunkSize - 1, $Payload.Count - 1))]) + $cJson = $chunk | ConvertTo-Json -Depth 5 -Compress -AsArray + $cBytes = [System.Text.Encoding]::UTF8.GetBytes($cJson) + $cMs = [System.IO.MemoryStream]::new() + $cGz = [System.IO.Compression.GZipStream]::new($cMs, [System.IO.Compression.CompressionLevel]::Optimal) + $cGz.Write($cBytes, 0, $cBytes.Length) + $cGz.Dispose() + $batches += , $cMs.ToArray() + $cMs.Dispose() + } + } + } + + $batchNum = 0 + foreach ($body in $batches) { + $batchNum++ + $headers = @{ + 'Authorization' = "Bearer $Token" + 'Content-Type' = 'application/json' + 'Content-Encoding' = 'gzip' + } + $response = Invoke-WebRequest -Uri $uri -Method Post -Headers $headers -Body $body -UseBasicParsing -ErrorAction Stop + if ($response.StatusCode -notin 200, 204) { + throw "Ingestion failed (batch $batchNum/$($batches.Count)): HTTP $($response.StatusCode) — $($response.Content)" + } + } +} + # ── Observation window ──────────────────────────────────────────────────────── function Resolve-ObservationWindow([string]$MonthParam) { @@ -2457,5 +2552,118 @@ $sorted = @($eligByRes.Values | Write-ResultsTable $sorted Write-SubscriptionSummaries $sorted +# Step 9: Optional Log Analytics ingestion +if ($sendToLogAnalytics) { + Write-Host -NoNewline 'Sending results to Log Analytics... ' + + # Acquire Azure Monitor ingestion token (same pattern as ARM token) + $rawMonitor = (Get-AzAccessToken -ResourceUrl 'https://monitor.azure.com').Token + $monitorToken = ($rawMonitor -is [securestring]) ? ($rawMonitor | ConvertFrom-SecureString -AsPlainText) : [string]$rawMonitor + $rawMonitor = $null + + $runId = [guid]::NewGuid().ToString() + $normalizedMonth = $window.NormalizedMonth + $isMonthToDate = $window.IsMonthToDate + $periodStartIso = $utcStart.ToString('o') + $periodEndIso = $utcEnd.ToString('o') + + # Build per-resource detail payload + $resourcePayload = @(foreach ($elig in $eligByRes.Values) { + @{ + RunId = $runId + Month = $normalizedMonth + PeriodStart = $periodStartIso + PeriodEnd = $periodEndIso + IsMonthToDate = $isMonthToDate + SubscriptionName = $elig.SubscriptionName + ResourceName = $elig.Name + ResourceId = $elig.ResourceId + ResourceGroup = $elig.ResourceGroupName + Kind = $elig.Kind + Location = $elig.Location + EligibleMinutes = $elig.EligibleMinutes + AvailableMinutes = $elig.AvailableMinutes + SuspectMinutes = $elig.SuspectMinutes + ConfirmedDowntimeMinutes = $elig.ConfirmedDowntimeMinutes + ExcusedMinutes = $elig.ExcusedMinutes + UnexplainedSuspectMinutes = $elig.UnexplainedSuspectMinutes + AvailabilityPct = if ($elig.AvailabilityPct -eq 'N/A') { -1 } else { [double]$elig.AvailabilityPct } + } + }) + + Send-ToLogAnalytics -Endpoint $DceEndpoint -RuleId $DcrImmutableId ` + -StreamName 'Custom-GetAvailResources_CL' -Token $monitorToken -Payload $resourcePayload + + # Build summary payload + $eligible = @($sorted | Where-Object { $_.AvailabilityPct -ne 'N/A' }) + $summaryPayload = [System.Collections.Generic.List[hashtable]]::new() + + $commonSummary = @{ + RunId = $runId + Month = $normalizedMonth + PeriodStart = $periodStartIso + PeriodEnd = $periodEndIso + IsMonthToDate = $isMonthToDate + } + + foreach ($subGroup in ($eligible | Group-Object SubscriptionName | Sort-Object Name)) { + foreach ($g in ($subGroup.Group | Group-Object { "$($_.Kind)|$($_.Location)" } | Sort-Object Name)) { + $items = @($g.Group) + $a = ($items | Measure-Object AvailableMinutes -Sum).Sum + $e = ($items | Measure-Object EligibleMinutes -Sum).Sum + $pct = if ($e -gt 0) { [math]::Round($a / $e * 100, 5) } else { 0 } + $summaryPayload.Add(($commonSummary + @{ + SummaryLevel = 'KindLocation' + SubscriptionName = $subGroup.Name + Kind = $items[0].Kind + Location = $items[0].Location + ResourceCount = $items.Count + EligibleMinutes = [math]::Round($e, 2) + AvailableMinutes = [math]::Round($a, 2) + AvailabilityPct = $pct + })) + } + $ta = ($subGroup.Group | Measure-Object AvailableMinutes -Sum).Sum + $te = ($subGroup.Group | Measure-Object EligibleMinutes -Sum).Sum + $tpct = if ($te -gt 0) { [math]::Round($ta / $te * 100, 5) } else { 0 } + $summaryPayload.Add(($commonSummary + @{ + SummaryLevel = 'SubscriptionTotal' + SubscriptionName = $subGroup.Name + Kind = '' + Location = '' + ResourceCount = $subGroup.Count + EligibleMinutes = [math]::Round($te, 2) + AvailableMinutes = [math]::Round($ta, 2) + AvailabilityPct = $tpct + })) + } + + # Cross-subscription overall (only if >1 subscription) + $subs = @($eligible | ForEach-Object { $_.SubscriptionName } | Select-Object -Unique) + if ($subs.Count -gt 1 -and $eligible.Count -gt 0) { + $oa = ($eligible | Measure-Object AvailableMinutes -Sum).Sum + $oe = ($eligible | Measure-Object EligibleMinutes -Sum).Sum + $opct = if ($oe -gt 0) { [math]::Round($oa / $oe * 100, 5) } else { 0 } + $summaryPayload.Add(($commonSummary + @{ + SummaryLevel = 'Overall' + SubscriptionName = '' + Kind = '' + Location = '' + ResourceCount = $eligible.Count + EligibleMinutes = [math]::Round($oe, 2) + AvailableMinutes = [math]::Round($oa, 2) + AvailabilityPct = $opct + })) + } + + if ($summaryPayload.Count -gt 0) { + Send-ToLogAnalytics -Endpoint $DceEndpoint -RuleId $DcrImmutableId ` + -StreamName 'Custom-GetAvailSummary_CL' -Token $monitorToken -Payload $summaryPayload.ToArray() + } + + $monitorToken = $null + Write-Host "OK (RunId: $runId, $($resourcePayload.Count) resource rows, $($summaryPayload.Count) summary rows)" +} + $sw.Stop() Write-Host "Completed in $($sw.Elapsed.ToString('hh\:mm\:ss\.ff'))" From 1bec05d4ac9c0763850064fb8c0e692ba64572b6 Mon Sep 17 00:00:00 2001 From: Marcello Formica Date: Mon, 27 Apr 2026 13:36:39 +0200 Subject: [PATCH 03/10] refactor: merge Bicep into single resource-group scoped file - Combine getavailability-resources.bicep into getavailability.bicep - Switch from subscription-scoped to resource-group scoped deployment - Remove resourceGroupName param (RG created separately via az group create) - Location defaults to resourceGroup().location - Delete getavailability-resources.bicep module - Update parameters.dev.bicepparam and Setup/README.md accordingly --- Setup/README.md | 37 +++-- Setup/getavailability-resources.bicep | 202 ------------------------ Setup/getavailability.bicep | 211 ++++++++++++++++++++++---- Setup/parameters.dev.bicepparam | 2 - 4 files changed, 198 insertions(+), 254 deletions(-) delete mode 100644 Setup/getavailability-resources.bicep diff --git a/Setup/README.md b/Setup/README.md index b6525d6..d7d3791 100644 --- a/Setup/README.md +++ b/Setup/README.md @@ -6,21 +6,20 @@ For a complete solution overview, pipeline description, and usage, see the [main ## What it deploys -The Bicep template creates a self-contained resource group with: +The Bicep template deploys the following resources into the target resource group: | # | Resource | Purpose | -|---|----------|---------| -| 1 | **Resource Group** | Dedicated container for all Get-Availability telemetry resources | -| 2 | **Log Analytics Workspace** | Stores availability data in custom tables; enables KQL queries and Workbooks | -| 3 | **Custom Table `GetAvailResources_CL`** | Per-resource availability detail (one row per resource per run) | -| 4 | **Custom Table `GetAvailSummary_CL`** | Aggregated summaries (per Kind+Location, per subscription, overall) | -| 5 | **Data Collection Endpoint (DCE)** | Ingestion URL for the Azure Monitor Ingestion API | -| 6 | **Data Collection Rule (DCR)** | Routes two custom streams to the corresponding tables with `TimeGenerated` injection | +|---|----------|--------| +| 1 | **Log Analytics Workspace** | Stores availability data in custom tables; enables KQL queries and Workbooks | +| 2 | **Custom Table `GetAvailResources_CL`** | Per-resource availability detail (one row per resource per run) | +| 3 | **Custom Table `GetAvailSummary_CL`** | Aggregated summaries (per Kind+Location, per subscription, overall) | +| 4 | **Data Collection Endpoint (DCE)** | Ingestion URL for the Azure Monitor Ingestion API | +| 5 | **Data Collection Rule (DCR)** | Routes two custom streams to the corresponding tables with `TimeGenerated` injection | ## Prerequisites - **Azure CLI** with Bicep support (`az bicep version`) -- **Contributor** role on the target subscription (to create the resource group and all resources) +- **Contributor** role on the target resource group - After deployment, the identity running the script needs **Monitoring Metrics Publisher** role on the DCR ## Parameters @@ -29,25 +28,27 @@ Configured in `parameters.dev.bicepparam`: | Parameter | Description | Example | |-----------|-------------|---------| -| `resourceGroupName` | Name of the dedicated resource group to create | `rg-getavail-itn-001` | -| `location` | Azure region for all resources | `italynorth` | +| `location` | Azure region for all resources (defaults to resource group location) | `italynorth` | | `logAnalyticsWorkspaceName` | Name of the Log Analytics workspace | `log-getavail-itn-001` | | `dataCollectionEndpointName` | Name of the Data Collection Endpoint | `dce-getavail-itn-001` | | `dataCollectionRuleName` | Name of the Data Collection Rule | `dcr-getavail-itn-001` | ## Deployment -This is a **subscription-scoped** deployment (no `--resource-group` flag): +This is a **resource-group scoped** deployment. Create the resource group first, then deploy: ```powershell +# Create resource group (one-time) +az group create --name rg-getavail-itn-001 --location italynorth --tags solution=Get-Availability + # Validate -az deployment sub validate --location italynorth --parameters .\parameters.dev.bicepparam +az deployment group validate --resource-group rg-getavail-itn-001 --parameters .\parameters.dev.bicepparam # What-if (dry run) -az deployment sub what-if --location italynorth --parameters .\parameters.dev.bicepparam +az deployment group what-if --resource-group rg-getavail-itn-001 --parameters .\parameters.dev.bicepparam # Deploy -az deployment sub create --location italynorth --parameters .\parameters.dev.bicepparam +az deployment group create --resource-group rg-getavail-itn-001 --parameters .\parameters.dev.bicepparam ``` ## Outputs @@ -56,14 +57,13 @@ After deployment, retrieve the values needed by the script: | Output | Description | |--------|-------------| -| `resourceGroupName` | Created resource group name | | `logAnalyticsWorkspaceId` | Workspace resource ID | | `dceIngestionEndpoint` | DCE ingestion URL (pass to `-DceEndpoint`) | | `dataCollectionRuleImmutableId` | DCR immutable ID (pass to `-DcrImmutableId`) | ```powershell # Retrieve outputs -$outputs = (az deployment sub show --name getavailability --query properties.outputs -o json | ConvertFrom-Json) +$outputs = (az deployment group show --resource-group rg-getavail-itn-001 --name getavailability --query properties.outputs -o json | ConvertFrom-Json) $outputs.dceIngestionEndpoint.value $outputs.dataCollectionRuleImmutableId.value ``` @@ -101,8 +101,7 @@ When `-DceEndpoint` and `-DcrImmutableId` are omitted, the script produces conso | File | Description | |------|-------------| -| `getavailability.bicep` | Main Bicep template (subscription-scoped, creates resource group) | -| `getavailability-resources.bicep` | Resource module (workspace, tables, DCE, DCR) | +| `getavailability.bicep` | Bicep template (resource-group scoped: workspace, tables, DCE, DCR) | | `parameters.dev.bicepparam` | Parameter file for dev environment | | `certlc.bicep` | CertLC infrastructure (separate solution, not related to Get-Availability) | | `PLAN-log-analytics-ingestion.md` | Implementation plan for the ingestion feature | \ No newline at end of file diff --git a/Setup/getavailability-resources.bicep b/Setup/getavailability-resources.bicep deleted file mode 100644 index a86d07d..0000000 --- a/Setup/getavailability-resources.bicep +++ /dev/null @@ -1,202 +0,0 @@ -/* - -Get-Availability — Resource module deployed into the dedicated resource group. - -Creates: Log Analytics workspace, two custom tables (GetAvailResources_CL, -GetAvailSummary_CL), Data Collection Endpoint, and Data Collection Rule. - -This file is invoked as a module from getavailability.bicep and should not -be deployed directly. - -*/ - -// ── Parameters ─────────────────────────────────────────────────────────────── - -param location string -param logAnalyticsWorkspaceName string -param dataCollectionEndpointName string -param dataCollectionRuleName string - -// ── Variables ──────────────────────────────────────────────────────────────── - -var commonTags = { - solution: 'Get-Availability' -} - -// ── Log Analytics Workspace ────────────────────────────────────────────────── - -resource logAnalyticsWorkspace 'Microsoft.OperationalInsights/workspaces@2023-09-01' = { - name: logAnalyticsWorkspaceName - location: location - properties: { - sku: { - name: 'PerGB2018' - } - retentionInDays: 30 - } - tags: commonTags -} - -// ── Custom Table: GetAvailResources_CL (per-resource detail) ───────────────── - -resource resourcesTable 'Microsoft.OperationalInsights/workspaces/tables@2022-10-01' = { - name: 'GetAvailResources_CL' - parent: logAnalyticsWorkspace - properties: { - retentionInDays: 30 - schema: { - name: 'GetAvailResources_CL' - columns: [ - { name: 'TimeGenerated', type: 'dateTime' } - { name: 'RunId', type: 'string' } - { name: 'Month', type: 'string' } - { name: 'PeriodStart', type: 'dateTime' } - { name: 'PeriodEnd', type: 'dateTime' } - { name: 'IsMonthToDate', type: 'boolean' } - { name: 'SubscriptionName', type: 'string' } - { name: 'ResourceName', type: 'string' } - { name: 'ResourceId', type: 'string' } - { name: 'ResourceGroup', type: 'string' } - { name: 'Kind', type: 'string' } - { name: 'Location', type: 'string' } - { name: 'EligibleMinutes', type: 'int' } - { name: 'AvailableMinutes', type: 'real' } - { name: 'SuspectMinutes', type: 'int' } - { name: 'ConfirmedDowntimeMinutes', type: 'int' } - { name: 'ExcusedMinutes', type: 'int' } - { name: 'UnexplainedSuspectMinutes', type: 'int' } - { name: 'AvailabilityPct', type: 'real' } - ] - } - } -} - -// ── Custom Table: GetAvailSummary_CL (aggregated summaries) ────────────────── - -resource summaryTable 'Microsoft.OperationalInsights/workspaces/tables@2022-10-01' = { - name: 'GetAvailSummary_CL' - parent: logAnalyticsWorkspace - properties: { - retentionInDays: 30 - schema: { - name: 'GetAvailSummary_CL' - columns: [ - { name: 'TimeGenerated', type: 'dateTime' } - { name: 'RunId', type: 'string' } - { name: 'Month', type: 'string' } - { name: 'PeriodStart', type: 'dateTime' } - { name: 'PeriodEnd', type: 'dateTime' } - { name: 'IsMonthToDate', type: 'boolean' } - { name: 'SummaryLevel', type: 'string' } - { name: 'SubscriptionName', type: 'string' } - { name: 'Kind', type: 'string' } - { name: 'Location', type: 'string' } - { name: 'ResourceCount', type: 'int' } - { name: 'EligibleMinutes', type: 'real' } - { name: 'AvailableMinutes', type: 'real' } - { name: 'AvailabilityPct', type: 'real' } - ] - } - } -} - -// ── Data Collection Endpoint ───────────────────────────────────────────────── - -resource dataCollectionEndpoint 'Microsoft.Insights/dataCollectionEndpoints@2023-03-11' = { - name: dataCollectionEndpointName - location: location - properties: { - networkAcls: { - publicNetworkAccess: 'Enabled' - } - } - tags: commonTags -} - -// ── Data Collection Rule (two streams, one per table) ──────────────────────── - -resource dataCollectionRule 'Microsoft.Insights/dataCollectionRules@2023-03-11' = { - name: dataCollectionRuleName - location: location - properties: { - dataCollectionEndpointId: dataCollectionEndpoint.id - - // Stream declarations — columns sent by the script (no TimeGenerated; injected by transform) - streamDeclarations: { - 'Custom-GetAvailResources_CL': { - columns: [ - { name: 'RunId', type: 'string' } - { name: 'Month', type: 'string' } - { name: 'PeriodStart', type: 'string' } - { name: 'PeriodEnd', type: 'string' } - { name: 'IsMonthToDate', type: 'boolean' } - { name: 'SubscriptionName', type: 'string' } - { name: 'ResourceName', type: 'string' } - { name: 'ResourceId', type: 'string' } - { name: 'ResourceGroup', type: 'string' } - { name: 'Kind', type: 'string' } - { name: 'Location', type: 'string' } - { name: 'EligibleMinutes', type: 'int' } - { name: 'AvailableMinutes', type: 'real' } - { name: 'SuspectMinutes', type: 'int' } - { name: 'ConfirmedDowntimeMinutes', type: 'int' } - { name: 'ExcusedMinutes', type: 'int' } - { name: 'UnexplainedSuspectMinutes', type: 'int' } - { name: 'AvailabilityPct', type: 'real' } - ] - } - 'Custom-GetAvailSummary_CL': { - columns: [ - { name: 'RunId', type: 'string' } - { name: 'Month', type: 'string' } - { name: 'PeriodStart', type: 'string' } - { name: 'PeriodEnd', type: 'string' } - { name: 'IsMonthToDate', type: 'boolean' } - { name: 'SummaryLevel', type: 'string' } - { name: 'SubscriptionName', type: 'string' } - { name: 'Kind', type: 'string' } - { name: 'Location', type: 'string' } - { name: 'ResourceCount', type: 'int' } - { name: 'EligibleMinutes', type: 'real' } - { name: 'AvailableMinutes', type: 'real' } - { name: 'AvailabilityPct', type: 'real' } - ] - } - } - - destinations: { - logAnalytics: [ - { - workspaceResourceId: logAnalyticsWorkspace.id - name: 'workspace' - } - ] - } - - dataFlows: [ - { - streams: [ 'Custom-GetAvailResources_CL' ] - destinations: [ 'workspace' ] - transformKql: 'source | extend TimeGenerated = now(), PeriodStart = todatetime(PeriodStart), PeriodEnd = todatetime(PeriodEnd)' - outputStream: 'Custom-GetAvailResources_CL' - } - { - streams: [ 'Custom-GetAvailSummary_CL' ] - destinations: [ 'workspace' ] - transformKql: 'source | extend TimeGenerated = now(), PeriodStart = todatetime(PeriodStart), PeriodEnd = todatetime(PeriodEnd)' - outputStream: 'Custom-GetAvailSummary_CL' - } - ] - } - dependsOn: [ - resourcesTable - summaryTable - ] - tags: commonTags -} - -// ── Outputs ────────────────────────────────────────────────────────────────── - -output logAnalyticsWorkspaceId string = logAnalyticsWorkspace.id -output dceIngestionEndpoint string = dataCollectionEndpoint.properties.logsIngestion.endpoint -output dataCollectionRuleImmutableId string = dataCollectionRule.properties.immutableId diff --git a/Setup/getavailability.bicep b/Setup/getavailability.bicep index 26d8cbc..5aaba39 100644 --- a/Setup/getavailability.bicep +++ b/Setup/getavailability.bicep @@ -2,28 +2,23 @@ Get-Availability — Bicep template for Log Analytics ingestion infrastructure. -Creates a dedicated resource group containing a Log Analytics workspace, -two custom tables, a Data Collection Endpoint (DCE), and a Data Collection -Rule (DCR) for ingesting Get-Availability script results. +Creates a Log Analytics workspace, two custom tables, a Data Collection +Endpoint (DCE), and a Data Collection Rule (DCR) for ingesting +Get-Availability script results. -Validate: az deployment sub validate --location --parameters .\parameters.dev.bicepparam -What-if: az deployment sub what-if --location --parameters .\parameters.dev.bicepparam -Deploy: az deployment sub create --location --parameters .\parameters.dev.bicepparam +Validate: az deployment group validate --resource-group --parameters .\parameters.dev.bicepparam +What-if: az deployment group what-if --resource-group --parameters .\parameters.dev.bicepparam +Deploy: az deployment group create --resource-group --parameters .\parameters.dev.bicepparam */ metadata name = 'Get-Availability Infrastructure' metadata description = 'Log Analytics workspace, custom tables, DCE, and DCR for Get-Availability telemetry ingestion' -targetScope = 'subscription' - // ── Parameters ─────────────────────────────────────────────────────────────── -@description('Name of the dedicated resource group to create.') -param resourceGroupName string - -@description('Azure region for all resources.') -param location string +@description('Azure region for all resources. Defaults to the resource group location.') +param location string = resourceGroup().location @description('Name of the Log Analytics workspace to create.') param logAnalyticsWorkspaceName string @@ -34,32 +29,186 @@ param dataCollectionEndpointName string @description('Name of the Data Collection Rule.') param dataCollectionRuleName string -// ── Resource Group ─────────────────────────────────────────────────────────── +// ── Variables ──────────────────────────────────────────────────────────────── + +var commonTags = { + solution: 'Get-Availability' +} + +// ── Log Analytics Workspace ────────────────────────────────────────────────── -resource rg 'Microsoft.Resources/resourceGroups@2024-11-01' = { - name: resourceGroupName +resource logAnalyticsWorkspace 'Microsoft.OperationalInsights/workspaces@2023-09-01' = { + name: logAnalyticsWorkspaceName location: location - tags: { - solution: 'Get-Availability' + properties: { + sku: { + name: 'PerGB2018' + } + retentionInDays: 30 } + tags: commonTags } -// ── Module: all resources inside the new resource group ────────────────────── +// ── Custom Table: GetAvailResources_CL (per-resource detail) ───────────────── + +resource resourcesTable 'Microsoft.OperationalInsights/workspaces/tables@2022-10-01' = { + name: 'GetAvailResources_CL' + parent: logAnalyticsWorkspace + properties: { + retentionInDays: 30 + schema: { + name: 'GetAvailResources_CL' + columns: [ + { name: 'TimeGenerated', type: 'dateTime' } + { name: 'RunId', type: 'string' } + { name: 'Month', type: 'string' } + { name: 'PeriodStart', type: 'dateTime' } + { name: 'PeriodEnd', type: 'dateTime' } + { name: 'IsMonthToDate', type: 'boolean' } + { name: 'SubscriptionName', type: 'string' } + { name: 'ResourceName', type: 'string' } + { name: 'ResourceId', type: 'string' } + { name: 'ResourceGroup', type: 'string' } + { name: 'Kind', type: 'string' } + { name: 'Location', type: 'string' } + { name: 'EligibleMinutes', type: 'int' } + { name: 'AvailableMinutes', type: 'real' } + { name: 'SuspectMinutes', type: 'int' } + { name: 'ConfirmedDowntimeMinutes', type: 'int' } + { name: 'ExcusedMinutes', type: 'int' } + { name: 'UnexplainedSuspectMinutes', type: 'int' } + { name: 'AvailabilityPct', type: 'real' } + ] + } + } +} -module resources 'getavailability-resources.bicep' = { - name: 'getavailability-resources' - scope: rg - params: { - location: location - logAnalyticsWorkspaceName: logAnalyticsWorkspaceName - dataCollectionEndpointName: dataCollectionEndpointName - dataCollectionRuleName: dataCollectionRuleName +// ── Custom Table: GetAvailSummary_CL (aggregated summaries) ────────────────── + +resource summaryTable 'Microsoft.OperationalInsights/workspaces/tables@2022-10-01' = { + name: 'GetAvailSummary_CL' + parent: logAnalyticsWorkspace + properties: { + retentionInDays: 30 + schema: { + name: 'GetAvailSummary_CL' + columns: [ + { name: 'TimeGenerated', type: 'dateTime' } + { name: 'RunId', type: 'string' } + { name: 'Month', type: 'string' } + { name: 'PeriodStart', type: 'dateTime' } + { name: 'PeriodEnd', type: 'dateTime' } + { name: 'IsMonthToDate', type: 'boolean' } + { name: 'SummaryLevel', type: 'string' } + { name: 'SubscriptionName', type: 'string' } + { name: 'Kind', type: 'string' } + { name: 'Location', type: 'string' } + { name: 'ResourceCount', type: 'int' } + { name: 'EligibleMinutes', type: 'real' } + { name: 'AvailableMinutes', type: 'real' } + { name: 'AvailabilityPct', type: 'real' } + ] + } + } +} + +// ── Data Collection Endpoint ───────────────────────────────────────────────── + +resource dataCollectionEndpoint 'Microsoft.Insights/dataCollectionEndpoints@2023-03-11' = { + name: dataCollectionEndpointName + location: location + properties: { + networkAcls: { + publicNetworkAccess: 'Enabled' + } + } + tags: commonTags +} + +// ── Data Collection Rule (two streams, one per table) ──────────────────────── + +resource dataCollectionRule 'Microsoft.Insights/dataCollectionRules@2023-03-11' = { + name: dataCollectionRuleName + location: location + properties: { + dataCollectionEndpointId: dataCollectionEndpoint.id + + // Stream declarations — columns sent by the script (no TimeGenerated; injected by transform) + streamDeclarations: { + 'Custom-GetAvailResources_CL': { + columns: [ + { name: 'RunId', type: 'string' } + { name: 'Month', type: 'string' } + { name: 'PeriodStart', type: 'string' } + { name: 'PeriodEnd', type: 'string' } + { name: 'IsMonthToDate', type: 'boolean' } + { name: 'SubscriptionName', type: 'string' } + { name: 'ResourceName', type: 'string' } + { name: 'ResourceId', type: 'string' } + { name: 'ResourceGroup', type: 'string' } + { name: 'Kind', type: 'string' } + { name: 'Location', type: 'string' } + { name: 'EligibleMinutes', type: 'int' } + { name: 'AvailableMinutes', type: 'real' } + { name: 'SuspectMinutes', type: 'int' } + { name: 'ConfirmedDowntimeMinutes', type: 'int' } + { name: 'ExcusedMinutes', type: 'int' } + { name: 'UnexplainedSuspectMinutes', type: 'int' } + { name: 'AvailabilityPct', type: 'real' } + ] + } + 'Custom-GetAvailSummary_CL': { + columns: [ + { name: 'RunId', type: 'string' } + { name: 'Month', type: 'string' } + { name: 'PeriodStart', type: 'string' } + { name: 'PeriodEnd', type: 'string' } + { name: 'IsMonthToDate', type: 'boolean' } + { name: 'SummaryLevel', type: 'string' } + { name: 'SubscriptionName', type: 'string' } + { name: 'Kind', type: 'string' } + { name: 'Location', type: 'string' } + { name: 'ResourceCount', type: 'int' } + { name: 'EligibleMinutes', type: 'real' } + { name: 'AvailableMinutes', type: 'real' } + { name: 'AvailabilityPct', type: 'real' } + ] + } + } + + destinations: { + logAnalytics: [ + { + workspaceResourceId: logAnalyticsWorkspace.id + name: 'workspace' + } + ] + } + + dataFlows: [ + { + streams: [ 'Custom-GetAvailResources_CL' ] + destinations: [ 'workspace' ] + transformKql: 'source | extend TimeGenerated = now(), PeriodStart = todatetime(PeriodStart), PeriodEnd = todatetime(PeriodEnd)' + outputStream: 'Custom-GetAvailResources_CL' + } + { + streams: [ 'Custom-GetAvailSummary_CL' ] + destinations: [ 'workspace' ] + transformKql: 'source | extend TimeGenerated = now(), PeriodStart = todatetime(PeriodStart), PeriodEnd = todatetime(PeriodEnd)' + outputStream: 'Custom-GetAvailSummary_CL' + } + ] } + dependsOn: [ + resourcesTable + summaryTable + ] + tags: commonTags } // ── Outputs ────────────────────────────────────────────────────────────────── -output resourceGroupName string = rg.name -output logAnalyticsWorkspaceId string = resources.outputs.logAnalyticsWorkspaceId -output dceIngestionEndpoint string = resources.outputs.dceIngestionEndpoint -output dataCollectionRuleImmutableId string = resources.outputs.dataCollectionRuleImmutableId +output logAnalyticsWorkspaceId string = logAnalyticsWorkspace.id +output dceIngestionEndpoint string = dataCollectionEndpoint.properties.logsIngestion.endpoint +output dataCollectionRuleImmutableId string = dataCollectionRule.properties.immutableId diff --git a/Setup/parameters.dev.bicepparam b/Setup/parameters.dev.bicepparam index 52ce1f8..4b8d186 100644 --- a/Setup/parameters.dev.bicepparam +++ b/Setup/parameters.dev.bicepparam @@ -1,7 +1,5 @@ using './getavailability.bicep' -param resourceGroupName = 'rg-getavail-itn-001' -param location = 'italynorth' param logAnalyticsWorkspaceName = 'log-getavail-itn-001' param dataCollectionEndpointName = 'dce-getavail-itn-001' param dataCollectionRuleName = 'dcr-getavail-itn-001' From 8b25b2f32d597638166acec67ded16654514a685 Mon Sep 17 00:00:00 2001 From: Marcello Formica Date: Mon, 27 Apr 2026 13:40:57 +0200 Subject: [PATCH 04/10] refactor: rename -Workspace to -SourceWorkspaceId for clarity Disambiguates the historical data source workspace from the ingestion target (DCE/DCR). The new name makes clear it expects a GUID and that it is NOT the workspace used for result ingestion. --- README.md | 14 +++++----- get-availability.ps1 | 62 +++++++++++++++++++++++--------------------- 2 files changed, 39 insertions(+), 37 deletions(-) diff --git a/README.md b/README.md index e67e6a9..3fc6511 100644 --- a/README.md +++ b/README.md @@ -51,14 +51,14 @@ For the C# version prerequisites and usage, see the [C# README](csharp/README.md | `-ActivityGraceMinutes` | `10` | Post-operation grace window for Activity Log lifecycle events | | `-Batch` | off | Use the regional Metrics Batch API instead of per-resource calls | | `-BatchSize` | `10` | Max resources per batch call (1–50); implies `-Batch` | -| `-Workspace` | *(none)* | Log Analytics workspace ID (GUID). Fetches Activity Log lifecycle events from the workspace via a single bulk KQL query (faster for large estates). Resource Health uses a hybrid approach: KQL transitions cover the period beyond the REST API's ~30-day retention, while REST API transitions (curated, with corrected causes) are authoritative for the last ~30 days. Provides complete Resource Health coverage across the full observation window. | +| `-SourceWorkspaceId` | *(none)* | Log Analytics workspace ID (GUID) used as a source for historical Activity Log and Resource Health data. Not the ingestion target. Fetches lifecycle events via a single bulk KQL query (faster for large estates). Resource Health uses a hybrid approach: KQL transitions cover the period beyond the REST API's ~30-day retention, while REST API transitions (curated, with corrected causes) are authoritative for the last ~30 days. | | `-DceEndpoint` | *(none)* | Data Collection Endpoint ingestion URL. When provided together with `-DcrImmutableId`, results are sent to Log Analytics custom tables via the Azure Monitor Ingestion API. | | `-DcrImmutableId` | *(none)* | Data Collection Rule immutable ID. Required together with `-DceEndpoint` to enable Log Analytics ingestion. | | `-Version` | | Print version and exit | For C# parameters, see the [C# README](csharp/README.md). -The observation window is a UTC calendar month: past months use the full calendar month, the current month is reported month-to-date. The requested month cannot start more than 90 days before the current UTC time. Metrics and Activity Log support that 90-day lookback; Health History is applied only for its overlap with the ~30-day REST API retention window. When `-Workspace` / `--workspace` is used, Health History coverage extends to the full observation period via a hybrid approach (Log Analytics for older transitions + REST API for the last ~30 days). +The observation window is a UTC calendar month: past months use the full calendar month, the current month is reported month-to-date. The requested month cannot start more than 90 days before the current UTC time. Metrics and Activity Log support that 90-day lookback; Health History is applied only for its overlap with the ~30-day REST API retention window. When `-SourceWorkspaceId` / `--workspace` is used, Health History coverage extends to the full observation period via a hybrid approach (Log Analytics for older transitions + REST API for the last ~30 days). ### Examples @@ -76,7 +76,7 @@ The observation window is a UTC calendar month: past months use the full calenda ./get-availability.ps1 -Subscriptions 'Contoso-Production','Contoso-Development' -Month 202603 -BatchSize 20 # Use Log Analytics for Activity Log + Resource Health (faster, extended retention) -./get-availability.ps1 -Subscriptions 'Contoso-Production' -Month 202603 -Workspace 'b233a4b7-3c43-433c-ac60-1f6ff217ddd4' +./get-availability.ps1 -Subscriptions 'Contoso-Production' -Month 202603 -SourceWorkspaceId 'b233a4b7-3c43-433c-ac60-1f6ff217ddd4' # Send results to Log Analytics custom tables ./get-availability.ps1 -Subscriptions 'Contoso-Production' -Month 202603 ` @@ -103,10 +103,10 @@ If the observation window extends beyond the Resource Health retention window, a WARNING: Resource Health history covers only part of this period (2026-02-16 18:54:00Z -> 2026-03-01 00:00:00Z, 17586 of 40320 min). Earlier minutes will use Activity Log and metric fallback rules. ``` -When `-Workspace` is used, the 30-day warning is suppressed (hybrid coverage applies) and an informational line is printed: +When `-SourceWorkspaceId` is used, the 30-day warning is suppressed (hybrid coverage applies) and an informational line is printed: ``` -Log Analytics workspace: b233a4b7-…-1f6ff217ddd4 (Activity Log via KQL, Resource Health via KQL + REST API hybrid) +Log Analytics source workspace: b233a4b7-…-1f6ff217ddd4 (Activity Log via KQL, Resource Health via KQL + REST API hybrid) ``` Table view (Kind is abbreviated: VM, SQL, Storage, Web): @@ -179,8 +179,8 @@ Matching minutes are treated as customer/admin lifecycle activity and removed fr **2. Health History** — Resource Health transitions are converted into three interval types (below). Two data source modes are supported: -- **REST API only** (default, no `-Workspace`): The [Activity Log REST API](https://learn.microsoft.com/azure/azure-monitor/platform/rest-activity-log#retrieve-activity-log-data) and the [Resource Health REST API](https://learn.microsoft.com/en-us/rest/api/resourcehealth/availability-statuses/list?view=rest-resourcehealth-2025-05-01) (`availabilityStatuses`, API version `2025-05-01`) are queried per-resource. Resource Health API has a ~30-day retention limit. -- **Hybrid: Log Analytics + REST API** (`-Workspace` / `--workspace`): A single bulk KQL query against the `AzureActivity` table fetches Activity Log lifecycle events and Resource Health transitions for all resources at once (faster for large estates: 1 query vs. thousands of REST calls). Resource Health transitions older than the REST API's ~30-day retention cutoff come from Log Analytics (workspace retention, typically 365 days). For the last ~30 days, the REST API is always queried and its transitions take precedence — REST data is authoritative because it provides curated synthetic entries that fill coverage gaps between health incidents and retroactively corrects cause classification. The two sources are merged chronologically to form a complete health timeline. Requires the target subscriptions to have diagnostic settings sending Activity Log data to the specified workspace. +- **REST API only** (default, no `-SourceWorkspaceId`): The [Activity Log REST API](https://learn.microsoft.com/azure/azure-monitor/platform/rest-activity-log#retrieve-activity-log-data) and the [Resource Health REST API](https://learn.microsoft.com/en-us/rest/api/resourcehealth/availability-statuses/list?view=rest-resourcehealth-2025-05-01) (`availabilityStatuses`, API version `2025-05-01`) are queried per-resource. Resource Health API has a ~30-day retention limit. +- **Hybrid: Log Analytics + REST API** (`-SourceWorkspaceId` / `--workspace`): A single bulk KQL query against the `AzureActivity` table fetches Activity Log lifecycle events and Resource Health transitions for all resources at once (faster for large estates: 1 query vs. thousands of REST calls). Resource Health transitions older than the REST API's ~30-day retention cutoff come from Log Analytics (workspace retention, typically 365 days). For the last ~30 days, the REST API is always queried and its transitions take precedence — REST data is authoritative because it provides curated synthetic entries that fill coverage gaps between health incidents and retroactively corrects cause classification. The two sources are merged chronologically to form a complete health timeline. Requires the target subscriptions to have diagnostic settings sending Activity Log data to the specified workspace. Health transition interval types: diff --git a/get-availability.ps1 b/get-availability.ps1 index ff3516d..76e56d1 100644 --- a/get-availability.ps1 +++ b/get-availability.ps1 @@ -30,15 +30,17 @@ Remaining null minutes become metric issues (excluded from eligibility), while remaining 0% minutes are trusted as downtime. - When -Workspace is specified, Activity Log lifecycle events are fetched - from a Log Analytics workspace via a single bulk KQL query instead of - per-resource REST API calls. This is faster for large estates and uses + When -SourceWorkspaceId is specified, Activity Log lifecycle events are + fetched from a Log Analytics workspace via a single bulk KQL query instead + of per-resource REST API calls. This is faster for large estates and uses workspace retention (typically 365 days). Resource Health data uses a hybrid approach: Log Analytics transitions cover the period beyond the REST API's ~30-day retention, while REST API transitions (curated, synthetic, with retroactively corrected causes) are authoritative for the last ~30 days. The two sources are merged to produce complete - coverage. Without -Workspace, the REST APIs are used directly. + coverage. Without -SourceWorkspaceId, the REST APIs are used directly. + Note: this is NOT the workspace used for result ingestion (see + -DceEndpoint / -DcrImmutableId for that). The observation window is a UTC calendar month selected via -Month YYYYMM. Current month runs month-to-date; past months run full-month. @@ -67,16 +69,16 @@ .PARAMETER BatchSize Max resources per batch call (default: 10, max 50). Implies -Batch. -.PARAMETER Workspace - Log Analytics workspace ID (GUID). When provided, Activity Log lifecycle - events are fetched from the AzureActivity table in this workspace via a - single bulk KQL query instead of per-resource REST API calls. Resource - Health uses a hybrid approach: Log Analytics transitions cover the period - beyond the REST API's ~30-day retention limit, while REST API transitions - (curated, with corrected causes) are authoritative for the last ~30 days. - Faster for large estates and provides complete Resource Health coverage - across the full observation window. Requires the workspace to receive - Activity Log diagnostic settings from the target subscriptions. +.PARAMETER SourceWorkspaceId + Log Analytics workspace ID (GUID) used as a source for historical Activity + Log and Resource Health data. This is NOT the ingestion target — it is + an existing workspace that receives Activity Log diagnostic settings from + the target subscriptions. When provided, lifecycle events are fetched via + a single bulk KQL query (faster for large estates, uses workspace + retention). Resource Health uses a hybrid approach: Log Analytics + transitions cover the period beyond the REST API's ~30-day retention + limit, while REST API transitions (curated, with corrected causes) are + authoritative for the last ~30 days. .PARAMETER DceEndpoint Data Collection Endpoint ingestion URL. When both -DceEndpoint and @@ -100,7 +102,7 @@ ./get-availability.ps1 -Subscriptions 'MySub' -Month 202506 -Batch -BatchSize 20 .EXAMPLE - ./get-availability.ps1 -Subscriptions 'MySub' -Month 202506 -Workspace 'b233a4b7-3c43-433c-ac60-1f6ff217ddd4' + ./get-availability.ps1 -Subscriptions 'MySub' -Month 202506 -SourceWorkspaceId 'b233a4b7-3c43-433c-ac60-1f6ff217ddd4' .EXAMPLE ./get-availability.ps1 -Subscriptions 'MySub' -Month 202506 -DceEndpoint 'https://dce-getavail-itn-001.italynorth-1.ingest.monitor.azure.com' -DcrImmutableId 'dcr-00000000000000000000000000000000' @@ -140,7 +142,7 @@ param( [Parameter(ParameterSetName = 'Run')] [ValidatePattern('^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$')] - [string]$Workspace, + [string]$SourceWorkspaceId, [Parameter(ParameterSetName = 'Run')] [ValidateNotNullOrEmpty()] @@ -358,9 +360,9 @@ function Get-ShortKind([string]$Kind) { ## Returns the effective start of the Resource Health coverage window, clamped ## to PeriodStart if coverage spans the full observation period. -## When using Log Analytics (-Workspace), the hybrid approach (LA for older -## transitions + REST API for the last ~30 days) provides coverage from -## PeriodStart. Without -Workspace, coverage is limited to ~30 days. +## When using Log Analytics (-SourceWorkspaceId), the hybrid approach (LA for +## older transitions + REST API for the last ~30 days) provides coverage from +## PeriodStart. Without -SourceWorkspaceId, coverage is limited to ~30 days. function Get-HealthCoverageStart([DateTimeOffset]$PeriodStart, [switch]$UseLogAnalytics) { if ($UseLogAnalytics) { return $PeriodStart } $now = [DateTimeOffset]::UtcNow @@ -1511,8 +1513,8 @@ function Invoke-SuspectGapInvestigation { # Shared HttpClient for REST API calls within the parallel block. # Always created: needed for Resource Health REST calls (sole source - # without -Workspace; hybrid supplement with -Workspace) and for - # Activity Log REST calls when -Workspace is not specified. + # without -SourceWorkspaceId; hybrid supplement with -SourceWorkspaceId) + # and for Activity Log REST calls when -SourceWorkspaceId is not specified. $gapHttpClient = [System.Net.Http.HttpClient]::new() $gapHttpClient.DefaultRequestHeaders.Add('Authorization', "Bearer $ArmToken") $gapHttpClient.Timeout = [TimeSpan]::FromMinutes(5) @@ -1925,12 +1927,12 @@ function Invoke-SuspectGapInvestigation { # ── Resource Health ─────────────────────────────────────────── # Query Resource Health transitions to classify platform faults, # unknown states, and customer-initiated events. - # Hybrid mode (-Workspace): LA transitions older than the REST API - # ~30-day cutoff are merged with REST API transitions for the last + # Hybrid mode (-SourceWorkspaceId): LA transitions older than the REST + # API ~30-day cutoff are merged with REST API transitions for the last # ~30 days. REST data is authoritative within its retention window: # it provides curated synthetic entries that fill coverage gaps # between incidents and retroactively corrects cause classification. - # REST-only mode (no -Workspace): REST API is the sole source. + # REST-only mode (no -SourceWorkspaceId): REST API is the sole source. $healthHistoryApplied = [DateTimeOffset]$hcStart -lt [DateTimeOffset]$pEnd $faultIntervals = @() $unknownIntervals = @() @@ -1950,7 +1952,7 @@ function Invoke-SuspectGapInvestigation { } # Fetch REST API health transitions — sole source without - # -Workspace; covers last ~30 days with curated authoritative + # -SourceWorkspaceId; covers last ~30 days with curated authoritative # data in hybrid mode. $url = "https://management.azure.com$($c.ResourceId)" + "/providers/Microsoft.ResourceHealth/availabilityStatuses" + @@ -2240,7 +2242,7 @@ $utcStart = $window.Start $utcEnd = $window.End $totalMinutes = $window.TotalMinutes -$healthCoverageStart = if ($Workspace) { +$healthCoverageStart = if ($SourceWorkspaceId) { Get-HealthCoverageStart $utcStart -UseLogAnalytics } else { Get-HealthCoverageStart $utcStart @@ -2250,8 +2252,8 @@ $healthCoveredMinutes = $healthCoverageStart -lt $utcEnd ? [int]($utcEnd - $heal $periodLabel = $window.IsMonthToDate ? "month $($window.NormalizedMonth) (month-to-date)" : "month $($window.NormalizedMonth)" Write-Host "Period: $periodLabel ($($utcStart.ToString('u')) -> $($utcEnd.ToString('u')), $totalMinutes min)" -if ($Workspace) { - Write-Host "Log Analytics workspace: $Workspace (Activity Log via KQL, Resource Health via KQL + REST API hybrid)" +if ($SourceWorkspaceId) { + Write-Host "Log Analytics source workspace: $SourceWorkspaceId (Activity Log via KQL, Resource Health via KQL + REST API hybrid)" } elseif ($healthCoverageStart -gt $utcStart -and $healthCoveredMinutes -gt 0) { Write-Host "WARNING: Resource Health history covers only part of this period ($($healthCoverageStart.ToString('u')) -> $($utcEnd.ToString('u')), $healthCoveredMinutes of $totalMinutes min). Earlier minutes will use Activity Log and metric fallback rules." } elseif ($healthCoveredMinutes -eq 0) { @@ -2364,12 +2366,12 @@ foreach ($res in $resources) { # complement REST API data in hybrid mode (LA covers pre-30-day, REST provides # curated authoritative data for the last ~30 days). $logAnalyticsData = $null -if ($Workspace -and $suspectCandidates.Count -gt 0) { +if ($SourceWorkspaceId -and $suspectCandidates.Count -gt 0) { Write-Host -NoNewline 'Fetching Activity Log + Resource Health history from Log Analytics... ' $laToken = (Get-AzAccessToken -ResourceUrl 'https://api.loganalytics.io').Token $laTokenStr = ($laToken -is [securestring]) ? ($laToken | ConvertFrom-SecureString -AsPlainText) : [string]$laToken $laToken = $null - $logAnalyticsData = Get-LogAnalyticsData -WorkspaceId $Workspace ` + $logAnalyticsData = Get-LogAnalyticsData -WorkspaceId $SourceWorkspaceId ` -SubscriptionIds $subIds -PeriodStart $utcStart -PeriodEnd $utcEnd ` -ArmToken $laTokenStr $laTokenStr = $null From 6a3d692ea17726a56fcde075a5d4da4fd6cb8f05 Mon Sep 17 00:00:00 2001 From: Marcello Formica Date: Mon, 27 Apr 2026 13:47:04 +0200 Subject: [PATCH 05/10] refactor: clean up Send-ToLogAnalytics, add missing doc comments - Remove unused $batchSize variable - Remove dead status-code check after -ErrorAction Stop - Extract duplicated gzip logic into local $compressJson scriptblock - Early-return on empty payload; hoist headers out of loop - Add ## doc comments to Resolve-ObservationWindow and Get-ResourceInventory --- get-availability.ps1 | 76 +++++++++++++++++++++----------------------- 1 file changed, 36 insertions(+), 40 deletions(-) diff --git a/get-availability.ps1 b/get-availability.ps1 index 76e56d1..bb923ba 100644 --- a/get-availability.ps1 +++ b/get-availability.ps1 @@ -177,8 +177,8 @@ if ($DceEndpoint -and $DcrImmutableId) { # ── Log Analytics Ingestion ─────────────────────────────────────────────────── ## Sends an array of objects to a Log Analytics custom table via the Azure -## Monitor Ingestion REST API. Handles gzip compression and batching for -## the 1 MB per-call payload limit. +## Monitor Ingestion REST API. Gzip-compresses the JSON payload and splits +## into multiple calls if the compressed size exceeds 900 KB (API limit is 1 MB). function Send-ToLogAnalytics { param( [string]$Endpoint, @@ -188,58 +188,51 @@ function Send-ToLogAnalytics { [object[]]$Payload ) + if ($Payload.Count -eq 0) { return } + $uri = "$Endpoint/dataCollectionRules/$RuleId/streams/${StreamName}?api-version=2023-01-01" + $headers = @{ + 'Authorization' = "Bearer $Token" + 'Content-Type' = 'application/json' + 'Content-Encoding' = 'gzip' + } - # Split into batches if payload is large (target < 900 KB compressed) - $batchSize = $Payload.Count - $batches = @() - if ($Payload.Count -gt 0) { - # Start with a single batch; split only if the compressed size exceeds the limit - $json = $Payload | ConvertTo-Json -Depth 5 -Compress -AsArray - $jsonBytes = [System.Text.Encoding]::UTF8.GetBytes($json) + # Gzip-compress a JSON array into a byte[] + $compressJson = { + param([object[]]$Items) + $json = $Items | ConvertTo-Json -Depth 5 -Compress -AsArray + $bytes = [System.Text.Encoding]::UTF8.GetBytes($json) $ms = [System.IO.MemoryStream]::new() $gz = [System.IO.Compression.GZipStream]::new($ms, [System.IO.Compression.CompressionLevel]::Optimal) - $gz.Write($jsonBytes, 0, $jsonBytes.Length) + $gz.Write($bytes, 0, $bytes.Length) $gz.Dispose() - $compressed = $ms.ToArray() + $result = $ms.ToArray() $ms.Dispose() + $result + } - if ($compressed.Length -lt 900KB) { - $batches = @(, $compressed) - } else { - # Split into smaller chunks and compress each - $chunkSize = [math]::Max(1, [math]::Floor($Payload.Count / [math]::Ceiling($compressed.Length / 900KB))) - for ($i = 0; $i -lt $Payload.Count; $i += $chunkSize) { - $chunk = @($Payload[$i..([math]::Min($i + $chunkSize - 1, $Payload.Count - 1))]) - $cJson = $chunk | ConvertTo-Json -Depth 5 -Compress -AsArray - $cBytes = [System.Text.Encoding]::UTF8.GetBytes($cJson) - $cMs = [System.IO.MemoryStream]::new() - $cGz = [System.IO.Compression.GZipStream]::new($cMs, [System.IO.Compression.CompressionLevel]::Optimal) - $cGz.Write($cBytes, 0, $cBytes.Length) - $cGz.Dispose() - $batches += , $cMs.ToArray() - $cMs.Dispose() - } - } + # Try full payload as a single call; split only if compressed size exceeds 900 KB + $compressed = & $compressJson $Payload + if ($compressed.Length -lt 900KB) { + Invoke-WebRequest -Uri $uri -Method Post -Headers $headers -Body $compressed -UseBasicParsing | Out-Null + return } - $batchNum = 0 - foreach ($body in $batches) { - $batchNum++ - $headers = @{ - 'Authorization' = "Bearer $Token" - 'Content-Type' = 'application/json' - 'Content-Encoding' = 'gzip' - } - $response = Invoke-WebRequest -Uri $uri -Method Post -Headers $headers -Body $body -UseBasicParsing -ErrorAction Stop - if ($response.StatusCode -notin 200, 204) { - throw "Ingestion failed (batch $batchNum/$($batches.Count)): HTTP $($response.StatusCode) — $($response.Content)" - } + # Estimate chunk size from compression ratio and send each chunk separately + $chunkSize = [math]::Max(1, [math]::Floor($Payload.Count / [math]::Ceiling($compressed.Length / 900KB))) + for ($i = 0; $i -lt $Payload.Count; $i += $chunkSize) { + $chunk = @($Payload[$i..([math]::Min($i + $chunkSize - 1, $Payload.Count - 1))]) + $body = & $compressJson $chunk + Invoke-WebRequest -Uri $uri -Method Post -Headers $headers -Body $body -UseBasicParsing | Out-Null } } # ── Observation window ──────────────────────────────────────────────────────── +## Parses a YYYYMM string into a start/end observation window, clamped to the +## current UTC minute. Returns Start, End, NormalizedMonth, IsMonthToDate, and +## TotalMinutes. Throws if the month is in the future, older than 90 days, or +## produces an empty period. function Resolve-ObservationWindow([string]$MonthParam) { $parsed = [datetime]::new(1, 1, 1) if (-not [datetime]::TryParseExact($MonthParam, 'yyyyMM', @@ -270,6 +263,9 @@ function Resolve-ObservationWindow([string]$MonthParam) { # ── Resource inventory via Azure Resource Graph ─────────────────────────────── +## Queries Azure Resource Graph for resources matching the requested -Kind and +## optional -ResourceName filter across the supplied subscriptions. Returns +## an array of lightweight resource descriptors used by the metrics pipeline. function Get-ResourceInventory { param( [string[]]$SubscriptionIds, From fd84d11799308f379c062461c1656abd06d0473d Mon Sep 17 00:00:00 2001 From: Marcello Formica Date: Mon, 27 Apr 2026 17:01:16 +0200 Subject: [PATCH 06/10] Restructure repo: Azure Function (Flex Consumption), Bicep infra, CORS, timer param, modules setup --- Bicep/getavailability.bicep | 532 ++++++++ Bicep/parameters.dev.bicepparam | 14 + Functions/GetAvail/.funcignore | 7 + Functions/GetAvail/.gitignore | 14 + Functions/GetAvail/Modules/README.md | 19 + .../GetAvail/RunGetAvailability/function.json | 10 + Functions/GetAvail/RunGetAvailability/run.ps1 | 107 ++ .../GetAvail/get-availability.ps1 | 0 Functions/GetAvail/host.json | 21 + Functions/GetAvail/profile.ps1 | 16 + Functions/GetAvail/requirements.psd1 | 10 + {csharp => Old}/Get-Availability.sln | 0 .../GetAvailability/GetAvailability.csproj | 0 .../Models/EligibilityResult.cs | 0 .../GetAvailability/Models/MetricScalars.cs | 0 .../GetAvailability/Models/TrackedResource.cs | 0 .../GetAvailability/Output/SummaryWriter.cs | 0 {csharp => Old}/GetAvailability/Program.cs | 0 .../Services/ActivityLogService.cs | 0 .../Services/BatchMetricsService.cs | 0 .../Services/LogAnalyticsService.cs | 0 .../Services/MetricsService.cs | 0 .../Services/ResourceHealthService.cs | 0 .../Services/ResourceInventoryService.cs | 0 .../Services/SubscriptionResolver.cs | 0 {csharp => Old}/README.md | 4 +- README.md | 156 ++- Setup/PLAN-log-analytics-ingestion.md | 289 ----- Setup/README.md | 107 -- Setup/certlc.bicep | 1136 ----------------- Setup/getavailability.bicep | 214 ---- Setup/parameters.dev.bicepparam | 5 - 32 files changed, 894 insertions(+), 1767 deletions(-) create mode 100644 Bicep/getavailability.bicep create mode 100644 Bicep/parameters.dev.bicepparam create mode 100644 Functions/GetAvail/.funcignore create mode 100644 Functions/GetAvail/.gitignore create mode 100644 Functions/GetAvail/Modules/README.md create mode 100644 Functions/GetAvail/RunGetAvailability/function.json create mode 100644 Functions/GetAvail/RunGetAvailability/run.ps1 rename get-availability.ps1 => Functions/GetAvail/get-availability.ps1 (100%) create mode 100644 Functions/GetAvail/host.json create mode 100644 Functions/GetAvail/profile.ps1 create mode 100644 Functions/GetAvail/requirements.psd1 rename {csharp => Old}/Get-Availability.sln (100%) rename {csharp => Old}/GetAvailability/GetAvailability.csproj (100%) rename {csharp => Old}/GetAvailability/Models/EligibilityResult.cs (100%) rename {csharp => Old}/GetAvailability/Models/MetricScalars.cs (100%) rename {csharp => Old}/GetAvailability/Models/TrackedResource.cs (100%) rename {csharp => Old}/GetAvailability/Output/SummaryWriter.cs (100%) rename {csharp => Old}/GetAvailability/Program.cs (100%) rename {csharp => Old}/GetAvailability/Services/ActivityLogService.cs (100%) rename {csharp => Old}/GetAvailability/Services/BatchMetricsService.cs (100%) rename {csharp => Old}/GetAvailability/Services/LogAnalyticsService.cs (100%) rename {csharp => Old}/GetAvailability/Services/MetricsService.cs (100%) rename {csharp => Old}/GetAvailability/Services/ResourceHealthService.cs (100%) rename {csharp => Old}/GetAvailability/Services/ResourceInventoryService.cs (100%) rename {csharp => Old}/GetAvailability/Services/SubscriptionResolver.cs (100%) rename {csharp => Old}/README.md (98%) delete mode 100644 Setup/PLAN-log-analytics-ingestion.md delete mode 100644 Setup/README.md delete mode 100644 Setup/certlc.bicep delete mode 100644 Setup/getavailability.bicep delete mode 100644 Setup/parameters.dev.bicepparam diff --git a/Bicep/getavailability.bicep b/Bicep/getavailability.bicep new file mode 100644 index 0000000..60a24a1 --- /dev/null +++ b/Bicep/getavailability.bicep @@ -0,0 +1,532 @@ +/* + +Get-Availability — Bicep template for the complete Get-Availability infrastructure. + +Creates a Log Analytics workspace, custom tables, DCE, DCR, Storage Account, +Flex Consumption Function App (with auto-wired app settings), Application Insights, +Private Endpoints, and RBAC role assignments. + +Validate: az deployment group validate --resource-group --parameters .\parameters.dev.bicepparam +What-if: az deployment group what-if --resource-group --parameters .\parameters.dev.bicepparam +Deploy: az deployment group create --resource-group --parameters .\parameters.dev.bicepparam + +*/ + +metadata name = 'Get-Availability Infrastructure' +metadata description = 'Complete infrastructure for the Get-Availability solution: Log Analytics, DCE, DCR, Function App, App Insights, Private Endpoints, and RBAC' + +// ── Parameters ─────────────────────────────────────────────────────────────── + +@description('Azure region for all resources. Defaults to the resource group location.') +param location string = resourceGroup().location + +@description('Name of the Log Analytics workspace to create.') +param logAnalyticsWorkspaceName string + +@description('Name of the Data Collection Endpoint.') +param dataCollectionEndpointName string + +@description('Name of the Data Collection Rule.') +param dataCollectionRuleName string + +@description('Name of the Storage Account for the Function App. Must be globally unique, 3-24 characters, lowercase letters and numbers only.') +@minLength(3) +@maxLength(24) +param storageAccountName string + +@description('Name of the Function App. Must be globally unique, 2-60 characters, alphanumerics and hyphens.') +@minLength(2) +@maxLength(60) +param functionAppName string + +@description('Name of the Application Insights instance for Function App monitoring.') +param applicationInsightsName string + +@description('Subnet resource ID for Function App VNet integration. Must be delegated to Microsoft.App/environments.') +param fnSubnetId string + +@description('Subnet resource ID for Private Endpoints.') +param peSubnetId string + +@description('Subscription ID containing existing Private DNS Zones.') +param dnsZonesSubscriptionId string + +@description('Resource group name containing existing Private DNS Zones.') +param dnsZonesResourceGroupName string + +@description('Comma-separated list of Azure subscription names or IDs to monitor (written to GETAVAIL_SUBSCRIPTIONS app setting).') +param getavailSubscriptions string + +@description('Comma-separated resource kinds to monitor. Default: vm,sql,storage,webapp') +param getavailKinds string = 'vm,sql,storage,webapp' + +@description('Log Analytics workspace customer ID used as source for Activity Log and Resource Health queries (SOURCE_WORKSPACE_ID app setting). Leave empty to skip.') +param sourceWorkspaceId string = '' + +@description('CRON expression for the timer trigger schedule. Default: 6 AM on the 1st of every month (0 0 6 1 * *).') +param timerSchedule string = '0 0 6 1 * *' + +// ── Variables ──────────────────────────────────────────────────────────────── + +var commonTags = { + solution: 'Get-Availability' +} + +// Azure built-in role definition IDs +var roleDefinitions = { + monitoringMetricsPublisher: '3913510d-42f4-4e42-8a64-420c390055eb' + storageBlobDataOwner: 'b7e6dc6d-f1e8-4753-8033-0f276bb0955b' +} + +// ── Existing Private DNS Zones ─────────────────────────────────────────────── + +resource blobDnsZone 'Microsoft.Network/privateDnsZones@2020-06-01' existing = { + name: 'privatelink.blob.${environment().suffixes.storage}' + scope: resourceGroup(dnsZonesSubscriptionId, dnsZonesResourceGroupName) +} + +resource webAppDnsZone 'Microsoft.Network/privateDnsZones@2020-06-01' existing = { + name: 'privatelink.azurewebsites.net' + scope: resourceGroup(dnsZonesSubscriptionId, dnsZonesResourceGroupName) +} + +// ── Log Analytics Workspace ────────────────────────────────────────────────── + +resource logAnalyticsWorkspace 'Microsoft.OperationalInsights/workspaces@2023-09-01' = { + name: logAnalyticsWorkspaceName + location: location + properties: { + sku: { + name: 'PerGB2018' + } + retentionInDays: 30 + } + tags: commonTags +} + +// ── Custom Table: GetAvailResources_CL (per-resource detail) ───────────────── + +resource resourcesTable 'Microsoft.OperationalInsights/workspaces/tables@2022-10-01' = { + name: 'GetAvailResources_CL' + parent: logAnalyticsWorkspace + properties: { + retentionInDays: 30 + schema: { + name: 'GetAvailResources_CL' + columns: [ + { name: 'TimeGenerated', type: 'dateTime' } + { name: 'RunId', type: 'string' } + { name: 'Month', type: 'string' } + { name: 'PeriodStart', type: 'dateTime' } + { name: 'PeriodEnd', type: 'dateTime' } + { name: 'IsMonthToDate', type: 'boolean' } + { name: 'SubscriptionName', type: 'string' } + { name: 'ResourceName', type: 'string' } + { name: 'ResourceId', type: 'string' } + { name: 'ResourceGroup', type: 'string' } + { name: 'Kind', type: 'string' } + { name: 'Location', type: 'string' } + { name: 'EligibleMinutes', type: 'int' } + { name: 'AvailableMinutes', type: 'real' } + { name: 'SuspectMinutes', type: 'int' } + { name: 'ConfirmedDowntimeMinutes', type: 'int' } + { name: 'ExcusedMinutes', type: 'int' } + { name: 'UnexplainedSuspectMinutes', type: 'int' } + { name: 'AvailabilityPct', type: 'real' } + ] + } + } +} + +// ── Custom Table: GetAvailSummary_CL (aggregated summaries) ────────────────── + +resource summaryTable 'Microsoft.OperationalInsights/workspaces/tables@2022-10-01' = { + name: 'GetAvailSummary_CL' + parent: logAnalyticsWorkspace + properties: { + retentionInDays: 30 + schema: { + name: 'GetAvailSummary_CL' + columns: [ + { name: 'TimeGenerated', type: 'dateTime' } + { name: 'RunId', type: 'string' } + { name: 'Month', type: 'string' } + { name: 'PeriodStart', type: 'dateTime' } + { name: 'PeriodEnd', type: 'dateTime' } + { name: 'IsMonthToDate', type: 'boolean' } + { name: 'SummaryLevel', type: 'string' } + { name: 'SubscriptionName', type: 'string' } + { name: 'Kind', type: 'string' } + { name: 'Location', type: 'string' } + { name: 'ResourceCount', type: 'int' } + { name: 'EligibleMinutes', type: 'real' } + { name: 'AvailableMinutes', type: 'real' } + { name: 'AvailabilityPct', type: 'real' } + ] + } + } +} + +// ── Data Collection Endpoint ───────────────────────────────────────────────── + +resource dataCollectionEndpoint 'Microsoft.Insights/dataCollectionEndpoints@2023-03-11' = { + name: dataCollectionEndpointName + location: location + properties: { + networkAcls: { + publicNetworkAccess: 'Enabled' + } + } + tags: commonTags +} + +// ── Data Collection Rule (two streams, one per table) ──────────────────────── + +resource dataCollectionRule 'Microsoft.Insights/dataCollectionRules@2023-03-11' = { + name: dataCollectionRuleName + location: location + properties: { + dataCollectionEndpointId: dataCollectionEndpoint.id + + // Stream declarations — columns sent by the script (no TimeGenerated; injected by transform) + streamDeclarations: { + 'Custom-GetAvailResources_CL': { + columns: [ + { name: 'RunId', type: 'string' } + { name: 'Month', type: 'string' } + { name: 'PeriodStart', type: 'string' } + { name: 'PeriodEnd', type: 'string' } + { name: 'IsMonthToDate', type: 'boolean' } + { name: 'SubscriptionName', type: 'string' } + { name: 'ResourceName', type: 'string' } + { name: 'ResourceId', type: 'string' } + { name: 'ResourceGroup', type: 'string' } + { name: 'Kind', type: 'string' } + { name: 'Location', type: 'string' } + { name: 'EligibleMinutes', type: 'int' } + { name: 'AvailableMinutes', type: 'real' } + { name: 'SuspectMinutes', type: 'int' } + { name: 'ConfirmedDowntimeMinutes', type: 'int' } + { name: 'ExcusedMinutes', type: 'int' } + { name: 'UnexplainedSuspectMinutes', type: 'int' } + { name: 'AvailabilityPct', type: 'real' } + ] + } + 'Custom-GetAvailSummary_CL': { + columns: [ + { name: 'RunId', type: 'string' } + { name: 'Month', type: 'string' } + { name: 'PeriodStart', type: 'string' } + { name: 'PeriodEnd', type: 'string' } + { name: 'IsMonthToDate', type: 'boolean' } + { name: 'SummaryLevel', type: 'string' } + { name: 'SubscriptionName', type: 'string' } + { name: 'Kind', type: 'string' } + { name: 'Location', type: 'string' } + { name: 'ResourceCount', type: 'int' } + { name: 'EligibleMinutes', type: 'real' } + { name: 'AvailableMinutes', type: 'real' } + { name: 'AvailabilityPct', type: 'real' } + ] + } + } + + destinations: { + logAnalytics: [ + { + workspaceResourceId: logAnalyticsWorkspace.id + name: 'workspace' + } + ] + } + + dataFlows: [ + { + streams: [ 'Custom-GetAvailResources_CL' ] + destinations: [ 'workspace' ] + transformKql: 'source | extend TimeGenerated = now(), PeriodStart = todatetime(PeriodStart), PeriodEnd = todatetime(PeriodEnd)' + outputStream: 'Custom-GetAvailResources_CL' + } + { + streams: [ 'Custom-GetAvailSummary_CL' ] + destinations: [ 'workspace' ] + transformKql: 'source | extend TimeGenerated = now(), PeriodStart = todatetime(PeriodStart), PeriodEnd = todatetime(PeriodEnd)' + outputStream: 'Custom-GetAvailSummary_CL' + } + ] + } + dependsOn: [ + resourcesTable + summaryTable + ] + tags: commonTags +} + +// ── Storage Account ────────────────────────────────────────────────────────── + +resource storageAccount 'Microsoft.Storage/storageAccounts@2025-01-01' = { + name: storageAccountName + location: location + sku: { + name: 'Standard_LRS' + } + kind: 'StorageV2' + properties: { + accessTier: 'Hot' + defaultToOAuthAuthentication: true + allowBlobPublicAccess: false + allowSharedKeyAccess: false + minimumTlsVersion: 'TLS1_2' + supportsHttpsTrafficOnly: true + networkAcls: { + bypass: 'AzureServices' + defaultAction: 'Deny' + } + publicNetworkAccess: 'Disabled' + encryption: { + services: { + blob: { + enabled: true + } + } + } + } + resource blobServices 'blobServices' = { + name: 'default' + properties: {} + } + tags: commonTags +} + +// ── Private Endpoint: Storage Account (blob) ───────────────────────────────── + +resource storageAccountBlobPrivateEndpoint 'Microsoft.Network/privateEndpoints@2024-10-01' = { + name: 'pe-blob-${storageAccountName}' + location: location + properties: { + subnet: { + id: peSubnetId + } + privateLinkServiceConnections: [ + { + name: 'pls-${storageAccountName}' + properties: { + privateLinkServiceId: storageAccount.id + groupIds: [ + 'blob' + ] + } + } + ] + customNetworkInterfaceName: 'nic-pe-${storageAccountName}' + } + tags: commonTags + + resource privateDnsZoneGroup 'privateDnsZoneGroups' = { + name: 'default' + properties: { + privateDnsZoneConfigs: [ + { + name: 'config1' + properties: { + privateDnsZoneId: blobDnsZone.id + } + } + ] + } + } +} + +// ── Application Insights ───────────────────────────────────────────────────── + +resource applicationInsights 'Microsoft.Insights/components@2020-02-02' = { + name: applicationInsightsName + location: location + kind: 'web' + properties: { + Application_Type: 'web' + WorkspaceResourceId: logAnalyticsWorkspace.id + DisableLocalAuth: true + } + dependsOn: [ + dataCollectionRule // Ensure workspace backend is fully active before App Insights connects + ] + tags: commonTags +} + +// ── Flex Consumption Plan ──────────────────────────────────────────────────── + +resource flexServicePlan 'Microsoft.Web/serverfarms@2024-11-01' = { + name: 'asp-${functionAppName}' + location: location + kind: 'functionapp' + sku: { + tier: 'FlexConsumption' + name: 'FC1' + } + properties: { + reserved: true + } + tags: commonTags +} + +// ── Function App ───────────────────────────────────────────────────────────── + +resource functionApp 'Microsoft.Web/sites@2024-11-01' = { + name: functionAppName + location: location + kind: 'functionapp,linux' + identity: { + type: 'SystemAssigned' + } + properties: { + serverFarmId: flexServicePlan.id + httpsOnly: true + virtualNetworkSubnetId: fnSubnetId + publicNetworkAccess: 'Disabled' + siteConfig: { + minTlsVersion: '1.2' + cors: { + allowedOrigins: [ + 'https://portal.azure.com' + ] + } + } + functionAppConfig: { + deployment: { + storage: { + type: 'blobContainer' + value: '${storageAccount.properties.primaryEndpoints.blob}azure-webjobs-hosts' + authentication: { + type: 'SystemAssignedIdentity' + } + } + } + scaleAndConcurrency: { + maximumInstanceCount: 100 + instanceMemoryMB: 2048 + } + runtime: { + name: 'powerShell' + version: '7.4' + } + } + } + resource appSettings 'config' = { + name: 'appsettings' + properties: { + // Function App infrastructure + AzureWebJobsStorage__credential: 'managedidentity' + AzureWebJobsStorage__blobServiceUri: storageAccount.properties.primaryEndpoints.blob + APPLICATIONINSIGHTS_AUTHENTICATION_STRING: 'Authorization=AAD' + APPLICATIONINSIGHTS_CONNECTION_STRING: applicationInsights.properties.ConnectionString + + // Get-Availability configuration — auto-wired from Bicep resources + GETAVAIL_SUBSCRIPTIONS: getavailSubscriptions + GETAVAIL_KINDS: getavailKinds + DCE_ENDPOINT: dataCollectionEndpoint.properties.logsIngestion.endpoint + DCR_IMMUTABLE_ID: dataCollectionRule.properties.immutableId + SOURCE_WORKSPACE_ID: sourceWorkspaceId + TIMER_SCHEDULE: timerSchedule + } + } + dependsOn: [ + storageAccountBlobPrivateEndpoint // Create function only after storage PE is ready + ] + tags: commonTags +} + +// ── Private Endpoint: Function App (sites) ─────────────────────────────────── + +resource functionAppPrivateEndpoint 'Microsoft.Network/privateEndpoints@2024-10-01' = { + name: 'pe-sites-${functionAppName}' + location: location + properties: { + subnet: { + id: peSubnetId + } + privateLinkServiceConnections: [ + { + name: 'pls-${functionAppName}' + properties: { + privateLinkServiceId: functionApp.id + groupIds: [ + 'sites' + ] + } + } + ] + customNetworkInterfaceName: 'nic-pe-${functionAppName}' + } + tags: commonTags + + resource privateDnsZoneGroup 'privateDnsZoneGroups' = { + name: 'default' + properties: { + privateDnsZoneConfigs: [ + { + name: 'config1' + properties: { + privateDnsZoneId: webAppDnsZone.id + } + } + ] + } + } +} + +// ── RBAC Role Assignments ──────────────────────────────────────────────────── + +// Function App → Monitoring Metrics Publisher → DCR (ingest custom logs) +resource functionAppDcrPublisher 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(subscription().id, resourceGroup().id, 'functionAppDcrPublisher') + scope: dataCollectionRule + properties: { + description: 'Function App -> Monitoring Metrics Publisher -> DCR' + roleDefinitionId: subscriptionResourceId( + 'Microsoft.Authorization/roleDefinitions', + roleDefinitions.monitoringMetricsPublisher + ) + principalId: functionApp.identity.principalId + principalType: 'ServicePrincipal' + } +} + +// Function App → Monitoring Metrics Publisher → Application Insights (Entra-only telemetry) +resource functionAppAppiPublisher 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(subscription().id, resourceGroup().id, 'functionAppAppiPublisher') + scope: applicationInsights + properties: { + description: 'Function App -> Monitoring Metrics Publisher -> Application Insights' + roleDefinitionId: subscriptionResourceId( + 'Microsoft.Authorization/roleDefinitions', + roleDefinitions.monitoringMetricsPublisher + ) + principalId: functionApp.identity.principalId + principalType: 'ServicePrincipal' + } +} + +// Function App → Storage Blob Data Owner → Storage Account (Flex Consumption deployment blobs) +resource functionAppStorageBlobDataOwner 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(subscription().id, resourceGroup().id, 'functionAppStorageBlobDataOwner') + scope: storageAccount + properties: { + description: 'Function App -> Storage Blob Data Owner -> Storage Account' + roleDefinitionId: subscriptionResourceId( + 'Microsoft.Authorization/roleDefinitions', + roleDefinitions.storageBlobDataOwner + ) + principalId: functionApp.identity.principalId + principalType: 'ServicePrincipal' + } +} + +// ── Outputs ────────────────────────────────────────────────────────────────── + +output logAnalyticsWorkspaceId string = logAnalyticsWorkspace.id +output dceIngestionEndpoint string = dataCollectionEndpoint.properties.logsIngestion.endpoint +output dataCollectionRuleImmutableId string = dataCollectionRule.properties.immutableId +output functionAppId string = functionApp.id +output applicationInsightsId string = applicationInsights.id +output storageAccountId string = storageAccount.id diff --git a/Bicep/parameters.dev.bicepparam b/Bicep/parameters.dev.bicepparam new file mode 100644 index 0000000..e8e1fae --- /dev/null +++ b/Bicep/parameters.dev.bicepparam @@ -0,0 +1,14 @@ +using './getavailability.bicep' + +param logAnalyticsWorkspaceName = 'log-getavail-itn-001' +param dataCollectionEndpointName = 'dce-getavail-itn-001' +param dataCollectionRuleName = 'dcr-getavail-itn-001' +param storageAccountName = 'flazstgetavailitn001' +param functionAppName = 'fn-getavail-itn-001' +param applicationInsightsName = 'appi-getavail-itn-001' +param peSubnetId = '/subscriptions/9068a229-f092-400e-8093-87e8e7d26ae1/resourceGroups/rg-alz-net-workloads-itn-001/providers/Microsoft.Network/virtualNetworks/vnet-alz-workloads-itn-001/subnets/snet-alz-pe-workloads-itn-001' +param fnSubnetId = '/subscriptions/9068a229-f092-400e-8093-87e8e7d26ae1/resourceGroups/rg-alz-net-workloads-itn-001/providers/Microsoft.Network/virtualNetworks/vnet-alz-workloads-itn-001/subnets/snet-alz-fn-workloads-itn-001' +param dnsZonesSubscriptionId = 'c4e6c176-bf9c-4e8c-87b2-ebdceea7085f' +param dnsZonesResourceGroupName = 'rg-alz-dns-hub-itn-001' +param getavailSubscriptions = 'Flaz-Connectivity,Flaz-Management,Flaz-Identity,Flaz-Workloads' +param sourceWorkspaceId = 'f25755bb-9b46-4aac-bfae-6a10c4c18440' diff --git a/Functions/GetAvail/.funcignore b/Functions/GetAvail/.funcignore new file mode 100644 index 0000000..aea08de --- /dev/null +++ b/Functions/GetAvail/.funcignore @@ -0,0 +1,7 @@ +.git* +.vscode +__azurite_db*__.json +__blobstorage__ +__queuestorage__ +local.settings.json +test diff --git a/Functions/GetAvail/.gitignore b/Functions/GetAvail/.gitignore new file mode 100644 index 0000000..706b839 --- /dev/null +++ b/Functions/GetAvail/.gitignore @@ -0,0 +1,14 @@ + +# Azure Functions artifacts +bin +obj +appsettings.json +local.settings.json + +# Azurite artifacts +__blobstorage__ +__queuestorage__ +__azurite_db*__.json + +# Saved modules (large, fetched before publishing) +Modules/Az.* diff --git a/Functions/GetAvail/Modules/README.md b/Functions/GetAvail/Modules/README.md new file mode 100644 index 0000000..60f8d02 --- /dev/null +++ b/Functions/GetAvail/Modules/README.md @@ -0,0 +1,19 @@ +# Modules Directory + +This folder holds the PowerShell modules required by the function app at runtime. + +Since the Flex Consumption plan **does not support managed dependencies**, modules +must be saved here manually using `Save-Module`: + +```powershell +Save-Module -Name Az.Accounts -Path . -Repository PSGallery -Force +Save-Module -Name Az.ResourceGraph -Path . -Repository PSGallery -Force +``` + +The `get-availability.ps1` script requires: +- **Az.Accounts** — `Get-AzAccessToken`, `Get-AzContext`, `Connect-AzAccount` +- **Az.ResourceGraph** — `Search-AzGraph` + +> Do not commit the module folders themselves to source control — they are large +> and version-pinned. Instead, run `Save-Module` as part of your deployment pipeline +> or locally before publishing with `func azure functionapp publish`. diff --git a/Functions/GetAvail/RunGetAvailability/function.json b/Functions/GetAvail/RunGetAvailability/function.json new file mode 100644 index 0000000..7f4e492 --- /dev/null +++ b/Functions/GetAvail/RunGetAvailability/function.json @@ -0,0 +1,10 @@ +{ + "bindings": [ + { + "name": "Timer", + "type": "timerTrigger", + "direction": "in", + "schedule": "%TIMER_SCHEDULE%" + } + ] +} diff --git a/Functions/GetAvail/RunGetAvailability/run.ps1 b/Functions/GetAvail/RunGetAvailability/run.ps1 new file mode 100644 index 0000000..867f466 --- /dev/null +++ b/Functions/GetAvail/RunGetAvailability/run.ps1 @@ -0,0 +1,107 @@ +<# +.SYNOPSIS + Timer-triggered function that runs get-availability.ps1 for the previous month. + +.DESCRIPTION + Triggered on a CRON schedule (default: 1st of every month at 06:00 UTC). + Can also be started manually from the Azure Portal or via the Functions runtime API. + + Reads configuration from App Settings (environment variables): + GETAVAIL_SUBSCRIPTIONS - Comma-separated list of subscription names or IDs (required) + GETAVAIL_KINDS - Comma-separated resource kinds (default: vm,sql,storage,webapp) + DCE_ENDPOINT - Data Collection Endpoint URL (optional, enables ingestion) + DCR_IMMUTABLE_ID - Data Collection Rule immutable ID (optional, paired with DCE_ENDPOINT) + SOURCE_WORKSPACE_ID - Log Analytics workspace ID for Resource Health queries (optional) + GETAVAIL_PARALLELISM - Parallel thread count (optional, default: script default) + GETAVAIL_BATCH - Set to "true" to enable batch metrics API (optional) + GETAVAIL_BATCH_SIZE - Batch size when batch mode is enabled (optional) + + The get-availability.ps1 script lives at the function app root (next to host.json) + and is deployed automatically by 'func azure functionapp publish'. +#> + +param($Timer) + +# Strict / fail fast +Set-StrictMode -Version 1.0 +$ErrorActionPreference = 'Stop' + +# ── Read configuration from App Settings ────────────────────────────────────── + +$subscriptionsRaw = $env:GETAVAIL_SUBSCRIPTIONS +if ([string]::IsNullOrWhiteSpace($subscriptionsRaw)) { + throw 'App Setting GETAVAIL_SUBSCRIPTIONS is required but not set.' +} +$subscriptions = @($subscriptionsRaw -split ',' | ForEach-Object { $_.Trim() } | Where-Object { $_ }) + +$kindsRaw = $env:GETAVAIL_KINDS +$kinds = if (-not [string]::IsNullOrWhiteSpace($kindsRaw)) { + @($kindsRaw -split ',' | ForEach-Object { $_.Trim() } | Where-Object { $_ }) +} else { + @('vm', 'sql', 'storage', 'webapp') +} + +# Compute previous month in YYYYMM format +$previousMonth = (Get-Date).ToUniversalTime().AddMonths(-1).ToString('yyyyMM') + +# ── Ensure Azure context ───────────────────────────────────────────────────── + +try { + $context = Get-AzContext + if (-not $context -or -not $context.Account -or $context.Account.Id -eq 'NotLoggedIn') { + Write-Warning 'No valid Azure context found. Attempting Identity-based login...' + Disable-AzContextAutosave -Scope Process | Out-Null + Connect-AzAccount -Identity -ErrorAction Stop | Out-Null + Write-Information 'Identity-based login succeeded.' + } else { + Write-Information "Using existing Azure context: $($context.Account.Id)" + } +} catch { + throw "Failed to verify or establish Azure login context: $_" +} + +# ── Build script arguments ──────────────────────────────────────────────────── + +$scriptPath = Join-Path $PSScriptRoot '..' 'get-availability.ps1' +if (-not (Test-Path $scriptPath)) { + throw "get-availability.ps1 not found at expected path: $scriptPath" +} + +$scriptArgs = @{ + Subscriptions = $subscriptions + Month = $previousMonth + Kinds = $kinds +} + +# Optional: Log Analytics ingestion +if (-not [string]::IsNullOrWhiteSpace($env:DCE_ENDPOINT) -and -not [string]::IsNullOrWhiteSpace($env:DCR_IMMUTABLE_ID)) { + $scriptArgs['DceEndpoint'] = $env:DCE_ENDPOINT + $scriptArgs['DcrImmutableId'] = $env:DCR_IMMUTABLE_ID +} + +# Optional: Source workspace for Resource Health +if (-not [string]::IsNullOrWhiteSpace($env:SOURCE_WORKSPACE_ID)) { + $scriptArgs['SourceWorkspaceId'] = $env:SOURCE_WORKSPACE_ID +} + +# Optional: Parallelism +if (-not [string]::IsNullOrWhiteSpace($env:GETAVAIL_PARALLELISM)) { + $scriptArgs['Parallelism'] = [int]$env:GETAVAIL_PARALLELISM +} + +# Optional: Batch mode +if ($env:GETAVAIL_BATCH -eq 'true') { + $scriptArgs['Batch'] = $true + if (-not [string]::IsNullOrWhiteSpace($env:GETAVAIL_BATCH_SIZE)) { + $scriptArgs['BatchSize'] = [int]$env:GETAVAIL_BATCH_SIZE + } +} + +# ── Execute ─────────────────────────────────────────────────────────────────── + +$timerStatus = if ($Timer.IsPastDue) { 'past due' } else { 'on time' } +Write-Information "GetAvail timer trigger fired ($timerStatus). Running for month $previousMonth with $($subscriptions.Count) subscription(s)." + +& $scriptPath @scriptArgs + +Write-Information "GetAvail completed for month $previousMonth." diff --git a/get-availability.ps1 b/Functions/GetAvail/get-availability.ps1 similarity index 100% rename from get-availability.ps1 rename to Functions/GetAvail/get-availability.ps1 diff --git a/Functions/GetAvail/host.json b/Functions/GetAvail/host.json new file mode 100644 index 0000000..a14a691 --- /dev/null +++ b/Functions/GetAvail/host.json @@ -0,0 +1,21 @@ +{ + "version": "2.0", + "logging": { + "logLevel": { + "default": "Warning", + "Function": "Information" + }, + "applicationInsights": { + "samplingSettings": { + "isEnabled": false + } + } + }, + "extensionBundle": { + "id": "Microsoft.Azure.Functions.ExtensionBundle", + "version": "[4.*, 5.0.0)" + }, + "managedDependency": { + "enabled": true + } +} diff --git a/Functions/GetAvail/profile.ps1 b/Functions/GetAvail/profile.ps1 new file mode 100644 index 0000000..20960a2 --- /dev/null +++ b/Functions/GetAvail/profile.ps1 @@ -0,0 +1,16 @@ +# Azure Functions profile.ps1 +# +# This profile.ps1 will get executed every "cold start" of your Function App. +# "cold start" occurs when: +# +# * A Function App starts up for the very first time +# * A Function App starts up after being de-allocated due to inactivity +# +# You can define helper functions, run commands, or specify environment variables +# NOTE: any variables defined that are not environment variables will get reset after the first execution + +# Authenticate with Azure PowerShell using MSI. +if ($env:MSI_SECRET) { + Disable-AzContextAutosave -Scope Process | Out-Null + Connect-AzAccount -Identity +} diff --git a/Functions/GetAvail/requirements.psd1 b/Functions/GetAvail/requirements.psd1 new file mode 100644 index 0000000..ab114e7 --- /dev/null +++ b/Functions/GetAvail/requirements.psd1 @@ -0,0 +1,10 @@ +# This file enables modules to be automatically managed by the Functions service. +# See https://aka.ms/functionsmanageddependency for additional information. +# +# NOTE: DO NOT USE WITH FLEX FUNCTIONS - managed dependencies are not supported in the Flex Consumption plan. +# Do "Save-Module -Name -Path Modules -Repository PSGallery -Force" to add modules to the Modules folder instead. + +@{ +# # For latest supported version, go to 'https://www.powershellgallery.com/packages/Az'. Uncomment the next line and replace the MAJOR_VERSION, e.g., 'Az' = '5.*' +# 'Az' = '14.*' +} diff --git a/csharp/Get-Availability.sln b/Old/Get-Availability.sln similarity index 100% rename from csharp/Get-Availability.sln rename to Old/Get-Availability.sln diff --git a/csharp/GetAvailability/GetAvailability.csproj b/Old/GetAvailability/GetAvailability.csproj similarity index 100% rename from csharp/GetAvailability/GetAvailability.csproj rename to Old/GetAvailability/GetAvailability.csproj diff --git a/csharp/GetAvailability/Models/EligibilityResult.cs b/Old/GetAvailability/Models/EligibilityResult.cs similarity index 100% rename from csharp/GetAvailability/Models/EligibilityResult.cs rename to Old/GetAvailability/Models/EligibilityResult.cs diff --git a/csharp/GetAvailability/Models/MetricScalars.cs b/Old/GetAvailability/Models/MetricScalars.cs similarity index 100% rename from csharp/GetAvailability/Models/MetricScalars.cs rename to Old/GetAvailability/Models/MetricScalars.cs diff --git a/csharp/GetAvailability/Models/TrackedResource.cs b/Old/GetAvailability/Models/TrackedResource.cs similarity index 100% rename from csharp/GetAvailability/Models/TrackedResource.cs rename to Old/GetAvailability/Models/TrackedResource.cs diff --git a/csharp/GetAvailability/Output/SummaryWriter.cs b/Old/GetAvailability/Output/SummaryWriter.cs similarity index 100% rename from csharp/GetAvailability/Output/SummaryWriter.cs rename to Old/GetAvailability/Output/SummaryWriter.cs diff --git a/csharp/GetAvailability/Program.cs b/Old/GetAvailability/Program.cs similarity index 100% rename from csharp/GetAvailability/Program.cs rename to Old/GetAvailability/Program.cs diff --git a/csharp/GetAvailability/Services/ActivityLogService.cs b/Old/GetAvailability/Services/ActivityLogService.cs similarity index 100% rename from csharp/GetAvailability/Services/ActivityLogService.cs rename to Old/GetAvailability/Services/ActivityLogService.cs diff --git a/csharp/GetAvailability/Services/BatchMetricsService.cs b/Old/GetAvailability/Services/BatchMetricsService.cs similarity index 100% rename from csharp/GetAvailability/Services/BatchMetricsService.cs rename to Old/GetAvailability/Services/BatchMetricsService.cs diff --git a/csharp/GetAvailability/Services/LogAnalyticsService.cs b/Old/GetAvailability/Services/LogAnalyticsService.cs similarity index 100% rename from csharp/GetAvailability/Services/LogAnalyticsService.cs rename to Old/GetAvailability/Services/LogAnalyticsService.cs diff --git a/csharp/GetAvailability/Services/MetricsService.cs b/Old/GetAvailability/Services/MetricsService.cs similarity index 100% rename from csharp/GetAvailability/Services/MetricsService.cs rename to Old/GetAvailability/Services/MetricsService.cs diff --git a/csharp/GetAvailability/Services/ResourceHealthService.cs b/Old/GetAvailability/Services/ResourceHealthService.cs similarity index 100% rename from csharp/GetAvailability/Services/ResourceHealthService.cs rename to Old/GetAvailability/Services/ResourceHealthService.cs diff --git a/csharp/GetAvailability/Services/ResourceInventoryService.cs b/Old/GetAvailability/Services/ResourceInventoryService.cs similarity index 100% rename from csharp/GetAvailability/Services/ResourceInventoryService.cs rename to Old/GetAvailability/Services/ResourceInventoryService.cs diff --git a/csharp/GetAvailability/Services/SubscriptionResolver.cs b/Old/GetAvailability/Services/SubscriptionResolver.cs similarity index 100% rename from csharp/GetAvailability/Services/SubscriptionResolver.cs rename to Old/GetAvailability/Services/SubscriptionResolver.cs diff --git a/csharp/README.md b/Old/README.md similarity index 98% rename from csharp/README.md rename to Old/README.md index d951bf6..5973ade 100644 --- a/csharp/README.md +++ b/Old/README.md @@ -33,7 +33,7 @@ If Azure authentication fails, the tool prints the SDK exception message directl ## Build ```bash -cd csharp/GetAvailability +cd Old/GetAvailability # Debug (JIT, for development) dotnet build @@ -61,7 +61,7 @@ dotnet publish -c Release -r win-x64 # output in bin/Release/net10.0/win-x64/p ./GetAvailability --subscriptions Contoso-Production --month 202603 --workspace b233a4b7-3c43-433c-ac60-1f6ff217ddd4 # Run directly without publishing -cd csharp/GetAvailability +cd Old/GetAvailability dotnet run -- --subscriptions Contoso-Production --month 202603 ``` diff --git a/README.md b/README.md index 3fc6511..53fcf3a 100644 --- a/README.md +++ b/README.md @@ -9,8 +9,8 @@ Two implementations are provided: | Version | Path | Runtime | Notes | |---|---|---|---| -| **C#** | [`csharp/`](csharp/README.md) | .NET 10 Native AOT (~15 MB standalone binary, no runtime required) | Fastest; recommended for production use | -| **PowerShell** | `get-availability.ps1` | PowerShell 7+ with `Az.Accounts` and `Az.ResourceGraph` modules | No build step; convenient for ad-hoc use; supports Log Analytics ingestion | +| **C#** (legacy) | [`Old/`](Old/README.md) | .NET 10 Native AOT (~15 MB standalone binary, no runtime required) | Moved to `Old/`; not actively maintained | +| **PowerShell** | [`Functions/GetAvail/get-availability.ps1`](Functions/GetAvail/get-availability.ps1) | PowerShell 7+ with `Az.Accounts` and `Az.ResourceGraph` modules | No build step; convenient for ad-hoc use; supports Log Analytics ingestion; also runs as an Azure Function | Both versions share the same pipeline, classification rules, output format, and invariants. @@ -37,7 +37,7 @@ The relationship `Suspect = Faults + Excused + Unresolved` always holds. If Azure authentication fails, the tool prints the module exception message directly. Re-run `Connect-AzAccount` to fix. -For the C# version prerequisites and usage, see the [C# README](csharp/README.md). +For the C# version prerequisites and usage, see the [C# README](Old/README.md). ### Parameters @@ -56,7 +56,7 @@ For the C# version prerequisites and usage, see the [C# README](csharp/README.md | `-DcrImmutableId` | *(none)* | Data Collection Rule immutable ID. Required together with `-DceEndpoint` to enable Log Analytics ingestion. | | `-Version` | | Print version and exit | -For C# parameters, see the [C# README](csharp/README.md). +For C# parameters, see the [C# README](Old/README.md). The observation window is a UTC calendar month: past months use the full calendar month, the current month is reported month-to-date. The requested month cannot start more than 90 days before the current UTC time. Metrics and Activity Log support that 90-day lookback; Health History is applied only for its overlap with the ~30-day REST API retention window. When `-SourceWorkspaceId` / `--workspace` is used, Health History coverage extends to the full observation period via a hybrid approach (Log Analytics for older transitions + REST API for the last ~30 days). @@ -64,30 +64,30 @@ The observation window is a UTC calendar month: past months use the full calenda ```powershell # Single subscription -./get-availability.ps1 -Subscriptions 'Contoso-Production' -Month 202603 +./Functions/GetAvail/get-availability.ps1 -Subscriptions 'Contoso-Production' -Month 202603 # Multiple subscriptions, filtered by kind -./get-availability.ps1 -Subscriptions 'Contoso-Development','Contoso-Production' -Month 202603 -Kinds vm,sql +./Functions/GetAvail/get-availability.ps1 -Subscriptions 'Contoso-Development','Contoso-Production' -Month 202603 -Kinds vm,sql # Single resource with custom grace window -./get-availability.ps1 -Subscriptions 'Contoso-Development' -Month 202603 -Resource myvm02 -ActivityGraceMinutes 15 +./Functions/GetAvail/get-availability.ps1 -Subscriptions 'Contoso-Development' -Month 202603 -Resource myvm02 -ActivityGraceMinutes 15 # Batch API with custom batch size -./get-availability.ps1 -Subscriptions 'Contoso-Production','Contoso-Development' -Month 202603 -BatchSize 20 +./Functions/GetAvail/get-availability.ps1 -Subscriptions 'Contoso-Production','Contoso-Development' -Month 202603 -BatchSize 20 # Use Log Analytics for Activity Log + Resource Health (faster, extended retention) -./get-availability.ps1 -Subscriptions 'Contoso-Production' -Month 202603 -SourceWorkspaceId 'b233a4b7-3c43-433c-ac60-1f6ff217ddd4' +./Functions/GetAvail/get-availability.ps1 -Subscriptions 'Contoso-Production' -Month 202603 -SourceWorkspaceId 'b233a4b7-3c43-433c-ac60-1f6ff217ddd4' # Send results to Log Analytics custom tables -./get-availability.ps1 -Subscriptions 'Contoso-Production' -Month 202603 ` +./Functions/GetAvail/get-availability.ps1 -Subscriptions 'Contoso-Production' -Month 202603 ` -DceEndpoint 'https://dce-getavail-itn-001.italynorth-1.ingest.monitor.azure.com' ` -DcrImmutableId 'dcr-00000000000000000000000000000000' # Pipe results to CSV -./get-availability.ps1 -Subscriptions 'Contoso-Production' -Month 202603 | Export-Csv availability.csv +./Functions/GetAvail/get-availability.ps1 -Subscriptions 'Contoso-Production' -Month 202603 | Export-Csv availability.csv ``` -For C# examples, see the [C# README](csharp/README.md). +For C# examples, see the [C# README](Old/README.md). ### Output @@ -243,7 +243,7 @@ AvailabilityPct = 40,066 / 40,125 × 100 = 99.85390% ## Implementation notes -These notes cover performance and implementation details specific to the PowerShell version. For C# implementation notes, see the [C# README](csharp/README.md). +These notes cover performance and implementation details specific to the PowerShell version. For C# implementation notes, see the [C# README](Old/README.md). - **`ForEach-Object -Parallel`** for concurrent metric, Activity Log, and Resource Health queries with configurable parallelism. - **Shared `HttpClient`** with connection pooling — avoids per-request TCP/TLS overhead; streams JSON responses directly into `System.Text.Json` without intermediate string allocation. Used for both per-resource metrics and gap investigation paths. @@ -266,4 +266,132 @@ When `-DceEndpoint` and `-DcrImmutableId` are provided, the script sends results Authentication uses `Get-AzAccessToken -ResourceUrl 'https://monitor.azure.com'`, which works identically for interactive sessions (`Connect-AzAccount`) and Azure Function managed identities. Payloads are gzip-compressed and batched at 900 KB to stay within API limits. -The infrastructure (Log Analytics workspace, custom tables, DCE, DCR) is deployed via the Bicep templates in the [`Setup/`](Setup/README.md) directory. +The infrastructure is deployed via the Bicep template in [`Bicep/`](Bicep/). The template creates the full stack (Log Analytics workspace, custom tables, DCE, DCR, Storage Account, Function App, Application Insights, Private Endpoints, and RBAC) and **auto-wires all Function App settings** — after deployment and `func publish`, the function runs with no manual configuration. + +## Infrastructure Deployment + +### What it deploys + +The Bicep template deploys the following resources into the target resource group: + +| # | Resource | Purpose | +|---|----------|--------| +| 1 | **Log Analytics Workspace** | Stores availability data in custom tables; enables KQL queries and Workbooks | +| 2 | **Custom Table `GetAvailResources_CL`** | Per-resource availability detail (one row per resource per run) | +| 3 | **Custom Table `GetAvailSummary_CL`** | Aggregated summaries (per Kind+Location, per subscription, overall) | +| 4 | **Data Collection Endpoint (DCE)** | Ingestion URL for the Azure Monitor Ingestion API | +| 5 | **Data Collection Rule (DCR)** | Routes two custom streams to the corresponding tables with `TimeGenerated` injection | +| 6 | **Storage Account** | Backing store for the Function App (deployment blobs) | +| 7 | **Flex Consumption Plan** | Serverless hosting plan for the Function App | +| 8 | **Function App** | Runs the Get-Availability script on a schedule with system-assigned managed identity | +| 9 | **Application Insights** | Monitoring and telemetry for the Function App (Entra-only auth) | +| 10 | **Private Endpoint (Storage blob)** | Private connectivity for the Function App to its backing storage | +| 11 | **Private Endpoint (Function App sites)** | Private connectivity for publishing and management | + +RBAC role assignments are created automatically: + +| Principal | Role | Scope | Why | +|-----------|------|-------|-----| +| Function App | Monitoring Metrics Publisher | DCR | Ingest custom logs via the Azure Monitor Ingestion API | +| Function App | Monitoring Metrics Publisher | Application Insights | Send telemetry when local auth is disabled | +| Function App | Storage Blob Data Owner | Storage Account | Flex Consumption plan deployment blobs | + +### Deployment prerequisites + +- **Azure CLI** with Bicep support (`az bicep version`) +- **Contributor** role on the target resource group +- A **subnet** delegated to `Microsoft.App/environments` for the Function App VNet integration +- A **subnet** for private endpoints +- Existing **Private DNS Zones** for `privatelink.blob.core.windows.net` and `privatelink.azurewebsites.net` + +### Bicep parameters + +Configured in `Bicep/parameters.dev.bicepparam`: + +| Parameter | Description | Example | +|-----------|-------------|---------| +| `location` | Azure region (defaults to resource group location) | `italynorth` | +| `logAnalyticsWorkspaceName` | Log Analytics workspace name | `log-getavail-itn-001` | +| `dataCollectionEndpointName` | Data Collection Endpoint name | `dce-getavail-itn-001` | +| `dataCollectionRuleName` | Data Collection Rule name | `dcr-getavail-itn-001` | +| `storageAccountName` | Storage account for the Function App | `stgetavailitn001` | +| `functionAppName` | Function App name | `fn-getavail-itn-001` | +| `applicationInsightsName` | Application Insights name | `appi-getavail-itn-001` | +| `fnSubnetId` | Subnet resource ID for Function App VNet integration | `/subscriptions/.../subnets/snet-fn` | +| `peSubnetId` | Subnet resource ID for private endpoints | `/subscriptions/.../subnets/snet-pe` | +| `dnsZonesSubscriptionId` | Subscription ID containing Private DNS Zones | `00000000-0000-...` | +| `dnsZonesResourceGroupName` | Resource group containing Private DNS Zones | `rg-dns-001` | +| `getavailSubscriptions` | Comma-separated subscription names/IDs to monitor | `Contoso-Production,Contoso-Dev` | +| `getavailKinds` | Resource kinds to monitor (default: `vm,sql,storage,webapp`) | `vm,sql` | +| `sourceWorkspaceId` | Log Analytics workspace ID for Activity Log / Resource Health queries (optional) | `f25755bb-...` | +| `timerSchedule` | CRON expression for the timer trigger (default: `0 0 6 1 * *` — 6 AM on the 1st of every month) | `0 0 8 1 * *` | + +### Deploy + +This is a **resource-group scoped** deployment. Create the resource group first, then deploy: + +```powershell +# Create resource group (one-time) +az group create --name rg-getavail-itn-001 --location italynorth --tags solution=Get-Availability + +# Validate +az deployment group validate --resource-group rg-getavail-itn-001 --parameters Bicep/parameters.dev.bicepparam + +# What-if (dry run) +az deployment group what-if --resource-group rg-getavail-itn-001 --parameters Bicep/parameters.dev.bicepparam + +# Deploy +az deployment group create --resource-group rg-getavail-itn-001 --parameters Bicep/parameters.dev.bicepparam +``` + +### Auto-wired app settings + +The Bicep template configures the Function App with all required settings — values are resolved from sibling resources at deploy time: + +| App Setting | Bicep source | Used by `run.ps1` | +|---|---|---| +| `DCE_ENDPOINT` | DCE ingestion endpoint | `$env:DCE_ENDPOINT` | +| `DCR_IMMUTABLE_ID` | DCR immutable ID | `$env:DCR_IMMUTABLE_ID` | +| `SOURCE_WORKSPACE_ID` | `sourceWorkspaceId` parameter | `$env:SOURCE_WORKSPACE_ID` | +| `GETAVAIL_SUBSCRIPTIONS` | `getavailSubscriptions` parameter | `$env:GETAVAIL_SUBSCRIPTIONS` | +| `GETAVAIL_KINDS` | `getavailKinds` parameter | `$env:GETAVAIL_KINDS` | +| `TIMER_SCHEDULE` | `timerSchedule` parameter | *(timer trigger via `%TIMER_SCHEDULE%`)* | +| `APPLICATIONINSIGHTS_CONNECTION_STRING` | App Insights connection string | *(Functions runtime)* | + +No manual post-deployment configuration is required. + +The template also configures CORS to allow `https://portal.azure.com`, so you can test-run the function directly from the Azure Portal. + +### Deployment outputs + +Outputs are available for reference or cross-stack integration: + +| Output | Description | +|--------|-------------| +| `logAnalyticsWorkspaceId` | Workspace resource ID | +| `dceIngestionEndpoint` | DCE ingestion URL | +| `dataCollectionRuleImmutableId` | DCR immutable ID | +| `functionAppId` | Function App resource ID | +| `applicationInsightsId` | Application Insights resource ID | +| `storageAccountId` | Storage Account resource ID | + +```powershell +# Retrieve outputs +$outputs = (az deployment group show --resource-group rg-getavail-itn-001 --name getavailability --query properties.outputs -o json | ConvertFrom-Json) +$outputs.dceIngestionEndpoint.value +$outputs.dataCollectionRuleImmutableId.value +``` + +### Publishing the Function App + +The `get-availability.ps1` script lives inside the function app folder (`Functions/GetAvail/`) and is deployed alongside the function code. All app settings (`DCE_ENDPOINT`, `DCR_IMMUTABLE_ID`, `SOURCE_WORKSPACE_ID`, `GETAVAIL_SUBSCRIPTIONS`, `GETAVAIL_KINDS`) are auto-wired by Bicep — after `func publish` the function is ready to run with no manual configuration. + +```powershell +# Save required modules (one-time or when upgrading) +Save-Module -Name Az.Accounts -Path Functions/GetAvail/Modules -Repository PSGallery -Force +Save-Module -Name Az.ResourceGraph -Path Functions/GetAvail/Modules -Repository PSGallery -Force + +# Publish (from the Functions/GetAvail directory) +cd Functions/GetAvail +func azure functionapp publish fn-getavail-itn-001 --powershell +``` diff --git a/Setup/PLAN-log-analytics-ingestion.md b/Setup/PLAN-log-analytics-ingestion.md deleted file mode 100644 index fcf51b2..0000000 --- a/Setup/PLAN-log-analytics-ingestion.md +++ /dev/null @@ -1,289 +0,0 @@ -# Plan: Ingest Get-Availability Results into Log Analytics Custom Tables - -## Goal - -After each run of Get-Availability, push the per-resource detail rows **and** the -aggregated summary rows into custom tables in a Log Analytics workspace so that -results can be queried with KQL, visualized in Workbooks, and trended over time. - ---- - -## 1. Custom Tables - -### Table A — `GetAvailResources_CL` (per-resource detail) - -One row per resource per run. Carries the full availability breakdown. - -| Column | Type | Source / Notes | -|----------------------------|------------|----------------------------------------------------------| -| `TimeGenerated` | datetime | Injected by DCR transform: `now()` | -| `RunId` | string | GUID generated once per execution (correlates all rows) | -| `Month` | string | Observation month, e.g. `"202604"` | -| `PeriodStart` | datetime | UTC start of observation window | -| `PeriodEnd` | datetime | UTC end of observation window | -| `IsMonthToDate` | boolean | `true` if the run was mid-month | -| `SubscriptionName` | string | Azure subscription display name | -| `ResourceName` | string | Resource name | -| `ResourceId` | string | Full ARM resource ID | -| `ResourceGroup` | string | Resource group name | -| `Kind` | string | `VirtualMachine`, `AzureSqlDatabase`, `StorageAccount`, `WebApp` | -| `Location` | string | Azure region | -| `EligibleMinutes` | int | Minutes eligible for availability measurement | -| `AvailableMinutes` | real | Actual available minutes (may be fractional due to degraded datapoints) | -| `SuspectMinutes` | int | Total suspect minutes from metric scan | -| `ConfirmedDowntimeMinutes` | int | Platform-fault minutes confirmed by Resource Health | -| `ExcusedMinutes` | int | Minutes excused from eligibility (lifecycle, customer, metric issues, zero-tx) | -| `UnexplainedSuspectMinutes`| int | Suspect minutes remaining after all classification | -| `AvailabilityPct` | real | Availability percentage (5 decimal places); -1 for N/A resources | - -### Table B — `GetAvailSummary_CL` (aggregated summaries) - -One row per aggregation group per run. Stores the subscription-level and -cross-subscription roll-ups. - -| Column | Type | Source / Notes | -|---------------------|------------|-------------------------------------------------------------------| -| `TimeGenerated` | datetime | Injected by DCR transform: `now()` | -| `RunId` | string | Same GUID as the detail rows (for correlation) | -| `Month` | string | Observation month, e.g. `"202604"` | -| `PeriodStart` | datetime | UTC start of observation window | -| `PeriodEnd` | datetime | UTC end of observation window | -| `IsMonthToDate` | boolean | `true` if mid-month | -| `SummaryLevel` | string | `KindLocation` / `SubscriptionTotal` / `Overall` | -| `SubscriptionName` | string | Subscription name (empty for `Overall` rows) | -| `Kind` | string | Resource kind (empty for `SubscriptionTotal` and `Overall` rows) | -| `Location` | string | Azure region (empty for `SubscriptionTotal` and `Overall` rows) | -| `ResourceCount` | int | Number of resources in this group | -| `EligibleMinutes` | real | Sum of eligible minutes across resources in the group | -| `AvailableMinutes` | real | Sum of available minutes across resources in the group | -| `AvailabilityPct` | real | Aggregate availability percentage for the group | - ---- - -## 2. Data Collection Endpoint (DCE) - -A single DCE is created to provide the ingestion URL. Both tables will share -this endpoint. Network access will be set to `Enabled` (can be locked down -later with Private Link if needed). - -**Resource:** `Microsoft.Insights/dataCollectionEndpoints` - ---- - -## 3. Data Collection Rule (DCR) - -A single DCR declares **two custom streams** — one per table — and routes each -stream to the corresponding custom table in the workspace. - -| Stream | Target Table | -|-----------------------------------|--------------------------| -| `Custom-GetAvailResources_CL` | `GetAvailResources_CL` | -| `Custom-GetAvailSummary_CL` | `GetAvailSummary_CL` | - -Each stream has a `transformKql` that injects `TimeGenerated = now()` and -performs any necessary type coercion (e.g. `todatetime()` on the period -timestamps). - ---- - -## 4. Bicep File: `getavailability.bicep` - -The Bicep file uses `targetScope = 'subscription'` so it can create its own -dedicated resource group. It receives parameters for the resource group name, -Log Analytics workspace name, region, DCE name, and DCR name. - -It creates (in order): - -1. Resource group (dedicated to Get-Availability telemetry) -2. Log Analytics workspace (inside the new resource group) -3. Custom table `GetAvailResources_CL` (child of the workspace) -4. Custom table `GetAvailSummary_CL` (child of the workspace) -5. Data Collection Endpoint -6. Data Collection Rule (with `dependsOn` on both tables and the DCE) - -Because the scope is `subscription`, the resource group is created via a -top-level `resource` declaration, and all other resources are deployed via a -Bicep module (or nested `module` with `scope: resourceGroup(...)`) targeting -the newly created resource group. - -Deployment command changes accordingly: -```powershell -# Subscription-scoped deployment (no --resource-group flag) -az deployment sub create --location --parameters .\parameters.dev.bicepparam -``` - -The Bicep file outputs: -- Resource group name -- Log Analytics workspace ID -- DCE ingestion endpoint URL -- DCR immutable ID -- DCR stream names (for the caller / script to use when posting data) - ---- - -## 5. Implementation Steps (for later) - -- [ ] **Step 1:** Write `getavailability.bicep` with resource group + Log Analytics workspace + custom tables + DCE + DCR - - `targetScope = 'subscription'` - - Create the resource group first, then deploy all other resources into it - (using a Bicep module scoped to the new resource group) -- [ ] **Step 2:** Update `parameters.dev.bicepparam`: - - Change the `using` directive from `'./certlc.bicep'` → `'./getavailability.bicep'` - - Remove **all** CertLC-specific parameters: - - `peSubnetId`, `fnSubnetId` (no private endpoints or function apps) - - `dnsZonesSubscriptionId`, `dnsZonesResourceGroupName` (no private DNS) - - `storageAccountName`, `functionAppName`, `applicationInsightsName` (not used) - - `automationAccountName`, `hybridWorkerGroupName`, `runbookName` (not used) - - `keyVaultName` (not used) - - All `automationAccountVar*` parameters (not used) - - Add **all** parameters required by `getavailability.bicep`: - - `resourceGroupName` — name of the dedicated resource group to create - - `location` — Azure region for all resources - - `logAnalyticsWorkspaceName` — name of the Log Analytics workspace to create - - `dataCollectionEndpointName` — name for the DCE - - `dataCollectionRuleName` — name for the DCR - - The resulting file should be minimal, e.g.: - ```bicepparam - using './getavailability.bicep' - param resourceGroupName = 'rg-getavail-itn-001' - param location = 'italynorth' - param logAnalyticsWorkspaceName = 'log-getavail-itn-001' - param dataCollectionEndpointName = 'dce-getavail-itn-001' - param dataCollectionRuleName = 'dcr-getavail-itn-001' - ``` -- [ ] **Step 3:** Add new **optional** parameters to `get-availability.ps1`: - - `-DceEndpoint [string]` — DCE logs ingestion URL (from Bicep output `dceIngestionEndpoint`) - - `-DcrImmutableId [string]` — DCR immutable ID (from Bicep output `dataCollectionRuleImmutableId`) - - Both are optional; ingestion happens **only** when both are provided - - When omitted, the script behaves exactly as today (console output only) - - Add validation: if one is supplied without the other, throw an error - - Use a `$sendToLogAnalytics = $DceEndpoint -and $DcrImmutableId` flag to guard all - ingestion code paths — zero overhead when ingestion is not requested -- [ ] **Step 4:** Add a helper function `Send-ToLogAnalytics` in `get-availability.ps1`: - - Uses the **Azure Monitor Ingestion** REST API - (`POST https://{dce-endpoint}/dataCollectionRules/{dcr-immutableId}/streams/{streamName}?api-version=2023-01-01`) - - **Authentication — dual execution context:** - The script runs in two environments and must acquire an Azure Monitor token in both: - 1. **Interactive** (`az login` / `Connect-AzAccount`): use `Get-AzAccessToken` - with `-ResourceUrl 'https://monitor.azure.com'`, same pattern as the existing - ARM token acquisition in Step 2 - 2. **Azure Function** (managed identity): the `Az.Accounts` module is available in - the PowerShell worker; `Connect-AzAccount -Identity` is typically run at function - startup (or by the host), so `Get-AzAccessToken` works identically — no code change - needed for this path - Implementation: acquire the monitor token once near the top of the ingestion block - (right after the existing ARM token), using the same `Get-AzAccessToken` + - `SecureString` handling pattern already in the script: - ```powershell - $rawMonitor = (Get-AzAccessToken -ResourceUrl 'https://monitor.azure.com').Token - $monitorToken = ($rawMonitor -is [securestring]) ` - ? ($rawMonitor | ConvertFrom-SecureString -AsPlainText) : [string]$rawMonitor - $rawMonitor = $null - ``` - - Accepts: endpoint URL, DCR immutable ID, stream name, bearer token, and an array of - PSObjects (the payload) - - Serializes the array to JSON with `ConvertTo-Json -Depth 5 -Compress` - - Sets `Content-Type: application/json` and `Content-Encoding: gzip` (gzip the body for efficiency) - - Handles the Ingestion API's 1 MB per call limit: if the payload exceeds ~900 KB, split into batches - - Returns nothing on success (204); throws on failure with status code and body -- [ ] **Step 5:** Build and send the **per-resource detail** payload after Step 8 (output): - - Generate a `$runId = [guid]::NewGuid().ToString()` once per execution - - Map each `$eligByRes` entry to a hashtable matching the `Custom-GetAvailResources_CL` stream schema: - ``` - @{ - RunId = $runId - Month = $normalizedMonth - PeriodStart = $utcStart.ToString('o') - PeriodEnd = $utcEnd.ToString('o') - IsMonthToDate = $isMonthToDate - SubscriptionName = $elig.SubscriptionName - ResourceName = $elig.Name - ResourceId = $elig.ResourceId - ResourceGroup = $elig.ResourceGroupName - Kind = $elig.Kind - Location = $elig.Location - EligibleMinutes = $elig.EligibleMinutes - AvailableMinutes = $elig.AvailableMinutes - SuspectMinutes = $elig.SuspectMinutes - ConfirmedDowntimeMinutes = $elig.ConfirmedDowntimeMinutes - ExcusedMinutes = $elig.ExcusedMinutes - UnexplainedSuspectMinutes= $elig.UnexplainedSuspectMinutes - AvailabilityPct = if ($elig.AvailabilityPct -eq 'N/A') { -1 } else { [double]$elig.AvailabilityPct } - } - ``` - - Call `Send-ToLogAnalytics` with stream name `Custom-GetAvailResources_CL` -- [ ] **Step 6:** Build and send the **summary** payload: - - Reuse the same grouping logic already in `Write-SubscriptionSummaries` to produce rows: - - One row per Kind+Location per subscription (`SummaryLevel = 'KindLocation'`) - - One row per subscription total (`SummaryLevel = 'SubscriptionTotal'`) - - One overall row across all subscriptions (`SummaryLevel = 'Overall'`, only if >1 subscription) - - Each row is a hashtable matching the `Custom-GetAvailSummary_CL` stream schema - - Call `Send-ToLogAnalytics` with stream name `Custom-GetAvailSummary_CL` -- [ ] **Step 7:** Add RBAC: the caller identity needs **Monitoring Metrics Publisher** role on the DCR - - Document this in the script help text and in the Bicep file comments - - The Bicep file should optionally accept a principal ID to assign the role (or leave it as a manual step) -- [ ] **Step 8:** Rewrite `Setup/README.md` for Get-Availability (replace CertLC content): - - **Remove entirely** all CertLC-specific content: - - Title, description, and solution overview references to CertLC - - Prerequisites: VNet/subnet requirements, Private DNS Zones, Hybrid Worker VM - - RBAC section: Owner role, Private DNS Zone Contributor, all CertLC role assignment tables - - Resources Created: all 16 CertLC resources (Storage Account, Function App, Automation Account, - Key Vault, Event Grid, Private Endpoints, DNS Zone Groups, Workbook, etc.) - - Parameters table: all CertLC parameters (peSubnetId, fnSubnetId, dnsZones*, storage*, function*, - automation*, keyVault*, automationAccountVar*, scheduleStartTime) - - Post-Deployment Steps: hybrid worker registration, runbook upload, certlcstats schedule, - function app deployment, CA permissions, workbook customization, end-to-end testing - - Security Notes section (CertLC-specific) - - Manual Configuration (on-premises CA) section - - Files section listing `certlc.bicep` - - **Replace with** Get-Availability infrastructure content: - - Title: "Get-Availability Setup" (or similar) - - Purpose: deploys custom Log Analytics tables, DCE, and DCR for ingesting - Get-Availability script results - - Prerequisites: Contributor role on the subscription (to create the resource - group and resources), Monitoring Metrics Publisher on the DCR for the caller identity - - Deployment commands: subscription-scoped (`az deployment sub create --location - --parameters .\parameters.dev.bicepparam`) — no `--resource-group` flag - - Resources Created: 6 resources — 1 resource group, 1 Log Analytics workspace, - 2 custom tables, 1 DCE, 1 DCR - - Parameters table: `resourceGroupName`, `location`, `logAnalyticsWorkspaceName`, - `dataCollectionEndpointName`, `dataCollectionRuleName` - - Outputs: DCE ingestion endpoint, DCR immutable ID - - Post-Deployment: how to use `-DceEndpoint` / `-DcrImmutableId` with the script - - Files section listing `getavailability.bicep`, `parameters.dev.bicepparam`, and this README - ---- - -## Design Decisions & Rationale - -- **Two tables** instead of one: the per-resource table has ~17 columns with - detailed investigation fields that don't apply to summaries. The summary - table has `SummaryLevel`, `ResourceCount` etc. that don't apply to individual - resources. Separate tables keep KQL queries cleaner and avoid wide sparse rows. -- **RunId + Month** as correlation keys: allows querying "latest run for month - X" or "all runs for month X" (useful when mid-month runs are repeated). -- **AvailabilityPct as real (-1 for N/A):** avoids a string column that would - complicate numeric KQL queries. -1 signals excluded resources. -- **Single DCR with two streams:** reduces resource count and keeps routing - in one place. The Ingestion API supports specifying the stream name per call. -- **`TimeGenerated` via DCR transform:** standard Log Analytics pattern; - the script doesn't need to supply it. -- **PowerShell-only implementation:** the ingestion feature targets the - PowerShell script (`get-availability.ps1`). The C# version is not updated - for this feature. -- **Strictly optional ingestion:** when `-DceEndpoint` and `-DcrImmutableId` are - omitted, zero ingestion code runs. No token is acquired, no payloads are built, - no REST calls are made. Console output is always produced regardless. -- **Dual execution context (interactive + Azure Function):** the script already - depends on `Az.Accounts`. Both `az login` (interactive) and managed-identity - (Azure Function) contexts expose `Get-AzAccessToken`, so the same code path - acquires the `https://monitor.azure.com` bearer token in both environments. - No conditional logic or separate auth path is needed. -- **REST API over SDK:** using the Azure Monitor Ingestion REST API directly - (with `Invoke-RestMethod`) avoids adding a PowerShell module dependency. - Authentication reuses the existing `Az.Accounts` session via `Get-AzAccessToken`. -- **Subscription-scoped Bicep with dedicated resource group:** the Bicep creates - its own resource group so the deployment is self-contained — no pre-existing - resource group or workspace is needed. The deployer only needs Contributor - on the subscription. This also keeps Get-Availability telemetry resources - isolated from other workloads. diff --git a/Setup/README.md b/Setup/README.md deleted file mode 100644 index d7d3791..0000000 --- a/Setup/README.md +++ /dev/null @@ -1,107 +0,0 @@ -# Get-Availability Setup - -This directory contains the Bicep infrastructure-as-code templates for deploying the Log Analytics ingestion infrastructure used by the Get-Availability script. - -For a complete solution overview, pipeline description, and usage, see the [main README](../README.md). - -## What it deploys - -The Bicep template deploys the following resources into the target resource group: - -| # | Resource | Purpose | -|---|----------|--------| -| 1 | **Log Analytics Workspace** | Stores availability data in custom tables; enables KQL queries and Workbooks | -| 2 | **Custom Table `GetAvailResources_CL`** | Per-resource availability detail (one row per resource per run) | -| 3 | **Custom Table `GetAvailSummary_CL`** | Aggregated summaries (per Kind+Location, per subscription, overall) | -| 4 | **Data Collection Endpoint (DCE)** | Ingestion URL for the Azure Monitor Ingestion API | -| 5 | **Data Collection Rule (DCR)** | Routes two custom streams to the corresponding tables with `TimeGenerated` injection | - -## Prerequisites - -- **Azure CLI** with Bicep support (`az bicep version`) -- **Contributor** role on the target resource group -- After deployment, the identity running the script needs **Monitoring Metrics Publisher** role on the DCR - -## Parameters - -Configured in `parameters.dev.bicepparam`: - -| Parameter | Description | Example | -|-----------|-------------|---------| -| `location` | Azure region for all resources (defaults to resource group location) | `italynorth` | -| `logAnalyticsWorkspaceName` | Name of the Log Analytics workspace | `log-getavail-itn-001` | -| `dataCollectionEndpointName` | Name of the Data Collection Endpoint | `dce-getavail-itn-001` | -| `dataCollectionRuleName` | Name of the Data Collection Rule | `dcr-getavail-itn-001` | - -## Deployment - -This is a **resource-group scoped** deployment. Create the resource group first, then deploy: - -```powershell -# Create resource group (one-time) -az group create --name rg-getavail-itn-001 --location italynorth --tags solution=Get-Availability - -# Validate -az deployment group validate --resource-group rg-getavail-itn-001 --parameters .\parameters.dev.bicepparam - -# What-if (dry run) -az deployment group what-if --resource-group rg-getavail-itn-001 --parameters .\parameters.dev.bicepparam - -# Deploy -az deployment group create --resource-group rg-getavail-itn-001 --parameters .\parameters.dev.bicepparam -``` - -## Outputs - -After deployment, retrieve the values needed by the script: - -| Output | Description | -|--------|-------------| -| `logAnalyticsWorkspaceId` | Workspace resource ID | -| `dceIngestionEndpoint` | DCE ingestion URL (pass to `-DceEndpoint`) | -| `dataCollectionRuleImmutableId` | DCR immutable ID (pass to `-DcrImmutableId`) | - -```powershell -# Retrieve outputs -$outputs = (az deployment group show --resource-group rg-getavail-itn-001 --name getavailability --query properties.outputs -o json | ConvertFrom-Json) -$outputs.dceIngestionEndpoint.value -$outputs.dataCollectionRuleImmutableId.value -``` - -## Post-Deployment: RBAC - -Grant the caller identity **Monitoring Metrics Publisher** on the DCR: - -```powershell -az role assignment create ` - --assignee ` - --role "Monitoring Metrics Publisher" ` - --scope "/subscriptions//resourceGroups//providers/Microsoft.Insights/dataCollectionRules/" -``` - -## Using with the script - -Once deployed, pass the DCE endpoint and DCR immutable ID to the script: - -```powershell -# Interactive -./get-availability.ps1 -Subscriptions 'MySub' -Month 202604 ` - -DceEndpoint 'https://dce-getavail-itn-001.italynorth-1.ingest.monitor.azure.com' ` - -DcrImmutableId 'dcr-00000000000000000000000000000000' - -# From an Azure Function (managed identity — same parameters, auth is automatic) -./get-availability.ps1 -Subscriptions 'MySub' -Month 202604 ` - -DceEndpoint $env:DCE_ENDPOINT ` - -DcrImmutableId $env:DCR_IMMUTABLE_ID -``` - -When `-DceEndpoint` and `-DcrImmutableId` are omitted, the script produces console output only (no ingestion). - -## Files - -| File | Description | -|------|-------------| -| `getavailability.bicep` | Bicep template (resource-group scoped: workspace, tables, DCE, DCR) | -| `parameters.dev.bicepparam` | Parameter file for dev environment | -| `certlc.bicep` | CertLC infrastructure (separate solution, not related to Get-Availability) | -| `PLAN-log-analytics-ingestion.md` | Implementation plan for the ingestion feature | \ No newline at end of file diff --git a/Setup/certlc.bicep b/Setup/certlc.bicep deleted file mode 100644 index 34ba543..0000000 --- a/Setup/certlc.bicep +++ /dev/null @@ -1,1136 +0,0 @@ -/* - -CERTLC - Bicep file for deploying the required resources for the CERTLC solution. - -Validate with: az deployment group validate --resource-group -parameters .\parameters.dev.bicepparam -What-if: az deployment group what-if --resource-group -parameters .\parameters.dev.bicepparam -Deploy with: az deployment group create --resource-group -parameters .\parameters.dev.bicepparam - -*/ - -metadata name = 'CertLC Infrastructure' -metadata description = 'Azure infrastructure deployment for Certificate Lifecycle Management solution with automated certificate enrollment, renewal, and monitoring' - -targetScope = 'resourceGroup' - -@description('The Azure region where resources will be deployed. Defaults to the resource group location.') -param location string = resourceGroup().location - -@description('The resource ID of the subnet for private endpoint connections. Format: /subscriptions/{subscriptionId}/resourceGroups/{rgName}/providers/Microsoft.Network/virtualNetworks/{vnetName}/subnets/{subnetName}') -param peSubnetId string - -@description('The resource ID of the subnet for the function app VNet integration. Must be delegated to Microsoft.App/environments for Flex Consumption plans. Format: /subscriptions/{subscriptionId}/resourceGroups/{rgName}/providers/Microsoft.Network/virtualNetworks/{vnetName}/subnets/{subnetName}') -param fnSubnetId string - -@description('The subscription ID where existing Private DNS Zones are located (for privatelink zones). Format: GUID') -param dnsZonesSubscriptionId string - -@description('The resource group name containing existing Private DNS Zones (e.g., privatelink.blob.core.windows.net, privatelink.vaultcore.azure.net)') -param dnsZonesResourceGroupName string - -@description('The name of the storage account to create. Must be globally unique, 3-24 characters, lowercase letters and numbers only. Used for function app storage and certificate lifecycle queue.') -@minLength(3) -@maxLength(24) -param storageAccountName string - -@description('The name of the function app to create. Must be globally unique, 2-60 characters, alphanumerics and hyphens. Hosts the queue processor and automation triggers.') -@minLength(2) -@maxLength(60) -param functionAppName string - -@description('The name of the Log Analytics workspace for centralized logging and monitoring. Stores diagnostic logs, custom certificate statistics, and application telemetry.') -param logAnalyticsWorkspaceName string - -@description('The name of the Application Insights instance for function app monitoring and performance tracking.') -param applicationInsightsName string - -@description('The name of the Automation Account to create. 6-50 characters, alphanumerics and hyphens. Executes certificate lifecycle runbooks on hybrid workers.') -@minLength(6) -@maxLength(50) -param automationAccountName string - -@description('The name of the hybrid runbook worker group. On-premises workers must be registered to this group to execute certificate operations.') -param hybridWorkerGroupName string - -@description('The name of the runbook to invoke for certificate lifecycle operations. Must match the runbook name deployed to the Automation Account.') -param runbookName string - -@description('The name of the Key Vault to create. Must be globally unique, 3-24 characters, alphanumerics and hyphens. Stores and manages certificates with automated lifecycle tracking.') -@minLength(3) -@maxLength(24) -param keyVaultName string - -@description('The name of the Data Collection Endpoint (DCE) to create. Ingestion endpoint for custom certificate statistics logs sent from automation runbooks.') -param dataCollectionEndpointName string - -@description('The name of the Data Collection Rule (DCR) to create. Defines transformation and routing of certificate statistics to Log Analytics custom table.') -param dataCollectionRuleName string - -@description('The Certificate Authority name for certificate enrollment. Format: CA_SERVER\\\\CA_NAME (e.g., PKI-CA01\\\\ContosoRootCA). Used by runbooks for ADCS operations.') -param automationAccountVarCA string - -@description('The root folder path on hybrid workers where PFX certificates are stored. Format: UNC path or local path (e.g., \\\\\\\\fileserver\\\\certs or C:\\\\\\\\Certificates).') -param automationAccountVarPfxRootFolder string - -@description('The SMTP From email address for certificate expiration notifications (e.g., certlc@contoso.com).') -param automationAccountVarSmtpFrom string - -@description('The SMTP server hostname or IP address for sending email notifications (e.g., smtp.office365.com or smtp.gmail.com).') -param automationAccountVarSmtpServer string - -@description('The SMTP username for authentication to the mail server. Required if the SMTP server requires authentication.') -param automationAccountVarSmtpUser string - -@description('The SMTP password for authentication. Stored encrypted in Automation Account variables.') -@secure() -param automationAccountVarSmtpPassword string - -@description('The start time for the certlcstats schedule. Defaults to 15 minutes from deployment time.') -param scheduleStartTime string = dateTimeAdd(utcNow('u'), 'PT15M') - -/*************/ -/* VARIABLES */ -/*************/ - -// Common tags for all resources -var commonTags = { - solution: 'CertLC' - purpose: 'Certificate Lifecycle Management' -} - -// Azure built-in role definition IDs -var roleDefinitions = { - storageQueueDataReader: '19e7f393-937e-4f77-808e-94535e297925' - storageQueueDataMessageSender: 'c6a89b2d-59bc-44d0-9896-0f6e12d7b80a' - keyVaultCertificatesOfficer: 'a4417e6f-fecd-4de8-b567-7b0420556985' - keyVaultSecretsOfficer: 'b86a8fe4-44ce-4948-aee5-eccb2c155cd7' - reader: 'acdd72a7-3385-48ef-bd42-f606fba81ae7' - monitoringMetricsPublisher: '3913510d-42f4-4e42-8a64-420c390055eb' - storageBlobDataOwner: 'b7e6dc6d-f1e8-4753-8033-0f276bb0955b' - storageQueueDataMessageProcessor: '8a0f0c08-91a1-4084-bc3d-661d67233fed' - storageQueueDataContributor: '974c5e8b-45b9-4653-ba55-5f855dd0fb88' - automationOperator: 'd3881f73-407a-4167-8283-e981cbba0404' -} - -/**********************/ -/* EXISTING RESOURCES */ -/**********************/ - -// References to existing Private DNS Zones in their subscription -resource blobDnsZone 'Microsoft.Network/privateDnsZones@2020-06-01' existing = { - name: 'privatelink.blob.${environment().suffixes.storage}' - scope: resourceGroup(dnsZonesSubscriptionId, dnsZonesResourceGroupName) -} - -resource keyVaultDnsZone 'Microsoft.Network/privateDnsZones@2020-06-01' existing = { - name: 'privatelink.vaultcore.azure.net' - scope: resourceGroup(dnsZonesSubscriptionId, dnsZonesResourceGroupName) -} - -resource queueDnsZone 'Microsoft.Network/privateDnsZones@2020-06-01' existing = { - name: 'privatelink.queue.${environment().suffixes.storage}' - scope: resourceGroup(dnsZonesSubscriptionId, dnsZonesResourceGroupName) -} - -resource webAppDnsZone 'Microsoft.Network/privateDnsZones@2020-06-01' existing = { - name: 'privatelink.azurewebsites.net' - scope: resourceGroup(dnsZonesSubscriptionId, dnsZonesResourceGroupName) -} - -resource automationAccountDnsZone 'Microsoft.Network/privateDnsZones@2020-06-01' existing = { - name: 'privatelink.azure-automation.net' - scope: resourceGroup(dnsZonesSubscriptionId, dnsZonesResourceGroupName) -} - -/*****************/ -/* NEW RESOURCES */ -/*****************/ - -// Storage Account -resource storageAccount 'Microsoft.Storage/storageAccounts@2025-01-01' = { - name: storageAccountName - location: location - sku: { - name: 'Standard_LRS' - } - kind: 'StorageV2' - properties: { - accessTier: 'Hot' - defaultToOAuthAuthentication: true - allowBlobPublicAccess: false - allowSharedKeyAccess: false - minimumTlsVersion: 'TLS1_2' - supportsHttpsTrafficOnly: true - networkAcls: { - bypass: 'AzureServices' - defaultAction: 'Deny' - } - publicNetworkAccess: 'Disabled' - encryption: { - services: { - blob: { - enabled: true - } - queue: { - enabled: true - } - } - } - } - resource blobServices 'blobServices' = { - name: 'default' - properties: {} - } - resource queueServices 'queueServices' = { - name: 'default' - properties: {} - resource queues 'queues' = { - name: 'certlc' - properties: {} - } - } - - tags: commonTags -} - -// Private endpoint for the storage account - blob -resource storageAccountBlobPrivateEndpoint 'Microsoft.Network/privateEndpoints@2024-10-01' = { - name: 'pe-blob-${storageAccountName}' - location: location - properties: { - subnet: { - id: peSubnetId - } - privateLinkServiceConnections: [ - { - name: 'pls-${storageAccountName}' - properties: { - privateLinkServiceId: storageAccount.id - groupIds: [ - 'blob' - ] - } - } - ] - customNetworkInterfaceName: 'nic-pe-${storageAccountName}' - } - tags: commonTags - - resource privateDnsZoneGroup 'privateDnsZoneGroups' = { - name: 'default' - properties: { - privateDnsZoneConfigs: [ - { - name: 'config1' - properties: { - privateDnsZoneId: blobDnsZone.id - } - } - ] - } - } -} - -// Private endpoint for the storage account - queue -resource storageAccountQueuePrivateEndpoint 'Microsoft.Network/privateEndpoints@2024-10-01' = { - name: 'pe-queue-${storageAccountName}' - location: location - properties: { - subnet: { - id: peSubnetId - } - privateLinkServiceConnections: [ - { - name: 'pls-${storageAccountName}' - properties: { - privateLinkServiceId: storageAccount.id - groupIds: [ - 'queue' - ] - } - } - ] - customNetworkInterfaceName: 'nic-pe-queue-${storageAccountName}' - } - tags: commonTags - - resource privateDnsZoneGroup 'privateDnsZoneGroups' = { - name: 'default' - properties: { - privateDnsZoneConfigs: [ - { - name: 'config1' - properties: { - privateDnsZoneId: queueDnsZone.id - } - } - ] - } - } -} - -// Log Analytics Workspace -resource logAnalyticsWorkspace 'Microsoft.OperationalInsights/workspaces@2023-09-01' = { - name: logAnalyticsWorkspaceName - location: location - properties: { - sku: { - name: 'PerGB2018' - } - retentionInDays: 30 - } - tags: commonTags -} - -// Data Collection Endpoint -resource dataCollectionEndpoint 'Microsoft.Insights/dataCollectionEndpoints@2023-03-11' = { - name: dataCollectionEndpointName - location: location - properties: { - networkAcls: { - publicNetworkAccess: 'Enabled' - } - } - tags: commonTags -} - -// Custom Table for Certificate Statistics -resource customTable 'Microsoft.OperationalInsights/workspaces/tables@2022-10-01' = { - name: 'certlc_CL' - parent: logAnalyticsWorkspace - properties: { - retentionInDays: 30 - schema: { - name: 'certlc_CL' - columns: [ - { - name: 'TimeGenerated' - type: 'datetime' - } - { - name: 'Thumbprint' - type: 'string' - } - { - name: 'Name' - type: 'string' - } - { - name: 'Created' - type: 'datetime' - } - { - name: 'Expires' - type: 'datetime' - } - { - name: 'Subject' - type: 'string' - } - { - name: 'Template' - type: 'string' - } - { - name: 'DNSNames' - type: 'string' - } - ] - } - } -} - -// Data Collection Rule for Certificate Statistics -resource dataCollectionRule 'Microsoft.Insights/dataCollectionRules@2023-03-11' = { - name: dataCollectionRuleName - location: location - properties: { - dataCollectionEndpointId: dataCollectionEndpoint.id - streamDeclarations: { - 'Custom-certlc_CL': { - columns: [ - { - name: 'Thumbprint' - type: 'string' - } - { - name: 'Name' - type: 'string' - } - { - name: 'Created' - type: 'datetime' - } - { - name: 'Expires' - type: 'datetime' - } - { - name: 'Subject' - type: 'string' - } - { - name: 'Template' - type: 'string' - } - { - name: 'DNSNames' - type: 'string' - } - ] - } - } - destinations: { - logAnalytics: [ - { - workspaceResourceId: logAnalyticsWorkspace.id - name: 'clv2ws1' - } - ] - } - dataFlows: [ - { - streams: [ - 'Custom-certlc_CL' - ] - destinations: [ - 'clv2ws1' - ] - transformKql: 'source | extend Created = todatetime(Created), Expires = todatetime(Expires) | extend TimeGenerated = now()' - outputStream: 'Custom-certlc_CL' - } - ] - } - dependsOn: [ - customTable // the DCR must be created after the custom table - ] - tags: commonTags -} - -// Application Insights -// IMPORTANT: Deploy AFTER all Log Analytics operations are complete to avoid "Workspace not active" errors -resource applicationInsights 'Microsoft.Insights/components@2020-02-02' = { - name: applicationInsightsName - location: location - kind: 'web' - properties: { - Application_Type: 'web' - WorkspaceResourceId: logAnalyticsWorkspace.id - DisableLocalAuth: true - } - dependsOn: [ - // Force serial deployment: Log Analytics → Custom Table → DCR → Automation Account → Diagnostics → App Insights - // This ensures the workspace backend is fully active before App Insights connects - automationAccountDiagnostics // Wait for diagnostic settings which write to workspace - keyVaultDiagnostics - ] - tags: commonTags -} - -// Flexible Consumption Plan for the function app -resource flexServicePlan 'Microsoft.Web/serverfarms@2024-11-01' = { - name: 'asp-${functionAppName}' - location: location - kind: 'functionapp' - sku: { - tier: 'FlexConsumption' - name: 'FC1' - } - properties: { - reserved: true - } - tags: commonTags -} - -// Function App -resource functionApp 'Microsoft.Web/sites@2024-11-01' = { - name: functionAppName - location: location - kind: 'functionapp,linux' - identity: { - type: 'SystemAssigned' - } - properties: { - serverFarmId: flexServicePlan.id - httpsOnly: true - virtualNetworkSubnetId: fnSubnetId - publicNetworkAccess: 'Disabled' - siteConfig: { - minTlsVersion: '1.2' - } - functionAppConfig: { - deployment: { - storage: { - type: 'blobContainer' - value: '${storageAccount.properties.primaryEndpoints.blob}azure-webjobs-hosts' - authentication: { - type: 'SystemAssignedIdentity' - } - } - } - scaleAndConcurrency: { - maximumInstanceCount: 100 - instanceMemoryMB: 2048 - } - runtime: { - name: 'powerShell' - version: '7.4' - } - } - } - resource appSettings 'config' = { - name: 'appsettings' - properties: { - AutomationAccountName: automationAccount.name - HybridWorkerGroupName: hybridWorkerGroupName - RunbookName: runbookName - ResourceGroupName: resourceGroup().name - AzureWebJobsStorage__credential: 'managedidentity' - AzureWebJobsStorage__blobServiceUri: storageAccount.properties.primaryEndpoints.blob - AzureWebJobsStorage__queueServiceUri: storageAccount.properties.primaryEndpoints.queue - APPLICATIONINSIGHTS_AUTHENTICATION_STRING: 'Authorization=AAD' - APPLICATIONINSIGHTS_CONNECTION_STRING: applicationInsights.properties.ConnectionString - } - } - dependsOn: [ - storageAccountBlobPrivateEndpoint // create the function only after the PEs for the storage account are ready - storageAccountQueuePrivateEndpoint - ] - tags: commonTags -} - -// Private endpoint for the function app -resource functionAppPrivateEndpoint 'Microsoft.Network/privateEndpoints@2024-10-01' = { - name: 'pe-sites-${functionAppName}' - location: location - properties: { - subnet: { - id: peSubnetId - } - privateLinkServiceConnections: [ - { - name: 'pls-${functionAppName}' - properties: { - privateLinkServiceId: functionApp.id - groupIds: [ - 'sites' - ] - } - } - ] - customNetworkInterfaceName: 'nic-pe-${functionAppName}' - } - tags: commonTags - - resource privateDnsZoneGroup 'privateDnsZoneGroups' = { - name: 'default' - properties: { - privateDnsZoneConfigs: [ - { - name: 'config1' - properties: { - privateDnsZoneId: webAppDnsZone.id - } - } - ] - } - } -} - -// Automation Account with its managed identity -resource automationAccount 'Microsoft.Automation/automationAccounts@2024-10-23' = { - name: automationAccountName - location: location - identity: { - type: 'SystemAssigned' - } - properties: { - publicNetworkAccess: false - sku: { - name: 'Basic' - } - } - dependsOn: [ - dataCollectionRule - dataCollectionEndpoint - ] - tags: commonTags - // variables - resource automationAccountVariables 'variables@2024-10-23' = { - name: 'certlc-ca' - properties: { - value: '"${replace(automationAccountVarCA, '\\', '\\\\')}"' - isEncrypted: true - } - } - resource automationAccountVariablesPfxRootFolder 'variables@2024-10-23' = { - name: 'certlc-pfxrootfolder' - properties: { - value: '"${replace(automationAccountVarPfxRootFolder, '\\', '\\\\')}"' - isEncrypted: true - } - } - resource automationAccountVariablesSmtpFrom 'variables@2024-10-23' = { - name: 'certlc-smtpfrom' - properties: { - value: '"${replace(automationAccountVarSmtpFrom, '\\', '\\\\')}"' - isEncrypted: true - } - } - resource automationAccountVariablesSmtpServer 'variables@2024-10-23' = { - name: 'certlc-smtpserver' - properties: { - value: '"${replace(automationAccountVarSmtpServer, '\\', '\\\\')}"' - isEncrypted: true - } - } - resource automationAccountVariablesSmtpUser 'variables@2024-10-23' = { - name: 'certlc-smtpuser' - properties: { - value: '"${replace(automationAccountVarSmtpUser, '\\', '\\\\')}"' - isEncrypted: true - } - } - resource automationAccountVariablesSmtpPassword 'variables@2024-10-23' = { - name: 'certlc-smtppassword' - properties: { - value: '"${replace(automationAccountVarSmtpPassword, '\\', '\\\\')}"' - isEncrypted: true - } - } - resource automationAccountVariablesKeyVault 'variables@2024-10-23' = { - name: 'certlc-stats-keyvault' - properties: { - value: '"${keyVault.name}"' - isEncrypted: true - } - } - resource automationAccountVariablesImmutableId 'variables@2024-10-23' = { - name: 'certlc-stats-immutableid' - properties: { - value: '"${dataCollectionRule.properties.immutableId}"' - isEncrypted: true - } - } - resource automationAccountVariablesStreamName 'variables@2024-10-23' = { - name: 'certlc-stats-streamname' - properties: { - value: '"Custom-certlc_CL"' - isEncrypted: true - } - } - resource automationAccountVariablesIngestionUrl 'variables@2024-10-23' = { - name: 'certlc-stats-ingestionurl' - properties: { - value: '"${dataCollectionEndpoint.properties.logsIngestion.endpoint}"' - isEncrypted: true - } - } - - // Runbook: certlc - resource runbookCertLC 'runbooks@2024-10-23' = { - name: 'certlc' - location: location - properties: { - runbookType: 'PowerShell' - logProgress: false - logVerbose: false - description: 'Certificate lifecycle management runbook for enrollment, renewal, and revocation' - runtimeEnvironment: 'PowerShell-7.2' - } - tags: commonTags - } - - // Runbook: certlcstats - resource runbookCertLCStats 'runbooks@2024-10-23' = { - name: 'certlcstats' - location: location - properties: { - runbookType: 'PowerShell' - logProgress: false - logVerbose: false - description: 'Certificate statistics collection runbook for monitoring and reporting' - runtimeEnvironment: 'PowerShell-7.2' - } - tags: commonTags - } - - // Schedule for certlcstats runbook - runs every hour - // Note: Schedule is created but NOT linked to runbook initially (disabled state) - // To enable: Link the schedule to the runbook in Azure Portal or via Azure CLI - resource scheduleCertLCStats 'schedules@2024-10-23' = { - name: 'schedule-certlcstats-hourly' - properties: { - description: 'Runs certlcstats runbook every hour to collect certificate statistics (manually link to enable)' - startTime: scheduleStartTime - frequency: 'Hour' - interval: 1 - timeZone: 'UTC' - } - } - - // Uncomment to automatically link schedule to runbook (enables automatic execution on hybrid worker group) - // resource jobScheduleCertLCStats 'jobSchedules@2024-10-23' = { - // name: guid(automationAccount.id, 'certlcstats-schedule') - // properties: { - // runbook: { - // name: runbookCertLCStats.name - // } - // schedule: { - // name: scheduleCertLCStats.name - // } - // runOn: hybridWorkerGroupName // Execute on hybrid worker group (not Azure sandbox) - // } - // } -} - -// Hybrid Worker Group -resource hybridWorkerGroup 'Microsoft.Automation/automationAccounts/hybridRunbookWorkerGroups@2023-11-01' = { - name: hybridWorkerGroupName - parent: automationAccount - properties: { - // Hybrid worker group properties - workers will be added separately - } -} - -// Private endpoint for the Automation Account - Webhook -resource automationAccountPrivateEndpoint 'Microsoft.Network/privateEndpoints@2024-10-01' = { - name: 'pe-webhook-${automationAccountName}' - location: location - properties: { - subnet: { - id: peSubnetId - } - privateLinkServiceConnections: [ - { - name: 'pls-${automationAccountName}' - properties: { - privateLinkServiceId: automationAccount.id - groupIds: [ - 'Webhook' - ] - } - } - ] - customNetworkInterfaceName: 'nic-pe-webhook-${automationAccountName}' - } - tags: commonTags - - resource privateDnsZoneGroup 'privateDnsZoneGroups' = { - name: 'default' - properties: { - privateDnsZoneConfigs: [ - { - name: 'config1' - properties: { - privateDnsZoneId: automationAccountDnsZone.id - } - } - ] - } - } -} - -// Private endpoint for the Automation Account - DSCAndHybridWorker -resource automationAccountPrivateEndpointDSCAndHybridWorker 'Microsoft.Network/privateEndpoints@2024-10-01' = { - name: 'pe-dscandhybridworker-${automationAccountName}' - location: location - properties: { - subnet: { - id: peSubnetId - } - privateLinkServiceConnections: [ - { - name: 'pls-dscandhybridworker-${automationAccountName}' - properties: { - privateLinkServiceId: automationAccount.id - groupIds: [ - 'DSCAndHybridWorker' - ] - } - } - ] - customNetworkInterfaceName: 'nic-pe-dscandhybridworker-${automationAccountName}' - } - tags: commonTags - - resource privateDnsZoneGroup 'privateDnsZoneGroups' = { - name: 'default' - properties: { - privateDnsZoneConfigs: [ - { - name: 'config1' - properties: { - privateDnsZoneId: automationAccountDnsZone.id - } - } - ] - } - } -} - -// Diagnostic Settings for Automation Account -resource automationAccountDiagnostics 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { - name: 'diag-${automationAccountName}' - scope: automationAccount - properties: { - workspaceId: logAnalyticsWorkspace.id - logs: [ - { - category: 'JobLogs' - enabled: true - } - { - category: 'JobStreams' - enabled: true - } - ] - metrics: [ - { - category: 'AllMetrics' - enabled: true - } - ] - } -} - -// KeyVault -resource keyVault 'Microsoft.KeyVault/vaults@2025-05-01' = { - name: keyVaultName - location: location - properties: { - sku: { - family: 'A' - name: 'standard' - } - tenantId: subscription().tenantId - enableSoftDelete: true - softDeleteRetentionInDays: 7 - enableRbacAuthorization: true - publicNetworkAccess: 'Disabled' - } - tags: commonTags -} - -// Private endpoint for the KeyVault -resource keyVaultPrivateEndpoint 'Microsoft.Network/privateEndpoints@2024-10-01' = { - name: 'pe-vault-${keyVaultName}' - location: location - properties: { - subnet: { - id: peSubnetId - } - privateLinkServiceConnections: [ - { - name: 'pls-${keyVaultName}' - properties: { - privateLinkServiceId: keyVault.id - groupIds: [ - 'vault' - ] - } - } - ] - customNetworkInterfaceName: 'nic-pe-${keyVaultName}' - } - tags: commonTags - - resource privateDnsZoneGroup 'privateDnsZoneGroups' = { - name: 'default' - properties: { - privateDnsZoneConfigs: [ - { - name: 'config1' - properties: { - privateDnsZoneId: keyVaultDnsZone.id - } - } - ] - } - } -} - -// Diagnostic Settings for Key Vault -resource keyVaultDiagnostics 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { - name: 'diag-${keyVaultName}' - scope: keyVault - properties: { - workspaceId: logAnalyticsWorkspace.id - logs: [ - { - category: 'AuditEvent' - enabled: true - } - { - category: 'AzurePolicyEvaluationDetails' - enabled: true - } - ] - metrics: [ - { - category: 'AllMetrics' - enabled: true - } - ] - } -} - -// Event Grid System Topic for the KeyVault -resource keyVaultEventGridSystemTopic 'Microsoft.EventGrid/systemTopics@2025-02-15' = { - name: 'egst-${keyVaultName}' - location: location - identity: { - type: 'SystemAssigned' - } - properties: { - source: keyVault.id - topicType: 'Microsoft.KeyVault.Vaults' - } - tags: commonTags -} - -// Event Grid subscription for the KeyVault to the queue -// This subscription filters only the CertificateNearExpiry events and sends them to the storage queue -resource keyVaultEventGridSubscription 'Microsoft.EventGrid/systemTopics/eventSubscriptions@2025-02-15' = { - parent: keyVaultEventGridSystemTopic - name: 'egs-${keyVaultEventGridSystemTopic.name}' - properties: { - destination: { - endpointType: 'StorageQueue' - properties: { - resourceId: storageAccount.id - queueName: 'certlc' - queueMessageTimeToLiveInSeconds: 86400 // 1 day - } - } - eventDeliverySchema: 'CloudEventSchemaV1_0' - filter: { - includedEventTypes: [ - 'Microsoft.KeyVault.CertificateNearExpiry' - ] - isSubjectCaseSensitive: false - } - retryPolicy: { - maxDeliveryAttempts: 30 - eventTimeToLiveInMinutes: 1440 // 1 day - } - } -} - -// Azure Monitor Workbook for Certificate Statistics -resource workbookCertLCStats 'Microsoft.Insights/workbooks@2023-06-01' = { - name: guid(resourceGroup().id, 'certlcstats') - location: location - kind: 'shared' - properties: { - displayName: 'certlcstats' - serializedData: '{"version":"Notebook/1.0","items":[],"styleSettings":{},"$schema":"https://github.com/Microsoft/Application-Insights-Workbooks/blob/master/schema/workbook.json"}' - category: 'workbook' - sourceId: logAnalyticsWorkspace.id - } - dependsOn: [ - applicationInsights // Wait for App Insights to ensure workspace is fully active - ] - tags: commonTags -} - -// Role Assignment: Grant the Event Grid System Topic the "Storage Queue Data Reader" role on the Storage Account -// this role allows Event Grid to read messages from the queue -resource eventGridStorageQueueDataReader 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - name: guid(subscription().id, resourceGroup().id, 'eventGridStorageQueueDataReader') - scope: storageAccount - properties: { - description: 'EventGrid SystemTopic -> Storage Queue Data Reader -> Storage Account' - roleDefinitionId: subscriptionResourceId( - 'Microsoft.Authorization/roleDefinitions', - roleDefinitions.storageQueueDataReader - ) - principalId: keyVaultEventGridSystemTopic.identity.principalId - principalType: 'ServicePrincipal' - } -} - -// Role Assignment: Grant the Event Grid System Topic the "Storage Queue Data Message Sender" role on the Storage Account -// this role allows Event Grid to send messages to the queue -resource eventGridStorageQueueDataMessageSender 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - name: guid(subscription().id, resourceGroup().id, 'eventGridStorageQueueDataMessageSender') - scope: storageAccount - properties: { - description: 'EventGrid SystemTopic -> Storage Queue Data Message Sender -> Storage Account' - roleDefinitionId: subscriptionResourceId( - 'Microsoft.Authorization/roleDefinitions', - roleDefinitions.storageQueueDataMessageSender - ) - principalId: keyVaultEventGridSystemTopic.identity.principalId - principalType: 'ServicePrincipal' - } -} - -// Role Assignment: Grant the Automation Account the "Key Vault Certificates Officer" role on the KeyVault -// this role allows the automation account to create and manage certificates in the KeyVault -resource automationAccountKeyVaultCertificatesOfficer 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - name: guid(subscription().id, resourceGroup().id, 'automationAccountKeyVaultCertificatesOfficer') - scope: keyVault - properties: { - description: 'Automation Account -> Key Vault Certificates Officer -> Key Vault' - roleDefinitionId: subscriptionResourceId( - 'Microsoft.Authorization/roleDefinitions', - roleDefinitions.keyVaultCertificatesOfficer - ) - principalId: automationAccount.identity.principalId - principalType: 'ServicePrincipal' - } -} - -// Role Assignment: Grant the Automation Account the "Key Vault Secrets Officer" role on the KeyVault -// this role allows the automation account to create and manage secrets (private keys of the certificates) in the KeyVault -resource automationAccountKeyVaultSecretsOfficer 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - name: guid(subscription().id, resourceGroup().id, 'automationAccountKeyVaultSecretsOfficer') - scope: keyVault - properties: { - description: 'Automation Account -> Key Vault Secrets Officer -> Key Vault' - roleDefinitionId: subscriptionResourceId( - 'Microsoft.Authorization/roleDefinitions', - roleDefinitions.keyVaultSecretsOfficer - ) - principalId: automationAccount.identity.principalId - principalType: 'ServicePrincipal' - } -} - -// Role Assignment: Grant the Automation Account the "Reader" role on the Automation Account -// This may seem strange, but it is required for the hybrid worker (that uses the automation account's identity) to read the automation account variables -resource automationAccountReader 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - name: guid(subscription().id, resourceGroup().id, 'automationAccountReader') - scope: automationAccount - properties: { - description: 'Automation Account -> Reader -> Automation Account (self)' - roleDefinitionId: subscriptionResourceId( - 'Microsoft.Authorization/roleDefinitions', - roleDefinitions.reader - ) - principalId: automationAccount.identity.principalId - principalType: 'ServicePrincipal' - } -} - -// Role Assignment: Grant the Automation Account the "Monitoring Metrics Publisher" role on the DCR -// (this is to allow the automation account to write custom logs to the DCR) -resource automationAccountMonitoringMetricsPublisher 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - name: guid(subscription().id, resourceGroup().id, 'automationAccountMonitoringMetricsPublisher') - scope: dataCollectionRule - properties: { - description: 'Automation Account -> Monitoring Metrics Publisher -> DCR' - roleDefinitionId: subscriptionResourceId( - 'Microsoft.Authorization/roleDefinitions', - roleDefinitions.monitoringMetricsPublisher - ) - principalId: automationAccount.identity.principalId - principalType: 'ServicePrincipal' - } -} - -// Role Assignment: Grant the Function App the "Storage Blob Data Owner" role on the Storage Account -resource functionAppStorageBlobDataOwner 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - name: guid(subscription().id, resourceGroup().id, 'functionAppStorageBlobDataOwner') - scope: storageAccount - properties: { - description: 'Function App -> Storage Blob Data Owner -> Storage Account' - roleDefinitionId: subscriptionResourceId( - 'Microsoft.Authorization/roleDefinitions', - roleDefinitions.storageBlobDataOwner - ) - principalId: functionApp.identity.principalId - principalType: 'ServicePrincipal' - } -} - -// Role Assignment: Grant the Function App the "Storage Queue Data Message Processor" role on the Storage Account -resource functionAppStorageQueueDataMessageProcessor 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - name: guid(subscription().id, resourceGroup().id, 'functionAppStorageQueueDataMessageProcessor') - scope: storageAccount - properties: { - description: 'Function App -> Storage Queue Data Message Processor -> Storage Account' - roleDefinitionId: subscriptionResourceId( - 'Microsoft.Authorization/roleDefinitions', - roleDefinitions.storageQueueDataMessageProcessor - ) - principalId: functionApp.identity.principalId - principalType: 'ServicePrincipal' - } -} - -// Role Assignment: Grant the Function App the "Storage Queue Data Contributor" role on the Storage Account -resource functionAppStorageQueueDataContributor 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - name: guid(subscription().id, resourceGroup().id, 'functionAppStorageQueueDataContributor') - scope: storageAccount - properties: { - description: 'Function App -> Storage Queue Data Contributor -> Storage Account' - roleDefinitionId: subscriptionResourceId( - 'Microsoft.Authorization/roleDefinitions', - roleDefinitions.storageQueueDataContributor - ) - principalId: functionApp.identity.principalId - principalType: 'ServicePrincipal' - } -} - -// Role Assignment: Grant the Function App the "Reader" role on the Automation Account -// (this is to allow the function app to read automation account information and trigger runbooks) -resource functionAppAutomationAccountReader 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - name: guid(subscription().id, resourceGroup().id, 'functionAppAutomationAccountReader') - scope: automationAccount - properties: { - description: 'Function App -> Reader -> Automation Account' - roleDefinitionId: subscriptionResourceId( - 'Microsoft.Authorization/roleDefinitions', - roleDefinitions.reader - ) - principalId: functionApp.identity.principalId - principalType: 'ServicePrincipal' - } -} - -// Role Assignment: Grant the Function App the "Automation Operator" role on the Automation Account -// (this is to allow the function app to start runbook jobs) -resource functionAppAutomationOperator 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - name: guid(subscription().id, resourceGroup().id, 'functionAppAutomationOperator') - scope: automationAccount - properties: { - description: 'Function App -> Automation Operator -> Automation Account' - roleDefinitionId: subscriptionResourceId( - 'Microsoft.Authorization/roleDefinitions', - roleDefinitions.automationOperator - ) - principalId: functionApp.identity.principalId - principalType: 'ServicePrincipal' - } -} - -// Role Assignment: Grant the Function App the "Monitoring Metrics Publisher" role on the Application Insights instance -// (this is to instrument the function app with App Insights) -resource functionAppMonitoringMetricsPublisher 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - name: guid(subscription().id, resourceGroup().id, 'functionAppMonitoringMetricsPublisher') - scope: applicationInsights - properties: { - description: 'Function App -> Monitoring Metrics Publisher -> Application Insights' - roleDefinitionId: subscriptionResourceId( - 'Microsoft.Authorization/roleDefinitions', - roleDefinitions.monitoringMetricsPublisher - ) - principalId: functionApp.identity.principalId - principalType: 'ServicePrincipal' - } -} - -// Output all resource IDs and important properties -output storageAccountId string = storageAccount.id -output storageAccountQueueUri string = storageAccount.properties.primaryEndpoints.queue -output automationAccountId string = automationAccount.id -output keyVaultId string = keyVault.id -output functionAppId string = functionApp.id -output logAnalyticsWorkspaceId string = logAnalyticsWorkspace.id -output applicationInsightsId string = applicationInsights.id -output dceIngestionEndpoint string = dataCollectionEndpoint.properties.logsIngestion.endpoint -@secure() -output dataCollectionRuleImmutableId string = dataCollectionRule.properties.immutableId diff --git a/Setup/getavailability.bicep b/Setup/getavailability.bicep deleted file mode 100644 index 5aaba39..0000000 --- a/Setup/getavailability.bicep +++ /dev/null @@ -1,214 +0,0 @@ -/* - -Get-Availability — Bicep template for Log Analytics ingestion infrastructure. - -Creates a Log Analytics workspace, two custom tables, a Data Collection -Endpoint (DCE), and a Data Collection Rule (DCR) for ingesting -Get-Availability script results. - -Validate: az deployment group validate --resource-group --parameters .\parameters.dev.bicepparam -What-if: az deployment group what-if --resource-group --parameters .\parameters.dev.bicepparam -Deploy: az deployment group create --resource-group --parameters .\parameters.dev.bicepparam - -*/ - -metadata name = 'Get-Availability Infrastructure' -metadata description = 'Log Analytics workspace, custom tables, DCE, and DCR for Get-Availability telemetry ingestion' - -// ── Parameters ─────────────────────────────────────────────────────────────── - -@description('Azure region for all resources. Defaults to the resource group location.') -param location string = resourceGroup().location - -@description('Name of the Log Analytics workspace to create.') -param logAnalyticsWorkspaceName string - -@description('Name of the Data Collection Endpoint.') -param dataCollectionEndpointName string - -@description('Name of the Data Collection Rule.') -param dataCollectionRuleName string - -// ── Variables ──────────────────────────────────────────────────────────────── - -var commonTags = { - solution: 'Get-Availability' -} - -// ── Log Analytics Workspace ────────────────────────────────────────────────── - -resource logAnalyticsWorkspace 'Microsoft.OperationalInsights/workspaces@2023-09-01' = { - name: logAnalyticsWorkspaceName - location: location - properties: { - sku: { - name: 'PerGB2018' - } - retentionInDays: 30 - } - tags: commonTags -} - -// ── Custom Table: GetAvailResources_CL (per-resource detail) ───────────────── - -resource resourcesTable 'Microsoft.OperationalInsights/workspaces/tables@2022-10-01' = { - name: 'GetAvailResources_CL' - parent: logAnalyticsWorkspace - properties: { - retentionInDays: 30 - schema: { - name: 'GetAvailResources_CL' - columns: [ - { name: 'TimeGenerated', type: 'dateTime' } - { name: 'RunId', type: 'string' } - { name: 'Month', type: 'string' } - { name: 'PeriodStart', type: 'dateTime' } - { name: 'PeriodEnd', type: 'dateTime' } - { name: 'IsMonthToDate', type: 'boolean' } - { name: 'SubscriptionName', type: 'string' } - { name: 'ResourceName', type: 'string' } - { name: 'ResourceId', type: 'string' } - { name: 'ResourceGroup', type: 'string' } - { name: 'Kind', type: 'string' } - { name: 'Location', type: 'string' } - { name: 'EligibleMinutes', type: 'int' } - { name: 'AvailableMinutes', type: 'real' } - { name: 'SuspectMinutes', type: 'int' } - { name: 'ConfirmedDowntimeMinutes', type: 'int' } - { name: 'ExcusedMinutes', type: 'int' } - { name: 'UnexplainedSuspectMinutes', type: 'int' } - { name: 'AvailabilityPct', type: 'real' } - ] - } - } -} - -// ── Custom Table: GetAvailSummary_CL (aggregated summaries) ────────────────── - -resource summaryTable 'Microsoft.OperationalInsights/workspaces/tables@2022-10-01' = { - name: 'GetAvailSummary_CL' - parent: logAnalyticsWorkspace - properties: { - retentionInDays: 30 - schema: { - name: 'GetAvailSummary_CL' - columns: [ - { name: 'TimeGenerated', type: 'dateTime' } - { name: 'RunId', type: 'string' } - { name: 'Month', type: 'string' } - { name: 'PeriodStart', type: 'dateTime' } - { name: 'PeriodEnd', type: 'dateTime' } - { name: 'IsMonthToDate', type: 'boolean' } - { name: 'SummaryLevel', type: 'string' } - { name: 'SubscriptionName', type: 'string' } - { name: 'Kind', type: 'string' } - { name: 'Location', type: 'string' } - { name: 'ResourceCount', type: 'int' } - { name: 'EligibleMinutes', type: 'real' } - { name: 'AvailableMinutes', type: 'real' } - { name: 'AvailabilityPct', type: 'real' } - ] - } - } -} - -// ── Data Collection Endpoint ───────────────────────────────────────────────── - -resource dataCollectionEndpoint 'Microsoft.Insights/dataCollectionEndpoints@2023-03-11' = { - name: dataCollectionEndpointName - location: location - properties: { - networkAcls: { - publicNetworkAccess: 'Enabled' - } - } - tags: commonTags -} - -// ── Data Collection Rule (two streams, one per table) ──────────────────────── - -resource dataCollectionRule 'Microsoft.Insights/dataCollectionRules@2023-03-11' = { - name: dataCollectionRuleName - location: location - properties: { - dataCollectionEndpointId: dataCollectionEndpoint.id - - // Stream declarations — columns sent by the script (no TimeGenerated; injected by transform) - streamDeclarations: { - 'Custom-GetAvailResources_CL': { - columns: [ - { name: 'RunId', type: 'string' } - { name: 'Month', type: 'string' } - { name: 'PeriodStart', type: 'string' } - { name: 'PeriodEnd', type: 'string' } - { name: 'IsMonthToDate', type: 'boolean' } - { name: 'SubscriptionName', type: 'string' } - { name: 'ResourceName', type: 'string' } - { name: 'ResourceId', type: 'string' } - { name: 'ResourceGroup', type: 'string' } - { name: 'Kind', type: 'string' } - { name: 'Location', type: 'string' } - { name: 'EligibleMinutes', type: 'int' } - { name: 'AvailableMinutes', type: 'real' } - { name: 'SuspectMinutes', type: 'int' } - { name: 'ConfirmedDowntimeMinutes', type: 'int' } - { name: 'ExcusedMinutes', type: 'int' } - { name: 'UnexplainedSuspectMinutes', type: 'int' } - { name: 'AvailabilityPct', type: 'real' } - ] - } - 'Custom-GetAvailSummary_CL': { - columns: [ - { name: 'RunId', type: 'string' } - { name: 'Month', type: 'string' } - { name: 'PeriodStart', type: 'string' } - { name: 'PeriodEnd', type: 'string' } - { name: 'IsMonthToDate', type: 'boolean' } - { name: 'SummaryLevel', type: 'string' } - { name: 'SubscriptionName', type: 'string' } - { name: 'Kind', type: 'string' } - { name: 'Location', type: 'string' } - { name: 'ResourceCount', type: 'int' } - { name: 'EligibleMinutes', type: 'real' } - { name: 'AvailableMinutes', type: 'real' } - { name: 'AvailabilityPct', type: 'real' } - ] - } - } - - destinations: { - logAnalytics: [ - { - workspaceResourceId: logAnalyticsWorkspace.id - name: 'workspace' - } - ] - } - - dataFlows: [ - { - streams: [ 'Custom-GetAvailResources_CL' ] - destinations: [ 'workspace' ] - transformKql: 'source | extend TimeGenerated = now(), PeriodStart = todatetime(PeriodStart), PeriodEnd = todatetime(PeriodEnd)' - outputStream: 'Custom-GetAvailResources_CL' - } - { - streams: [ 'Custom-GetAvailSummary_CL' ] - destinations: [ 'workspace' ] - transformKql: 'source | extend TimeGenerated = now(), PeriodStart = todatetime(PeriodStart), PeriodEnd = todatetime(PeriodEnd)' - outputStream: 'Custom-GetAvailSummary_CL' - } - ] - } - dependsOn: [ - resourcesTable - summaryTable - ] - tags: commonTags -} - -// ── Outputs ────────────────────────────────────────────────────────────────── - -output logAnalyticsWorkspaceId string = logAnalyticsWorkspace.id -output dceIngestionEndpoint string = dataCollectionEndpoint.properties.logsIngestion.endpoint -output dataCollectionRuleImmutableId string = dataCollectionRule.properties.immutableId diff --git a/Setup/parameters.dev.bicepparam b/Setup/parameters.dev.bicepparam deleted file mode 100644 index 4b8d186..0000000 --- a/Setup/parameters.dev.bicepparam +++ /dev/null @@ -1,5 +0,0 @@ -using './getavailability.bicep' - -param logAnalyticsWorkspaceName = 'log-getavail-itn-001' -param dataCollectionEndpointName = 'dce-getavail-itn-001' -param dataCollectionRuleName = 'dcr-getavail-itn-001' From 30753acbc279568313b67911568a2a57a3e6fd4b Mon Sep 17 00:00:00 2001 From: Marcello Formica Date: Mon, 27 Apr 2026 17:46:01 +0200 Subject: [PATCH 07/10] Remove C# from CI/CD workflows, update PS1 paths, add RBAC docs --- .github/workflows/ci.yml | 14 +++----------- .github/workflows/release.yml | 19 +++---------------- README.md | 33 +++++++++++++++++++++++++++++++-- 3 files changed, 37 insertions(+), 29 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 75010f7..39091f3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,25 +5,17 @@ on: branches: [main] jobs: - build: - runs-on: windows-latest + validate: + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Setup .NET - uses: actions/setup-dotnet@v4 - with: - dotnet-version: '10.0.x' - - - name: Build - run: dotnet build csharp/GetAvailability/GetAvailability.csproj -c Release - - name: Verify PS1 syntax shell: pwsh run: | $tokens = $null; $errors = $null $null = [System.Management.Automation.Language.Parser]::ParseFile( - 'get-availability.ps1', [ref]$tokens, [ref]$errors) + 'Functions/GetAvail/get-availability.ps1', [ref]$tokens, [ref]$errors) if ($errors.Count -gt 0) { $errors | ForEach-Object { Write-Error $_.ToString() } exit 1 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 56c4a11..45cf6f5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,15 +9,10 @@ permissions: jobs: release: - runs-on: windows-latest + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Setup .NET - uses: actions/setup-dotnet@v4 - with: - dotnet-version: '10.0.x' - - name: Extract version from tag id: version shell: pwsh @@ -25,22 +20,15 @@ jobs: $version = "${{ github.ref_name }}".TrimStart('v') echo "VERSION=$version" >> $env:GITHUB_OUTPUT - - name: Publish AOT - run: dotnet publish csharp/GetAvailability/GetAvailability.csproj -c Release -r win-x64 -p:Version=${{ steps.version.outputs.VERSION }} - - name: Prepare release assets shell: pwsh run: | $version = '${{ steps.version.outputs.VERSION }}' - $publishDir = 'csharp/GetAvailability/bin/Release/net10.0/win-x64/publish' - $stageDir = 'release-assets' + $stageDir = 'release-assets' New-Item -ItemType Directory -Path $stageDir -Force | Out-Null - # AOT executable - Copy-Item "$publishDir/GetAvailability.exe" "$stageDir/GetAvailability.exe" - # PowerShell script — stamp version - (Get-Content 'get-availability.ps1' -Raw) -replace "0\.0\.0-dev", $version | + (Get-Content 'Functions/GetAvail/get-availability.ps1' -Raw) -replace "0\.0\.0-dev", $version | Set-Content "$stageDir/get-availability.ps1" -NoNewline - name: Create GitHub Release @@ -48,5 +36,4 @@ jobs: with: generate_release_notes: true files: | - release-assets/GetAvailability.exe release-assets/get-availability.ps1 diff --git a/README.md b/README.md index 53fcf3a..26dbb9f 100644 --- a/README.md +++ b/README.md @@ -344,6 +344,37 @@ az deployment group what-if --resource-group rg-getavail-itn-001 --parameters Bi az deployment group create --resource-group rg-getavail-itn-001 --parameters Bicep/parameters.dev.bicepparam ``` +### Post-deployment: cross-subscription Reader role + +The Bicep template creates RBAC assignments within the deployment resource group (Metrics Publisher, Storage Blob Data Owner). However, the function also needs **Reader** access on every subscription listed in `getavailSubscriptions` so that `Get-AzSubscription` and `Search-AzGraph` can enumerate and query resources there. + +After deployment, retrieve the managed identity principal ID and assign **Reader** on each target subscription: + +```powershell +# Get the Function App managed identity principal ID +$principalId = (az functionapp identity show ` + --name fn-getavail-itn-001 ` + --resource-group rg-getavail-itn-001 ` + --query principalId -o tsv) + +# Assign Reader on each subscription in getavailSubscriptions +$subscriptions = @('Flaz-Connectivity', 'Flaz-Management', 'Flaz-Identity', 'Flaz-Workloads') +foreach ($sub in $subscriptions) { + $subId = az account show --subscription $sub --query id -o tsv + az role assignment create --assignee $principalId --role Reader --scope "/subscriptions/$subId" +} +``` + +> **Note:** You only need to do this once per subscription (or when the managed identity is recreated). The Workloads subscription (where the Function App lives) may already have Reader via inheritance — include it for completeness. + +If `sourceWorkspaceId` points to a Log Analytics workspace (e.g. a Sentinel workspace for Activity Log / Resource Health queries), the managed identity also needs **Log Analytics Reader** on that workspace: + +```powershell +# Assign Log Analytics Reader on the source workspace (if used) +az role assignment create --assignee $principalId --role "Log Analytics Reader" ` + --scope "" +``` + ### Auto-wired app settings The Bicep template configures the Function App with all required settings — values are resolved from sibling resources at deploy time: @@ -358,8 +389,6 @@ The Bicep template configures the Function App with all required settings — va | `TIMER_SCHEDULE` | `timerSchedule` parameter | *(timer trigger via `%TIMER_SCHEDULE%`)* | | `APPLICATIONINSIGHTS_CONNECTION_STRING` | App Insights connection string | *(Functions runtime)* | -No manual post-deployment configuration is required. - The template also configures CORS to allow `https://portal.azure.com`, so you can test-run the function directly from the Azure Portal. ### Deployment outputs From b6572839e80647605cadb210293a4a5abd75a077 Mon Sep 17 00:00:00 2001 From: Marcello Formica Date: Mon, 27 Apr 2026 18:10:02 +0200 Subject: [PATCH 08/10] Exclude perpetually-deallocated VMs from availability counts --- Functions/GetAvail/get-availability.ps1 | 41 +++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/Functions/GetAvail/get-availability.ps1 b/Functions/GetAvail/get-availability.ps1 index bb923ba..176b170 100644 --- a/Functions/GetAvail/get-availability.ps1 +++ b/Functions/GetAvail/get-availability.ps1 @@ -214,7 +214,7 @@ function Send-ToLogAnalytics { # Try full payload as a single call; split only if compressed size exceeds 900 KB $compressed = & $compressJson $Payload if ($compressed.Length -lt 900KB) { - Invoke-WebRequest -Uri $uri -Method Post -Headers $headers -Body $compressed -UseBasicParsing | Out-Null + Invoke-WebRequest -Uri $uri -Method Post -Headers $headers -Body $compressed -UseBasicParsing -ProgressAction SilentlyContinue | Out-Null return } @@ -223,7 +223,7 @@ function Send-ToLogAnalytics { for ($i = 0; $i -lt $Payload.Count; $i += $chunkSize) { $chunk = @($Payload[$i..([math]::Min($i + $chunkSize - 1, $Payload.Count - 1))]) $body = & $compressJson $chunk - Invoke-WebRequest -Uri $uri -Method Post -Headers $headers -Body $body -UseBasicParsing | Out-Null + Invoke-WebRequest -Uri $uri -Method Post -Headers $headers -Body $body -UseBasicParsing -ProgressAction SilentlyContinue | Out-Null } } @@ -2330,6 +2330,39 @@ if ($Batch) { -EndDate $utcEnd -ThrottleLimit $Parallelism -ArmToken $armToken } +# Step 5b: Detect perpetually-deallocated VMs +# VMs that are currently deallocated/stopped AND had zero healthy minutes for the +# entire period are excused — the VM was never running so availability is N/A. +# This handles cases where the deallocate event fell outside Activity Log and +# Resource Health retention windows. +$perpetuallyDeallocated = [System.Collections.Generic.HashSet[string]]::new([StringComparer]::OrdinalIgnoreCase) +$vmCandidates = @($resources | Where-Object { + $_.Kind -eq 'VirtualMachine' -and + $metricResults.ContainsKey($_.ResourceId.ToLowerInvariant()) -and + $metricResults[$_.ResourceId.ToLowerInvariant()].SuspectMinutes -ge $totalMinutes +}) +if ($vmCandidates.Count -gt 0) { + foreach ($vm in $vmCandidates) { + try { + $ivUrl = "https://management.azure.com$($vm.ResourceId)/instanceView?api-version=2024-07-01" + $ivResp = Invoke-RestMethod -Uri $ivUrl -Headers @{ Authorization = "Bearer $armToken" } -Method Get + $powerCode = ($ivResp.statuses | Where-Object { $_.code -like 'PowerState/*' } | Select-Object -First 1).code + if ($powerCode -eq 'PowerState/deallocated' -or $powerCode -eq 'PowerState/stopped') { + $perpetuallyDeallocated.Add($vm.ResourceId) | Out-Null + $elig = $eligByRes[$vm.ResourceId.ToLowerInvariant()] + $elig.EligibleMinutes = 0 + $elig.ExcusedMinutes = $totalMinutes + $elig.SuspectMinutes = $totalMinutes + $elig.AvailabilityPct = 'N/A' + Write-Host " [$($vm.Name)] Deallocated for entire period — excluded from availability counts" + } + } + catch { + Write-Warning "Instance View check failed for '$($vm.Name)': $_" + } + } +} + # Step 6: Build suspect candidates and investigate via Activity Log + Resource Health # Build the list of resources that have at least one suspect minute — # these need Activity Log + Resource Health investigation. Combine null @@ -2337,6 +2370,7 @@ if ($Batch) { $suspectCandidates = [System.Collections.Generic.List[object]]::new() foreach ($res in $resources) { $key = $res.ResourceId.ToLowerInvariant() + if ($perpetuallyDeallocated.Contains($res.ResourceId)) { continue } $mr = $metricResults[$key] if ($mr -and $mr.SuspectMinutes -gt 0) { $allTicks = [System.Collections.Generic.List[long]]::new() @@ -2406,6 +2440,9 @@ foreach ($res in $resources) { continue } + # Skip perpetually-deallocated VMs — already handled in Step 5b + if ($perpetuallyDeallocated.Contains($res.ResourceId)) { continue } + $activityLogExcludedGapMinutes = 0 $healthExplainedGapMinutes = 0 $metricIssueNullMinutes = 0 From dd82abbeef039dfad1b6d79d75df48914f262287 Mon Sep 17 00:00:00 2001 From: Marcello Formica Date: Mon, 27 Apr 2026 19:13:59 +0200 Subject: [PATCH 09/10] Fix Log Analytics ingestion: byte[] unrolling, DCR kind:Direct, transforms - Replace gzip scriptblock with nested function + [byte[]] cast to prevent PowerShell pipeline array unrolling that corrupted request bodies - Switch from Invoke-WebRequest to Invoke-RestMethod with retry logic (3 attempts, exponential backoff) and status code logging - Add kind: 'Direct' to DCR (required for Logs Ingestion API) - Fix DCR stream declarations: PeriodStart/PeriodEnd as string with todatetime() in transform KQL to avoid extend-on-existing-column errors --- Bicep/getavailability.bicep | 13 ++++---- Functions/GetAvail/get-availability.ps1 | 43 +++++++++++++++++-------- 2 files changed, 37 insertions(+), 19 deletions(-) diff --git a/Bicep/getavailability.bicep b/Bicep/getavailability.bicep index 60a24a1..cd9178c 100644 --- a/Bicep/getavailability.bicep +++ b/Bicep/getavailability.bicep @@ -185,6 +185,7 @@ resource dataCollectionEndpoint 'Microsoft.Insights/dataCollectionEndpoints@2023 resource dataCollectionRule 'Microsoft.Insights/dataCollectionRules@2023-03-11' = { name: dataCollectionRuleName location: location + kind: 'Direct' properties: { dataCollectionEndpointId: dataCollectionEndpoint.id @@ -194,8 +195,8 @@ resource dataCollectionRule 'Microsoft.Insights/dataCollectionRules@2023-03-11' columns: [ { name: 'RunId', type: 'string' } { name: 'Month', type: 'string' } - { name: 'PeriodStart', type: 'string' } - { name: 'PeriodEnd', type: 'string' } + { name: 'PeriodStart', type: 'datetime' } + { name: 'PeriodEnd', type: 'datetime' } { name: 'IsMonthToDate', type: 'boolean' } { name: 'SubscriptionName', type: 'string' } { name: 'ResourceName', type: 'string' } @@ -216,8 +217,8 @@ resource dataCollectionRule 'Microsoft.Insights/dataCollectionRules@2023-03-11' columns: [ { name: 'RunId', type: 'string' } { name: 'Month', type: 'string' } - { name: 'PeriodStart', type: 'string' } - { name: 'PeriodEnd', type: 'string' } + { name: 'PeriodStart', type: 'datetime' } + { name: 'PeriodEnd', type: 'datetime' } { name: 'IsMonthToDate', type: 'boolean' } { name: 'SummaryLevel', type: 'string' } { name: 'SubscriptionName', type: 'string' } @@ -244,13 +245,13 @@ resource dataCollectionRule 'Microsoft.Insights/dataCollectionRules@2023-03-11' { streams: [ 'Custom-GetAvailResources_CL' ] destinations: [ 'workspace' ] - transformKql: 'source | extend TimeGenerated = now(), PeriodStart = todatetime(PeriodStart), PeriodEnd = todatetime(PeriodEnd)' + transformKql: 'source | extend TimeGenerated = now()' outputStream: 'Custom-GetAvailResources_CL' } { streams: [ 'Custom-GetAvailSummary_CL' ] destinations: [ 'workspace' ] - transformKql: 'source | extend TimeGenerated = now(), PeriodStart = todatetime(PeriodStart), PeriodEnd = todatetime(PeriodEnd)' + transformKql: 'source | extend TimeGenerated = now()' outputStream: 'Custom-GetAvailSummary_CL' } ] diff --git a/Functions/GetAvail/get-availability.ps1 b/Functions/GetAvail/get-availability.ps1 index 176b170..13106bd 100644 --- a/Functions/GetAvail/get-availability.ps1 +++ b/Functions/GetAvail/get-availability.ps1 @@ -197,24 +197,41 @@ function Send-ToLogAnalytics { 'Content-Encoding' = 'gzip' } - # Gzip-compress a JSON array into a byte[] - $compressJson = { - param([object[]]$Items) + # Gzip-compress a JSON array into a byte[]. + # Returns [byte[]] directly — NOT through the pipeline — to avoid PowerShell + # unrolling the array into individual System.Byte objects. + function Compress-JsonPayload([object[]]$Items) { $json = $Items | ConvertTo-Json -Depth 5 -Compress -AsArray $bytes = [System.Text.Encoding]::UTF8.GetBytes($json) $ms = [System.IO.MemoryStream]::new() - $gz = [System.IO.Compression.GZipStream]::new($ms, [System.IO.Compression.CompressionLevel]::Optimal) - $gz.Write($bytes, 0, $bytes.Length) - $gz.Dispose() - $result = $ms.ToArray() - $ms.Dispose() - $result + try { + $gz = [System.IO.Compression.GZipStream]::new($ms, [System.IO.Compression.CompressionLevel]::Optimal, $true) + $gz.Write($bytes, 0, $bytes.Length) + $gz.Dispose() + [byte[]]$ms.ToArray() + } finally { $ms.Dispose() } + } + + # POST with retry (3 attempts, exponential backoff) + function Send-Chunk([byte[]]$Body) { + $maxAttempts = 3 + for ($attempt = 1; $attempt -le $maxAttempts; $attempt++) { + try { + Invoke-RestMethod -Uri $uri -Method Post -Headers $headers -Body $Body -TimeoutSec 60 -StatusCodeVariable statusCode + return + } catch { + if ($attempt -eq $maxAttempts) { throw } + $delay = [math]::Pow(2, $attempt) + Write-Warning "Ingestion attempt $attempt to $StreamName failed (HTTP $statusCode): $($_.Exception.Message). Retrying in ${delay}s..." + Start-Sleep -Seconds $delay + } + } } # Try full payload as a single call; split only if compressed size exceeds 900 KB - $compressed = & $compressJson $Payload + [byte[]]$compressed = Compress-JsonPayload $Payload if ($compressed.Length -lt 900KB) { - Invoke-WebRequest -Uri $uri -Method Post -Headers $headers -Body $compressed -UseBasicParsing -ProgressAction SilentlyContinue | Out-Null + Send-Chunk $compressed return } @@ -222,8 +239,8 @@ function Send-ToLogAnalytics { $chunkSize = [math]::Max(1, [math]::Floor($Payload.Count / [math]::Ceiling($compressed.Length / 900KB))) for ($i = 0; $i -lt $Payload.Count; $i += $chunkSize) { $chunk = @($Payload[$i..([math]::Min($i + $chunkSize - 1, $Payload.Count - 1))]) - $body = & $compressJson $chunk - Invoke-WebRequest -Uri $uri -Method Post -Headers $headers -Body $body -UseBasicParsing -ProgressAction SilentlyContinue | Out-Null + [byte[]]$body = Compress-JsonPayload $chunk + Send-Chunk $body } } From 91413475b2b31afc71a8a6254da35c8d7ac2b852 Mon Sep 17 00:00:00 2001 From: Marcello Formica Date: Mon, 27 Apr 2026 19:25:01 +0200 Subject: [PATCH 10/10] Housekeeping: remove dead code, modernize PS7 patterns, sync README - host.json: remove managedDependency (incompatible with Flex Consumption) - profile.ps1: replace legacy MSI_SECRET with FUNCTIONS_WORKER_RUNTIME check - requirements.psd1: simplify comments for Flex Consumption - get-availability.ps1: extract Get-PlainToken helper (4 call sites), remove -UseBasicParsing (no-op in PS Core), drop ProgressPreference save/restore in Test-BatchEndpoints, remove stale C# reference - README.md: consolidate C# legacy references into single blockquote --- Functions/GetAvail/get-availability.ps1 | 35 ++++++++++--------------- Functions/GetAvail/host.json | 3 --- Functions/GetAvail/profile.ps1 | 14 +++------- Functions/GetAvail/requirements.psd1 | 9 +++---- README.md | 17 ++---------- 5 files changed, 23 insertions(+), 55 deletions(-) diff --git a/Functions/GetAvail/get-availability.ps1 b/Functions/GetAvail/get-availability.ps1 index 13106bd..b2f865f 100644 --- a/Functions/GetAvail/get-availability.ps1 +++ b/Functions/GetAvail/get-availability.ps1 @@ -6,7 +6,7 @@ Reports month-scoped availability for VMs, Azure SQL databases, Storage Accounts, and Web Apps. .DESCRIPTION - Mirrors the C# Get-Availability pipeline: + Availability pipeline: 1. Resolve subscriptions 2. Inventory via Resource Graph 3. Fetch metrics (Azure Monitor, parallel) @@ -1053,12 +1053,8 @@ function Test-BatchEndpoints { $body = '{"resourceids":[]}' try { - $oldPref = $ProgressPreference; $ProgressPreference = 'SilentlyContinue' - try { - Invoke-WebRequest -Uri $testUri -Method POST -Body $body -ContentType 'application/json' ` - -Headers @{ Authorization = "Bearer $MetricsToken" } -UseBasicParsing -ErrorAction Stop | Out-Null - } - finally { $ProgressPreference = $oldPref } + Invoke-WebRequest -Uri $testUri -Method POST -Body $body -ContentType 'application/json' ` + -Headers @{ Authorization = "Bearer $MetricsToken" } -ErrorAction Stop | Out-Null Write-Host " $region`: OK" } catch { @@ -2245,6 +2241,13 @@ function Write-SubscriptionSummaries([object[]]$Sorted) { } } +## Acquires a plain-text bearer token via Az.Accounts. +## Handles both SecureString (Az.Accounts ≥ 5.x) and legacy plain string returns. +function Get-PlainToken([string]$ResourceUrl) { + $raw = (Get-AzAccessToken -ResourceUrl $ResourceUrl).Token + ($raw -is [securestring]) ? ($raw | ConvertFrom-SecureString -AsPlainText) : [string]$raw +} + # ── Main ────────────────────────────────────────────────────────────────────── $sw = [System.Diagnostics.Stopwatch]::StartNew() @@ -2289,10 +2292,7 @@ $resolvedSubs = @(foreach ($name in $Subscriptions) { $subIds = @($resolvedSubs.Id) $subIdToName = @{}; foreach ($s in $resolvedSubs) { $subIdToName[$s.Id] = $s.Name } -# Acquire ARM token — Az.Accounts may return SecureString or plain string -$rawToken = (Get-AzAccessToken -ResourceUrl 'https://management.azure.com').Token -$armToken = ($rawToken -is [securestring]) ? ($rawToken | ConvertFrom-SecureString -AsPlainText) : [string]$rawToken -$rawToken = $null +$armToken = Get-PlainToken 'https://management.azure.com' Write-Host 'OK' Write-Host "Processing $($resolvedSubs.Count) subscription(s): $($resolvedSubs.Name -join ', ')" @@ -2309,9 +2309,7 @@ Write-Host "Found $($resources.Count) resource(s) across $($resolvedSubs.Count) if ($resources.Count -eq 0) { Write-Host 'No resources found.'; return } if ($Batch) { - $rawMetrics = (Get-AzAccessToken -ResourceUrl 'https://metrics.monitor.azure.com').Token - $metricsToken = ($rawMetrics -is [securestring]) ? ($rawMetrics | ConvertFrom-SecureString -AsPlainText) : [string]$rawMetrics - $rawMetrics = $null + $metricsToken = Get-PlainToken 'https://metrics.monitor.azure.com' Write-Host "Mode: Batch (batch-size=$BatchSize)" } @@ -2415,9 +2413,7 @@ foreach ($res in $resources) { $logAnalyticsData = $null if ($SourceWorkspaceId -and $suspectCandidates.Count -gt 0) { Write-Host -NoNewline 'Fetching Activity Log + Resource Health history from Log Analytics... ' - $laToken = (Get-AzAccessToken -ResourceUrl 'https://api.loganalytics.io').Token - $laTokenStr = ($laToken -is [securestring]) ? ($laToken | ConvertFrom-SecureString -AsPlainText) : [string]$laToken - $laToken = $null + $laTokenStr = Get-PlainToken 'https://api.loganalytics.io' $logAnalyticsData = Get-LogAnalyticsData -WorkspaceId $SourceWorkspaceId ` -SubscriptionIds $subIds -PeriodStart $utcStart -PeriodEnd $utcEnd ` -ArmToken $laTokenStr @@ -2608,10 +2604,7 @@ Write-SubscriptionSummaries $sorted if ($sendToLogAnalytics) { Write-Host -NoNewline 'Sending results to Log Analytics... ' - # Acquire Azure Monitor ingestion token (same pattern as ARM token) - $rawMonitor = (Get-AzAccessToken -ResourceUrl 'https://monitor.azure.com').Token - $monitorToken = ($rawMonitor -is [securestring]) ? ($rawMonitor | ConvertFrom-SecureString -AsPlainText) : [string]$rawMonitor - $rawMonitor = $null + $monitorToken = Get-PlainToken 'https://monitor.azure.com' $runId = [guid]::NewGuid().ToString() $normalizedMonth = $window.NormalizedMonth diff --git a/Functions/GetAvail/host.json b/Functions/GetAvail/host.json index a14a691..9554c3b 100644 --- a/Functions/GetAvail/host.json +++ b/Functions/GetAvail/host.json @@ -14,8 +14,5 @@ "extensionBundle": { "id": "Microsoft.Azure.Functions.ExtensionBundle", "version": "[4.*, 5.0.0)" - }, - "managedDependency": { - "enabled": true } } diff --git a/Functions/GetAvail/profile.ps1 b/Functions/GetAvail/profile.ps1 index 20960a2..d3d2757 100644 --- a/Functions/GetAvail/profile.ps1 +++ b/Functions/GetAvail/profile.ps1 @@ -1,16 +1,10 @@ # Azure Functions profile.ps1 # -# This profile.ps1 will get executed every "cold start" of your Function App. -# "cold start" occurs when: -# -# * A Function App starts up for the very first time -# * A Function App starts up after being de-allocated due to inactivity -# -# You can define helper functions, run commands, or specify environment variables -# NOTE: any variables defined that are not environment variables will get reset after the first execution +# Runs on every "cold start" of the Function App. Sets up Azure context +# so subsequent function invocations can call Az cmdlets immediately. -# Authenticate with Azure PowerShell using MSI. -if ($env:MSI_SECRET) { +# Authenticate with Azure PowerShell using the Function App's managed identity. +if ($env:FUNCTIONS_WORKER_RUNTIME -eq 'powershell') { Disable-AzContextAutosave -Scope Process | Out-Null Connect-AzAccount -Identity } diff --git a/Functions/GetAvail/requirements.psd1 b/Functions/GetAvail/requirements.psd1 index ab114e7..0c0b158 100644 --- a/Functions/GetAvail/requirements.psd1 +++ b/Functions/GetAvail/requirements.psd1 @@ -1,10 +1,7 @@ -# This file enables modules to be automatically managed by the Functions service. -# See https://aka.ms/functionsmanageddependency for additional information. -# -# NOTE: DO NOT USE WITH FLEX FUNCTIONS - managed dependencies are not supported in the Flex Consumption plan. -# Do "Save-Module -Name -Path Modules -Repository PSGallery -Force" to add modules to the Modules folder instead. +# Module manifest for Azure Functions managed dependencies. +# Flex Consumption does not support managed dependencies — use Save-Module +# to vendor modules into the Modules/ folder instead. @{ -# # For latest supported version, go to 'https://www.powershellgallery.com/packages/Az'. Uncomment the next line and replace the MAJOR_VERSION, e.g., 'Az' = '5.*' # 'Az' = '14.*' } diff --git a/README.md b/README.md index 26dbb9f..760f7b4 100644 --- a/README.md +++ b/README.md @@ -5,14 +5,9 @@ Reports month-scoped availability for Azure Virtual Machines, Azure SQL Databases, Azure Storage Accounts, and Azure Web Apps across one or more Azure subscriptions. -Two implementations are provided: +No build step; runs as a standalone PowerShell 7 script or as an Azure Function on a schedule. Supports optional Log Analytics ingestion for dashboarding. -| Version | Path | Runtime | Notes | -|---|---|---|---| -| **C#** (legacy) | [`Old/`](Old/README.md) | .NET 10 Native AOT (~15 MB standalone binary, no runtime required) | Moved to `Old/`; not actively maintained | -| **PowerShell** | [`Functions/GetAvail/get-availability.ps1`](Functions/GetAvail/get-availability.ps1) | PowerShell 7+ with `Az.Accounts` and `Az.ResourceGraph` modules | No build step; convenient for ad-hoc use; supports Log Analytics ingestion; also runs as an Azure Function | - -Both versions share the same pipeline, classification rules, output format, and invariants. +> A legacy C# (Native AOT) implementation is preserved in [`Old/`](Old/README.md) but is not actively maintained. For each resource, the tool answers: @@ -37,8 +32,6 @@ The relationship `Suspect = Faults + Excused + Unresolved` always holds. If Azure authentication fails, the tool prints the module exception message directly. Re-run `Connect-AzAccount` to fix. -For the C# version prerequisites and usage, see the [C# README](Old/README.md). - ### Parameters | Parameter | Default | Description | @@ -56,8 +49,6 @@ For the C# version prerequisites and usage, see the [C# README](Old/README.md). | `-DcrImmutableId` | *(none)* | Data Collection Rule immutable ID. Required together with `-DceEndpoint` to enable Log Analytics ingestion. | | `-Version` | | Print version and exit | -For C# parameters, see the [C# README](Old/README.md). - The observation window is a UTC calendar month: past months use the full calendar month, the current month is reported month-to-date. The requested month cannot start more than 90 days before the current UTC time. Metrics and Activity Log support that 90-day lookback; Health History is applied only for its overlap with the ~30-day REST API retention window. When `-SourceWorkspaceId` / `--workspace` is used, Health History coverage extends to the full observation period via a hybrid approach (Log Analytics for older transitions + REST API for the last ~30 days). ### Examples @@ -87,8 +78,6 @@ The observation window is a UTC calendar month: past months use the full calenda ./Functions/GetAvail/get-availability.ps1 -Subscriptions 'Contoso-Production' -Month 202603 | Export-Csv availability.csv ``` -For C# examples, see the [C# README](Old/README.md). - ### Output The header line shows the observation window and total minutes: @@ -243,8 +232,6 @@ AvailabilityPct = 40,066 / 40,125 × 100 = 99.85390% ## Implementation notes -These notes cover performance and implementation details specific to the PowerShell version. For C# implementation notes, see the [C# README](Old/README.md). - - **`ForEach-Object -Parallel`** for concurrent metric, Activity Log, and Resource Health queries with configurable parallelism. - **Shared `HttpClient`** with connection pooling — avoids per-request TCP/TLS overhead; streams JSON responses directly into `System.Text.Json` without intermediate string allocation. Used for both per-resource metrics and gap investigation paths. - **Compiled metric processor** — the ~44k-datapoint-per-resource JSON processing loop is compiled as C# via `Add-Type` and runs at native .NET speed.