From 52ec7334d3f5ba311ed369eaff2e8f1b9317ad3a Mon Sep 17 00:00:00 2001 From: Connor Dalton Date: Fri, 13 Nov 2026 14:53:17 -0700 Subject: [PATCH 1/2] Azure disk, Key Vault, storage and SQL MI fetchers: unread lists give null counts, not 0 When a list could not be read (403, throttling, timeout), every total, count and percentage built from it is now null instead of 0. Sub-lists a count is built from (diagnostic settings, containers, replication status, protectors, databases, failover groups) null their counts the same way. The runs still fail as partial_failure. Descriptions say so; versions bumped. Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 29 ++ .../azure/_shared/sql_managed_instance.py | 11 +- .../azure/disk_encryption_status/fetcher.py | 9 +- .../azure/disk_encryption_status/fetcher.yaml | 12 +- .../azure/key_vault_configuration/fetcher.py | 26 +- .../key_vault_configuration/fetcher.yaml | 16 +- .../sql_managed_instance_backups/fetcher.py | 58 ++-- .../sql_managed_instance_backups/fetcher.yaml | 24 +- .../fetcher.py | 57 ++-- .../fetcher.yaml | 12 +- .../storage_encryption_status/fetcher.py | 33 ++- .../storage_encryption_status/fetcher.yaml | 26 +- ...zure_unread_lists_give_null_counts_data.py | 265 ++++++++++++++++++ 13 files changed, 480 insertions(+), 98 deletions(-) create mode 100644 tests/test_azure_unread_lists_give_null_counts_data.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 51d2f4979..a616dc8e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2388,6 +2388,35 @@ schemas and the `paramify` CLI — not the internal code. ### Fixed +- **Disk, Key Vault, storage and SQL Managed Instance fetchers: an unread list + gives null counts, not 0.** When a list the run needs is refused (403), + throttled or times out, every total, count and percentage built from it is + null. Before, they read 0, which looked like "none found". Each run still + fails as `partial_failure`. + - `azure_disk_encryption_status` 0.3.0: when the disk list is not read, every + disk count and percentage is null (including `total_disks`). + - `azure_key_vault_configuration` 0.5.0: when the vault list is not read, + every count is null. When any vault's diagnostic settings are not read, + `vaults_with_diagnostic_settings`, `vaults_with_audit_logging`, + `vaults_without_audit_logging` and `audit_logging_percentage` are null too; + `vaults_audit_logging_unknown` still says how many. + - `azure_storage_encryption_status` 0.5.0: when the account list is not read, + every count is null (`total_storage_accounts` included). When any account's + container list is not read, the container counts and `in_scope_containers` + and `excluded_containers` are null; when any geo-redundant account's + replication status is not read, `geo_replication_live_accounts` and + `geo_failover_ready_accounts` are null. + - `azure_sql_managed_instance_encryption` 0.4.0: when the instance list is + not read, every count is null (`total_managed_instances` and + `in_scope_managed_instances` included). The key counts are null when any + instance's protector is not read, and the TDE counts when any instance's + database list is not read. + - `azure_sql_managed_instance_backups` 0.5.0: when the instance list is not + read, every instance, database and failover group count is null. The + database retention counts are null when any instance's database list is not + read, and `instances_in_failover_group` when any instance's failover group + membership is unknown. + - **`azure_entra_authentication_policy` no longer shows Microsoft's password defaults when the tenant's settings could not be read.** Before, a failed `/groupSettings` read fell back to the template, so `password_settings_source` diff --git a/fetchers/azure/_shared/sql_managed_instance.py b/fetchers/azure/_shared/sql_managed_instance.py index 587cdd522..b49c559ac 100644 --- a/fetchers/azure/_shared/sql_managed_instance.py +++ b/fetchers/azure/_shared/sql_managed_instance.py @@ -29,13 +29,18 @@ def _client(): return collector.guard("sql.SqlManagementClient (init)", _client) -def list_instances(client, collector: Collector, project: Callable) -> list[dict]: - """Every instance, projected, with `resource_group`; one without a group is recorded and dropped.""" +def list_instances(client, collector: Collector, project: Callable, unread=()) -> list[dict]: + """Every instance, projected, with `resource_group`; one without a group is recorded and dropped. + + A list that could not be read gives `unread` (an empty list by default; pass None to tell it from no instances). + """ projected = collector.guard( "sql.managed_instances.list", lambda: [project(i) for i in client.managed_instances.list()], - default=[], + default=None, ) + if projected is None: + return list(unread) if unread is not None else None instances = [] for instance in projected: group = resource_group_from_id(instance.get("id")) diff --git a/fetchers/azure/disk_encryption_status/fetcher.py b/fetchers/azure/disk_encryption_status/fetcher.py index 6e0e64437..31cbe2080 100644 --- a/fetchers/azure/disk_encryption_status/fetcher.py +++ b/fetchers/azure/disk_encryption_status/fetcher.py @@ -209,12 +209,16 @@ def _cmk_coverage(disks: list[dict], prefix: str = "") -> dict: } -def summarize(disks: list[dict]) -> dict: +def summarize(disks: list[dict] | None) -> dict: """CMK coverage is the headline, computed three ways: all / attached / unattached. + `None` means the disk list was not read: every count and percentage is then null, not 0. + Managed disks are encrypted at rest unconditionally, so a generic "encrypted" percentage would be a constant 100 and prove nothing. """ + if disks is None: + return {k: None for k in summarize([])} attached = [d for d in disks if d["attached"]] unattached = [d for d in disks if not d["attached"]] return { @@ -318,6 +322,7 @@ def main() -> int: if not listed: exclusions.mark_unread() exclusions.warn_unmatched(logger) + summary = summarize(disks) disks = disks or [] evidence = build_payload( @@ -330,7 +335,7 @@ def main() -> int: "excluded": excluded, }, summary={ - **summarize(disks), + **summary, "provider_registration_status": registration, **exclusions.summary(excluded, KINDS, {"disks": len(disks) if listed else None}), }, diff --git a/fetchers/azure/disk_encryption_status/fetcher.yaml b/fetchers/azure/disk_encryption_status/fetcher.yaml index 4a1210603..c96046593 100644 --- a/fetchers/azure/disk_encryption_status/fetcher.yaml +++ b/fetchers/azure/disk_encryption_status/fetcher.yaml @@ -1,5 +1,5 @@ name: azure_disk_encryption_status -version: 0.2.1 +version: 0.3.0 description: >- Shows which key encrypts each managed disk at rest, whether Azure Disk Encryption is on, whether the disk is attached, and how exposed its export path is. category: azure @@ -84,12 +84,12 @@ evidence_set: - in_scope_disks — Number of disks read and in scope; null when the disk list could not be read; 0 means there are none or the patterns took out every disk, which a check can catch by requiring more than 0 Which disks exist? - - total_disks — Number of in-scope managed disks, attached or not; reads 0, not null, when the disk list could not be read, so read it together with in_scope_disks - - attached_total_disks — Number of disks attached to a virtual machine (the disk names an owner or a shared-disk owner list) - - unattached_total_disks — Number of disks attached to no virtual machine; they still hold the data of the machine they were detached from + - total_disks — Number of in-scope managed disks, attached or not; null when the disk list could not be read, as is every count and percentage below + - attached_total_disks — Number of disks attached to a virtual machine (the disk names an owner or a shared-disk owner list); null when the disk list could not be read + - unattached_total_disks — Number of disks attached to no virtual machine; they still hold the data of the machine they were detached from; null when the disk list could not be read - provider_registration_status — Whether the Compute provider (Microsoft.Compute) is registered on the subscription: registered, not_registered or unknown - - os_disks — Number of disks that report an operating system type - - data_disks — Number of disks that report no operating system type + - os_disks — Number of disks that report an operating system type; null when the disk list could not be read + - data_disks — Number of disks that report no operating system type; null when the disk list could not be read Which key encrypts the disks at rest? - customer_managed_key_disks — Number of disks encrypted with a customer-managed key (an encryption type other than the platform key, held in a disk encryption set). Managed disks are always encrypted at rest, so this is what varies diff --git a/fetchers/azure/key_vault_configuration/fetcher.py b/fetchers/azure/key_vault_configuration/fetcher.py index 3f963d743..030685a9b 100644 --- a/fetchers/azure/key_vault_configuration/fetcher.py +++ b/fetchers/azure/key_vault_configuration/fetcher.py @@ -374,8 +374,27 @@ def with_audit_logging(vault: dict, settings) -> dict: } -def summarize(vaults: list[dict]) -> dict: - """RBAC adoption and recoverability are the headlines.""" +AUDIT_COUNTS = ( + "vaults_with_diagnostic_settings", "vaults_with_audit_logging", "vaults_without_audit_logging", + "audit_logging_percentage", +) + + +def summarize(vaults: list[dict] | None) -> dict: + """RBAC adoption and recoverability are the headlines. + + `None` means the vault list was not read: every count is null, not 0. The counts built from + diagnostic settings are null too while any vault's settings could not be read. + """ + if vaults is None: + return {k: None for k in summarize([])} + counts = _summarize(vaults) + if any(v.get("audit_logging_enabled") is None for v in vaults): + counts.update({k: None for k in AUDIT_COUNTS}) + return counts + + +def _summarize(vaults: list[dict]) -> dict: total = len(vaults) rbac = sum(1 for v in vaults if v["rbac_authorization_enabled"]) recoverable = sum(1 for v in vaults if v["recoverable"]) @@ -537,6 +556,7 @@ def main() -> int: exclusions.record_invalid(collector) listed = vaults is not None + summary = summarize(vaults) vaults = vaults or [] if not listed: exclusions.mark_unread() @@ -552,7 +572,7 @@ def main() -> int: "excluded": excluded, }, summary={ - **summarize(vaults), + **summary, "provider_registration_status": registration, **exclusions.summary(excluded, ("key_vault",), {"key_vaults": len(vaults) if listed else None}), }, diff --git a/fetchers/azure/key_vault_configuration/fetcher.yaml b/fetchers/azure/key_vault_configuration/fetcher.yaml index cd63024e0..6b19576fe 100644 --- a/fetchers/azure/key_vault_configuration/fetcher.yaml +++ b/fetchers/azure/key_vault_configuration/fetcher.yaml @@ -1,5 +1,5 @@ name: azure_key_vault_configuration -version: 0.4.1 +version: 0.5.0 description: >- Shows how each key vault grants access, whether deleted vaults and secrets can be recovered, how it's exposed to the network, and whether its audit events are logged. category: azure @@ -86,20 +86,20 @@ evidence_set: - in_scope_key_vaults — Number of key vaults read and in scope; null when the key vault list could not be read; 0 means there are none or the patterns took out every vault, which a check can catch by requiring more than 0 Which vaults were checked? - - total_key_vaults — Number of in-scope key vaults; every count and percentage below covers only these; reads 0, not null, when the key vault list could not be read, so read it together with in_scope_key_vaults + - total_key_vaults — Number of in-scope key vaults; every count and percentage below covers only these; null when the key vault list could not be read, as is every count and percentage below - provider_registration_status — Whether Microsoft.KeyVault is registered on the subscription: registered, not_registered or unknown How is access to the vaults granted? - rbac_authorization_vaults — Number of vaults that use Azure RBAC role assignments for access to keys, secrets and certificates (the role assignments aren't in this evidence) - access_policy_vaults — Number of vaults that use the older vault access policies instead of Azure RBAC - - rbac_authorization_percentage — Percentage of vaults that use Azure RBAC = rbac_authorization_vaults / total_key_vaults × 100 (a whole number, rounded down; reads 0 when total_key_vaults is 0) + - rbac_authorization_percentage — Percentage of vaults that use Azure RBAC = rbac_authorization_vaults / total_key_vaults × 100 (a whole number, rounded down; reads 0 when total_key_vaults is 0, null when the key vault list could not be read) - total_access_policies — Number of access policy entries across all vaults, each giving a user, group or app permissions on a vault's keys, secrets, certificates or storage accounts; 0 when the key vault list could not be read Can a deleted vault or secret be recovered? - soft_delete_vaults — Number of vaults with soft delete on, so deleted vaults, keys and secrets are kept for the retention period and can be restored; a vault that doesn't report the setting counts as off - purge_protection_vaults — Number of vaults with purge protection on, so nobody can permanently delete the vault or its contents before the retention period ends - recoverable_vaults — Number of recoverable vaults: both soft delete and purge protection on - - recoverable_percentage — Percentage of vaults that are recoverable = recoverable_vaults / total_key_vaults × 100 (a whole number, rounded down; reads 0 when total_key_vaults is 0) + - recoverable_percentage — Percentage of vaults that are recoverable = recoverable_vaults / total_key_vaults × 100 (a whole number, rounded down; reads 0 when total_key_vaults is 0, null when the key vault list could not be read) Is network access restricted? - public_network_access_disabled_vaults — Number of vaults with public network access disabled, reachable only through private endpoints; a vault that doesn't report the setting counts as enabled @@ -110,8 +110,8 @@ evidence_set: - premium_sku_vaults — Number of vaults on the Premium tier, which can hold keys in hardware security modules (HSMs); whether each key is HSM-backed isn't checked Is every operation on keys and secrets logged? - - vaults_with_diagnostic_settings — Number of vaults with at least one diagnostic setting, whatever it exports - - vaults_with_audit_logging — Number of vaults with audit logging: a diagnostic setting that exports the AuditEvent category (directly, or through the audit or allLogs group) to a storage account, Log Analytics workspace, event hub or partner solution; AuditEvent records every read or change of a key, secret or certificate - - vaults_without_audit_logging — Number of vaults whose diagnostic settings were read and include no audit logging + - vaults_with_diagnostic_settings — Number of vaults with at least one diagnostic setting, whatever it exports; null when any vault's diagnostic settings couldn't be read + - vaults_with_audit_logging — Number of vaults with audit logging: a diagnostic setting that exports the AuditEvent category (directly, or through the audit or allLogs group) to a storage account, Log Analytics workspace, event hub or partner solution; AuditEvent records every read or change of a key, secret or certificate; null when any vault's diagnostic settings couldn't be read + - vaults_without_audit_logging — Number of vaults whose diagnostic settings were read and include no audit logging; null when any vault's diagnostic settings couldn't be read - vaults_audit_logging_unknown — Number of vaults whose diagnostic settings couldn't be read - - audit_logging_percentage — Percentage of vaults with audit logging = vaults_with_audit_logging / total_key_vaults × 100 (a whole number, rounded down; reads 0 when total_key_vaults is 0); vaults whose settings couldn't be read count in the total but not as logged + - audit_logging_percentage — Percentage of vaults with audit logging = vaults_with_audit_logging / total_key_vaults × 100 (a whole number, rounded down; reads 0 when total_key_vaults is 0, null when the key vault list could not be read); null when any vault's diagnostic settings couldn't be read diff --git a/fetchers/azure/sql_managed_instance_backups/fetcher.py b/fetchers/azure/sql_managed_instance_backups/fetcher.py index a8b67a99a..fef1d9ac7 100644 --- a/fetchers/azure/sql_managed_instance_backups/fetcher.py +++ b/fetchers/azure/sql_managed_instance_backups/fetcher.py @@ -257,9 +257,17 @@ def split_failover_groups( return kept, dropped +def _if_listed(listed: bool, count: int): + return count if listed else None + + def summarize( - instances: list[dict], failover_groups: list[dict], failover_groups_collected: bool + instances: list[dict] | None, failover_groups: list[dict], failover_groups_collected: bool ) -> dict: + """`None` means the instance list was not read: every count is null, not 0. Counts built from a + database list or a failover group list that failed for some instance are null too.""" + if instances is None: + return {k: None for k in summarize([], [], True)} databases = [d for i in instances for d in i["databases"] if not d["is_system_database"]] redundancy: dict[str, int] = {} for i in instances: @@ -269,29 +277,34 @@ def summarize( return { "total_managed_instances": len(instances), "total_user_databases": len(databases) if all_listed else None, - "databases_retention_at_least_7_days": sum( + "databases_retention_at_least_7_days": _if_listed(all_listed, sum( 1 for d in databases if d["short_term_retention_at_least_7_days"] is True + )), + "databases_retention_below_7_days": _if_listed( + all_listed, sum(i["databases_below_7_days"] or 0 for i in instances) ), - "databases_retention_below_7_days": sum(i["databases_below_7_days"] or 0 for i in instances), - "databases_missing_retention_policy": sum( + "databases_missing_retention_policy": _if_listed(all_listed, sum( 1 for d in databases if d["short_term_retention_policy_found"] is False - ), - "databases_retention_unknown": sum( + )), + "databases_retention_unknown": _if_listed(all_listed, sum( 1 for d in databases if d["short_term_retention_at_least_7_days"] is None - ), - "databases_with_long_term_retention": sum( + )), + "databases_with_long_term_retention": _if_listed(all_listed, sum( 1 for d in databases if d["long_term_retention"]["configured"] is True - ), - "databases_long_term_retention_unknown": sum( + )), + "databases_long_term_retention_unknown": _if_listed(all_listed, sum( 1 for d in databases if d["long_term_retention"]["configured"] is None - ), + )), "instances_by_current_backup_storage_redundancy": redundancy, "instances_backup_storage_zone_redundant": sum( 1 for i in instances if i["backup_storage_zone_redundant"] ), "instances_stopped": sum(1 for i in instances if lower(i.get("state")) == "stopped"), "instances_zone_redundant": sum(1 for i in instances if i["zone_redundant"]), - "instances_in_failover_group": sum(1 for i in instances if i["in_failover_group"] is True), + "instances_in_failover_group": _if_listed( + all(i["in_failover_group"] is not None for i in instances), + sum(1 for i in instances if i["in_failover_group"] is True), + ), "instances_failover_group_unknown": sum(1 for i in instances if i["in_failover_group"] is None), "total_instance_failover_groups": len(failover_groups) if failover_groups_collected else None, "failover_groups_manual_policy": sum( @@ -500,17 +513,20 @@ def list_failover_groups(client, group_name: str, location: str) -> list[dict]: def collect(subscription_id, cred, collector: Collector, exclusions: ResourceExclusions | None = None, - excluded: list | None = None) -> tuple[list[dict], list[dict], set]: - """Failover groups are listed and matched across every instance; excluded instances get no database reads.""" + excluded: list | None = None) -> tuple[list[dict] | None, list[dict], set]: + """Failover groups are listed and matched across every instance; excluded instances get no database reads. + The instances are None when the instance list could not be read.""" client = sql_client(subscription_id, cred, collector) if client is None: if exclusions is not None: exclusions.mark_unread() - return [], [], set() + return None, [], set() failures = len(collector.failures) - projected = list_instances(client, collector, project_instance) + projected = list_instances(client, collector, project_instance, unread=None) if exclusions is not None and len(collector.failures) > failures: exclusions.mark_unread() + if projected is None: + return None, [], set() in_scope, out = exclusions.split(projected, "managed_instance") if exclusions else (projected, []) in_scope_pairs = {(i["resource_group"], i.get("location")) for i in in_scope} @@ -600,7 +616,7 @@ def main() -> int: subscription_id = sub["subscription_id"] cred = collector.guard("azure.identity.DefaultAzureCredential", credential) - instances: list[dict] = [] + instances: list[dict] | None = None groups: list[dict] = [] unlisted: set = set() excluded: list[dict] = [] @@ -633,6 +649,9 @@ def main() -> int: window["days_requested"] = days_requested exclusions.record_invalid(collector) + listed = instances is not None + summary = summarize(instances, groups, not unlisted) + instances = instances or [] failovers, out = attach_failovers( None if events is None else failover_records(events), instances, groups, excluded, exclusions) excluded += out @@ -641,7 +660,8 @@ def main() -> int: exclusions.mark_unread() exclusions.warn_unmatched(logger) boundary = exclusions.summary(excluded, ("managed_instance", "instance_failover_group", "failover"), { - "managed_instances": len(instances), "instance_failover_groups": len(groups), + "managed_instances": len(instances) if listed else None, + "instance_failover_groups": len(groups) if listed and not unlisted else None, "failovers": None if failovers is None else len(failovers)}) if failovers is None: # No failover was read, so none was tested against the patterns. @@ -660,7 +680,7 @@ def main() -> int: "excluded": excluded, }, summary={ - **summarize(instances, groups, not unlisted), + **summary, **summarize_failovers(failovers, window, now), "provider_registration_status": registration, **boundary, diff --git a/fetchers/azure/sql_managed_instance_backups/fetcher.yaml b/fetchers/azure/sql_managed_instance_backups/fetcher.yaml index b255330ee..da3ffcf6c 100644 --- a/fetchers/azure/sql_managed_instance_backups/fetcher.yaml +++ b/fetchers/azure/sql_managed_instance_backups/fetcher.yaml @@ -1,5 +1,5 @@ name: azure_sql_managed_instance_backups -version: 0.4.1 +version: 0.5.0 description: >- Shows whether each SQL Managed Instance's databases keep enough backup retention, whether backup storage and compute survive a zone or region loss, and the failovers run recently. category: azure @@ -106,33 +106,33 @@ evidence_set: - excluded_managed_instances — Number of managed instances left out of scope because a pattern matched their resource group, region or id; each is listed under excluded with the pattern, its databases and their retention are never read, and no other count includes it - excluded_instance_failover_groups — Number of instance failover groups left out of scope because every instance of the group in this subscription is excluded; each is listed under excluded, naming that instance in matched_id - excluded_failovers — Number of failovers of an excluded failover group or instance, or of one no longer listed whose resource id a pattern matches; null when the failover history wasn't read in full, since no failover was tested against the patterns - - in_scope_managed_instances — Number of managed instances read and in scope; 0 means none was read or the patterns took out every instance, which a check can catch by requiring more than 0 - - in_scope_instance_failover_groups — Number of instance failover groups read and in scope; 0 means none was read or the patterns took out every group + - in_scope_managed_instances — Number of managed instances read and in scope; null when the instance list could not be read; 0 means none was read or the patterns took out every instance, which a check can catch by requiring more than 0 + - in_scope_instance_failover_groups — Number of instance failover groups read and in scope; null when the instance list could not be read or a failover group list failed for a resource group and region holding an in-scope instance; 0 means none was read or the patterns took out every group - in_scope_failovers — Number of failovers read and in scope; null when the failover history wasn't read in full What was checked? - - total_managed_instances — Number of in-scope SQL Managed Instances; reads 0, not null, when the instance list could not be read (the run then fails) + - total_managed_instances — Number of in-scope SQL Managed Instances; null when the instance list could not be read (the run then fails), as is every count below that is built from instances - total_user_databases — Number of user databases on in-scope instances (system databases left out); null if any instance's database list couldn't be read - instances_stopped — Number of instances whose state is Stopped; Azure takes no backups while an instance is stopped, so read retention together with this - provider_registration_status — Whether the SQL provider (Microsoft.Sql) is registered on the subscription: registered, not_registered or unknown Do databases keep short-term (point-in-time-restore) backups long enough? - - databases_retention_at_least_7_days — Number of user databases whose point-in-time-restore retention is 7 days or more (Azure's default is 7; the range is 1 to 35) - - databases_retention_below_7_days — Number of user databases whose retention policy was read and is under 7 days - - databases_missing_retention_policy — Number of user databases with no short-term retention policy (the read answered 404) - - databases_retention_unknown — Number of user databases whose retention couldn't be judged because the policy read failed or returned no day count; a database with no policy is in databases_missing_retention_policy instead + - databases_retention_at_least_7_days — Number of user databases whose point-in-time-restore retention is 7 days or more (Azure's default is 7; the range is 1 to 35); null when any instance's database list couldn't be read + - databases_retention_below_7_days — Number of user databases whose retention policy was read and is under 7 days; null when any instance's database list couldn't be read + - databases_missing_retention_policy — Number of user databases with no short-term retention policy (the read answered 404); null when any instance's database list couldn't be read + - databases_retention_unknown — Number of user databases whose retention couldn't be judged because the policy read failed or returned no day count; a database with no policy is in databases_missing_retention_policy instead; null when any instance's database list couldn't be read Are long-term backups kept? - - databases_with_long_term_retention — Number of user databases with a weekly, monthly or yearly long-term retention tier on (a duration of PT0S or none means that tier is off) - - databases_long_term_retention_unknown — Number of user databases whose long-term retention policy couldn't be read, so they count in neither field above nor as unconfigured + - databases_with_long_term_retention — Number of user databases with a weekly, monthly or yearly long-term retention tier on (a duration of PT0S or none means that tier is off); null when any instance's database list couldn't be read + - databases_long_term_retention_unknown — Number of user databases whose long-term retention policy couldn't be read, so they count in neither field above nor as unconfigured; null when any instance's database list couldn't be read Is backup storage redundant? - - instances_by_current_backup_storage_redundancy — Number of instances for each backup storage redundancy currently in effect, keyed by the value Azure reports (Local, Zone, Geo or GeoZone), or unknown when it reports none; a change to the requested value applies only to future backups and can take up to 24 hours. Geo keeps its primary-region copies in a single availability zone + - instances_by_current_backup_storage_redundancy — Number of instances for each backup storage redundancy currently in effect, keyed by the value Azure reports (Local, Zone, Geo or GeoZone), or unknown when it reports none, and null when the instance list could not be read; a change to the requested value applies only to future backups and can take up to 24 hours. Geo keeps its primary-region copies in a single availability zone - instances_backup_storage_zone_redundant — Number of instances whose current backup storage redundancy is Zone or GeoZone Does the instance survive a zone or region loss? - instances_zone_redundant — Number of instances with compute zone redundancy on, a setting separate from backup storage redundancy (Azure omits it when off, so absent counts as off) - - instances_in_failover_group — Number of instances that are in an instance failover group + - instances_in_failover_group — Number of instances that are in an instance failover group; null when any instance's failover group membership is unknown - instances_failover_group_unknown — Number of instances whose failover group membership is unknown: the failover group list for their resource group and region failed and no other region's list showed them in a group - total_instance_failover_groups — Number of in-scope instance failover groups, each counted once although Azure lists it from every region it spans; null if a failover group list failed for a resource group and region holding an in-scope instance (a failed list where every instance is excluded is only logged as a warning, and the run doesn't fail) - failover_groups_manual_policy — Number of failover groups whose failover policy is Manual, so a failover needs someone to start it; null if a failover group list failed for a resource group and region holding an in-scope instance diff --git a/fetchers/azure/sql_managed_instance_encryption/fetcher.py b/fetchers/azure/sql_managed_instance_encryption/fetcher.py index 363a3aa74..05a613e50 100644 --- a/fetchers/azure/sql_managed_instance_encryption/fetcher.py +++ b/fetchers/azure/sql_managed_instance_encryption/fetcher.py @@ -92,45 +92,57 @@ def instance_record(instance: dict, protector: dict | None, databases: list[dict "databases_collected": databases is not None, "databases": databases or [], "total_user_databases": None if users is None else len(users), - "tde_enabled_user_databases": tde.count(True), - "tde_disabled_user_databases": tde.count(False), - "tde_unknown_user_databases": tde.count(None), + "tde_enabled_user_databases": None if users is None else tde.count(True), + "tde_disabled_user_databases": None if users is None else tde.count(False), + "tde_unknown_user_databases": None if users is None else tde.count(None), "all_user_databases_tde_enabled": None if users is None else all_enabled(tde), } -def summarize(instances: list[dict]) -> dict: +def _total(values: list): + """Sum, or null while any term is null (an unread list).""" + return None if None in values else sum(values) + + +def summarize(instances: list[dict] | None) -> dict: + """`None` means the instance list was not read: every count is null, not 0. A count built from a + protector or database list that failed for some instance is null too.""" + if instances is None: + return {k: None for k in summarize([])} + protectors_read = all(i["encryption_protector"] is not None for i in instances) return { "total_managed_instances": len(instances), - "instances_customer_managed_key": sum(1 for i in instances if i["customer_managed_key"]), - "instances_key_auto_rotation": sum( - 1 for i in instances if (i["encryption_protector"] or {}).get("auto_rotation_enabled") + "instances_customer_managed_key": ( + sum(1 for i in instances if i["customer_managed_key"]) if protectors_read else None ), - "total_user_databases": ( - sum(i["total_user_databases"] for i in instances) - if all(i["databases_collected"] for i in instances) - else None + "instances_key_auto_rotation": ( + sum(1 for i in instances if i["encryption_protector"]["auto_rotation_enabled"]) + if protectors_read else None ), - "tde_enabled_user_databases": sum(i["tde_enabled_user_databases"] for i in instances), - "tde_disabled_user_databases": sum(i["tde_disabled_user_databases"] for i in instances), - "tde_unknown_user_databases": sum(i["tde_unknown_user_databases"] for i in instances), + "total_user_databases": _total([i["total_user_databases"] for i in instances]), + "tde_enabled_user_databases": _total([i["tde_enabled_user_databases"] for i in instances]), + "tde_disabled_user_databases": _total([i["tde_disabled_user_databases"] for i in instances]), + "tde_unknown_user_databases": _total([i["tde_unknown_user_databases"] for i in instances]), } # --- collection (lazy azure imports) --- def collect(subscription_id, cred, collector: Collector, exclusions: ResourceExclusions | None = None, - excluded: list | None = None) -> list[dict]: - """Instances the exclusions take out are appended to `excluded` and get none of the per-instance reads.""" + excluded: list | None = None) -> list[dict] | None: + """Instances the exclusions take out are appended to `excluded` and get none of the per-instance reads. + None when the instance list could not be read.""" client = sql_client(subscription_id, cred, collector) if client is None: if exclusions is not None: exclusions.mark_unread() - return [] + return None failures = len(collector.failures) - instances = list_instances(client, collector, project_instance) + instances = list_instances(client, collector, project_instance, unread=None) if exclusions is not None and len(collector.failures) > failures: exclusions.mark_unread() + if instances is None: + return None if exclusions and excluded is not None: instances, out = exclusions.split(instances, "managed_instance") excluded += sorted(out, key=lambda e: e["id"] or "") @@ -184,7 +196,7 @@ def main() -> int: subscription_id = sub["subscription_id"] cred = collector.guard("azure.identity.DefaultAzureCredential", credential) - instances: list[dict] = [] + instances: list[dict] | None = None excluded: list[dict] = [] exclusions = ResourceExclusions.from_env() registration = REGISTRATION_UNKNOWN @@ -212,6 +224,9 @@ def main() -> int: exclusions.mark_unread() exclusions.record_invalid(collector) exclusions.warn_unmatched(logger) + summary = summarize(instances) + in_scope = None if instances is None else len(instances) + instances = instances or [] evidence = build_payload( subscription_id=subscription_id, @@ -223,9 +238,9 @@ def main() -> int: "excluded": excluded, }, summary={ - **summarize(instances), + **summary, "provider_registration_status": registration, - **exclusions.summary(excluded, ("managed_instance",), {"managed_instances": len(instances)}), + **exclusions.summary(excluded, ("managed_instance",), {"managed_instances": in_scope}), }, ) diff --git a/fetchers/azure/sql_managed_instance_encryption/fetcher.yaml b/fetchers/azure/sql_managed_instance_encryption/fetcher.yaml index 5995ff2a6..b71f43994 100644 --- a/fetchers/azure/sql_managed_instance_encryption/fetcher.yaml +++ b/fetchers/azure/sql_managed_instance_encryption/fetcher.yaml @@ -1,5 +1,5 @@ name: azure_sql_managed_instance_encryption -version: 0.3.1 +version: 0.4.0 description: >- Shows whether each SQL Managed Instance's encryption protector is a customer-managed key with auto-rotation, and whether transparent data encryption is on for every user database. category: azure @@ -82,18 +82,18 @@ evidence_set: - resource_exclusion_patterns — List of the configured resource_exclusions patterns (rg:, location: or id:); empty when none are set, so every instance is in scope - resource_exclusion_patterns_unmatched — List of configured patterns that matched no managed instance, such as a mistyped name; null if the instance list couldn't be read - excluded_managed_instances — Number of managed instances left out of scope because a pattern matched their resource group, region or id; each is listed under excluded with the pattern, is not read further, and no other count includes it - - in_scope_managed_instances — Number of managed instances read and in scope; 0 means none was read or the patterns took out every instance, which a check can catch by requiring more than 0 + - in_scope_managed_instances — Number of managed instances read and in scope; null when the instance list could not be read; 0 means none was read or the patterns took out every instance, which a check can catch by requiring more than 0 What was checked? - - total_managed_instances — Number of in-scope SQL Managed Instances; reads 0, not null, when the instance list could not be read (the run then fails) + - total_managed_instances — Number of in-scope SQL Managed Instances; null when the instance list could not be read (the run then fails), as is every count below - total_user_databases — Number of user databases on in-scope instances (system databases left out); null if any instance's database list couldn't be read - provider_registration_status — Whether the SQL provider (Microsoft.Sql) is registered on the subscription: registered, not_registered or unknown Is the encryption key customer-managed? - - instances_customer_managed_key — Number of instances whose TDE protector is a customer-managed Key Vault key (server key type AzureKeyVault); an instance whose protector couldn't be read doesn't count - - instances_key_auto_rotation — Number of instances with key auto-rotation on, where Azure omits the setting when it was never turned on, so absent counts as off; an instance whose protector couldn't be read doesn't count + - instances_customer_managed_key — Number of instances whose TDE protector is a customer-managed Key Vault key (server key type AzureKeyVault); null when any instance's protector couldn't be read + - instances_key_auto_rotation — Number of instances with key auto-rotation on, where Azure omits the setting when it was never turned on, so absent counts as off; null when any instance's protector couldn't be read Is every user database encrypted? - tde_enabled_user_databases — Number of user databases with TDE Enabled - tde_disabled_user_databases — Number of user databases with TDE Disabled - - tde_unknown_user_databases — Number of user databases whose TDE state couldn't be read, so they're never counted as unencrypted; an instance whose database list couldn't be read contributes none to any of the three counts + - tde_unknown_user_databases — Number of user databases whose TDE state couldn't be read, so they're never counted as unencrypted; all three counts are null when any instance's database list couldn't be read diff --git a/fetchers/azure/storage_encryption_status/fetcher.py b/fetchers/azure/storage_encryption_status/fetcher.py index 9d4b58c24..d79997c93 100644 --- a/fetchers/azure/storage_encryption_status/fetcher.py +++ b/fetchers/azure/storage_encryption_status/fetcher.py @@ -488,12 +488,24 @@ def file_service_properties_record(properties: dict) -> dict: } -def summarize(accounts: list[dict]) -> dict: +CONTAINER_COUNTS = ( + "total_containers", "containers_with_public_access_level", "anonymously_accessible_containers", + "containers_with_immutability_policy", "containers_with_legal_hold", +) +REPLICATION_COUNTS = ("geo_replication_live_accounts", "geo_failover_ready_accounts") + + +def summarize(accounts: list[dict] | None) -> dict: """CMK coverage is the headline, not an encrypted/total percentage. + `None` means the account list was not read: every count is null, not 0. A count built from + a list that failed for some account (its containers, its replication status) is null too. + Azure Storage encrypts at rest unconditionally, so a generic "encrypted" percentage would be a constant 100 and prove nothing. """ + if accounts is None: + return {k: None for k in summarize([])} cmk = sum(1 for a in accounts if a["customer_managed_key"]) total = len(accounts) return { @@ -529,10 +541,19 @@ def summarize(accounts: list[dict]) -> dict: for a in accounts if dig(a, "file_service_properties", "share_delete_retention_policy", "enabled") ), - **summarize_replication_keys_containers(accounts), + **_null_unread(accounts, summarize_replication_keys_containers(accounts)), } +def _null_unread(accounts: list[dict], counts: dict) -> dict: + """Null the counts built from a container list or replication status that failed for some account.""" + if any(a.get("containers_status") == CONTAINERS_FAILED for a in accounts): + counts.update({k: None for k in CONTAINER_COUNTS}) + if any((a.get("geo_replication_stats") or {}).get("stats_status") == STATS_FAILED for a in accounts): + counts.update({k: None for k in REPLICATION_COUNTS}) + return counts + + def summarize_replication_keys_containers(accounts: list[dict]) -> dict: geo = [a for a in accounts if a.get("geo_redundant")] stats = [a["geo_replication_stats"] for a in geo if a.get("geo_replication_stats")] @@ -799,8 +820,10 @@ def main() -> int: listed = accounts is not None accounts, out = exclude_containers(exclusions, accounts or []) excluded += out - if not listed or any(a.get("containers_status") == CONTAINERS_FAILED for a in accounts): + containers_read = listed and not any(a.get("containers_status") == CONTAINERS_FAILED for a in accounts) + if not containers_read: exclusions.mark_unread() + summary = summarize(accounts if listed else None) exclusions.warn_unmatched(logger) evidence = build_payload( @@ -813,11 +836,11 @@ def main() -> int: "excluded": excluded, }, summary={ - **summarize(accounts), + **summary, "provider_registration_status": registration, **exclusions.summary(excluded, ("storage_account", "container"), { "storage_accounts": len(accounts) if listed else None, - "containers": sum(len(a.get("containers") or []) for a in accounts), + "containers": sum(len(a.get("containers") or []) for a in accounts) if containers_read else None, }), }, ) diff --git a/fetchers/azure/storage_encryption_status/fetcher.yaml b/fetchers/azure/storage_encryption_status/fetcher.yaml index f7a55c71c..6675b9177 100644 --- a/fetchers/azure/storage_encryption_status/fetcher.yaml +++ b/fetchers/azure/storage_encryption_status/fetcher.yaml @@ -1,5 +1,5 @@ name: azure_storage_encryption_status -version: 0.4.1 +version: 0.5.0 description: >- Shows how each storage account is encrypted and exposed, how old its access keys are, whether it replicates to a healthy second region, which containers are public, and its owner. category: azure @@ -101,21 +101,21 @@ evidence_set: Summary fields: What was checked? - - total_storage_accounts — Number of in-scope storage accounts; every account count and percentage below covers only these; reads 0, not null, when the storage account list could not be read, so read it together with in_scope_storage_accounts + - total_storage_accounts — Number of in-scope storage accounts; every account count and percentage below covers only these; null when the storage account list could not be read, as is every count and percentage below - provider_registration_status — Whether Azure Storage (Microsoft.Storage) is registered on the subscription: registered, not_registered or unknown Which resources are left out of scope? - resource_exclusion_patterns — List of the configured resource_exclusions patterns (rg:, location: or id:); empty when none are set, so every resource is in scope - resource_exclusion_patterns_unmatched — List of configured patterns that matched no storage account or blob container, such as a mistyped name; null when the storage account list or any account's container list could not be read - excluded_storage_accounts — Number of storage accounts left out of scope because a pattern matched their resource group, region or id; each is listed under excluded with the pattern, no other count includes them, and their containers are never read; null when the storage account list could not be read - - excluded_containers — Number of blob containers an id: pattern took out of an in-scope account; each is listed under excluded. An excluded account's containers are never read, so they are not counted here + - excluded_containers — Number of blob containers an id: pattern took out of an in-scope account; each is listed under excluded. An excluded account's containers are never read, so they are not counted here; null when the storage account list or any account's container list could not be read - in_scope_storage_accounts — Number of storage accounts in scope; null when the storage account list could not be read, 0 means none, or the patterns took out every account, which a check can catch by requiring more than 0 - - in_scope_containers — Number of blob containers listed and in scope; never null, and accounts whose container list failed add none + - in_scope_containers — Number of blob containers listed and in scope; null when the storage account list or any account's container list could not be read Who holds the encryption keys? - customer_managed_key_storage — Number of storage accounts encrypted with a customer-managed key held in Key Vault - platform_managed_key_storage — Number of storage accounts encrypted with Microsoft's platform-managed key - - cmk_percentage — Percentage of storage accounts using a customer-managed key = customer_managed_key_storage / total_storage_accounts × 100 (a whole number, rounded down; reads 0 when total_storage_accounts is 0) + - cmk_percentage — Percentage of storage accounts using a customer-managed key = customer_managed_key_storage / total_storage_accounts × 100 (a whole number, rounded down; reads 0 when total_storage_accounts is 0, null when the storage account list could not be read) - infrastructure_encryption_accounts — Number of storage accounts with infrastructure (double) encryption on Is data encrypted in transit? @@ -138,9 +138,9 @@ evidence_set: - geo_redundant_accounts — Number of storage accounts that replicate to a second region (GRS, RA-GRS, GZRS or RA-GZRS) - read_access_geo_redundant_accounts — Number of storage accounts whose second-region copy can be read (RA-GRS or RA-GZRS) - geo_replication_stats_collected_accounts — Number of geo-redundant storage accounts whose replication status was read - - geo_replication_live_accounts — Number of geo-redundant storage accounts whose replication status is Live (replicating normally) + - geo_replication_live_accounts — Number of geo-redundant storage accounts whose replication status is Live (replicating normally); null when any account's replication status could not be read - geo_replication_max_sync_lag_minutes — Number of minutes the second-region copy furthest behind trails its primary: the time since it last synced, measured when its replication status was read; null when there is no geo-redundant account or any one's last sync time is unknown - - geo_failover_ready_accounts — Number of geo-redundant storage accounts that Azure reports can fail over to the second region now + - geo_failover_ready_accounts — Number of geo-redundant storage accounts that Azure reports can fail over to the second region now; null when any account's replication status could not be read - secondary_unavailable_accounts — Number of geo-redundant storage accounts whose second region is reported unavailable - geo_failed_over_accounts — Number of storage accounts that have failed over to their second region at least once @@ -148,14 +148,14 @@ evidence_set: - blob_versioning_accounts — Number of storage accounts with blob versioning on - container_soft_delete_accounts — Number of storage accounts that keep deleted blob containers for a retention period (container soft delete) - share_soft_delete_accounts — Number of storage accounts that keep deleted file shares for a retention period (share soft delete) - - containers_with_immutability_policy — Number of blob containers with a time-based retention (immutability) policy - - containers_with_legal_hold — Number of blob containers under a legal hold + - containers_with_immutability_policy — Number of blob containers with a time-based retention (immutability) policy; null when any account's container list could not be read + - containers_with_legal_hold — Number of blob containers under a legal hold; null when any account's container list could not be read Which containers can be read anonymously? - - total_containers — Number of blob containers across all storage accounts whose containers were listed - - containers_with_public_access_level — Number of blob containers set to the Blob or Container public access level - - anonymously_accessible_containers — Number of blob containers that can be read without signing in through the account's public endpoint: the container has a public access level, its account does not forbid public blob access (Azure allows it when the setting is on or was never set), and the account's network admits public clients. The network does not when public network access is Disabled, or when it isn't SecuredByPerimeter and the firewall's default action is Deny with no IP (IPv4 or IPv6) or virtual network rule; a Deny whose IPv4 or virtual network rule list the API didn't return counts as admitting, and a missing IPv6 list as none. Not evaluated: which addresses a rule admits, network security perimeters, the trusted-services bypass and private endpoints - - accounts_with_containers_not_collected — Number of storage accounts whose container list failed; their containers are missing from the counts + - total_containers — Number of blob containers across all storage accounts whose containers were listed; null when any account's container list could not be read + - containers_with_public_access_level — Number of blob containers set to the Blob or Container public access level; null when any account's container list could not be read + - anonymously_accessible_containers — Number of blob containers that can be read without signing in through the account's public endpoint: the container has a public access level, its account does not forbid public blob access (Azure allows it when the setting is on or was never set), and the account's network admits public clients. The network does not when public network access is Disabled, or when it isn't SecuredByPerimeter and the firewall's default action is Deny with no IP (IPv4 or IPv6) or virtual network rule; a Deny whose IPv4 or virtual network rule list the API didn't return counts as admitting, and a missing IPv6 list as none. Not evaluated: which addresses a rule admits, network security perimeters, the trusted-services bypass and private endpoints; null as total_containers + - accounts_with_containers_not_collected — Number of storage accounts whose container list failed; their containers are missing from the counts, which are null (null here too when the storage account list could not be read) Is each account owned and classified? - accounts_with_owner_tag — Number of storage accounts with an owner tag (the first configured owner tag key present, such as Owner) diff --git a/tests/test_azure_unread_lists_give_null_counts_data.py b/tests/test_azure_unread_lists_give_null_counts_data.py new file mode 100644 index 000000000..03c57fd39 --- /dev/null +++ b/tests/test_azure_unread_lists_give_null_counts_data.py @@ -0,0 +1,265 @@ +"""Disk, Key Vault, storage and SQL Managed Instance fetchers: a list that could not be read gives null counts, never 0. + +A fake client refuses the list (or a sub-list a count is built from) and the test asserts that every total, count and +percentage built from it is null; the lists the run did read keep their numbers. +""" + +from __future__ import annotations + +import importlib.util +import json +import logging +import sys +from pathlib import Path +from types import SimpleNamespace as ns + +REPO_ROOT = Path(__file__).resolve().parent.parent +AZURE = REPO_ROOT / "fetchers" / "azure" +sys.path.insert(0, str(AZURE / "_shared")) + +from resource_exclusions import ENV # noqa: E402 + + +def _load(name): + spec = importlib.util.spec_from_file_location(f"azure_{name}_null_counts_under_test", AZURE / name / "fetcher.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +disks = _load("disk_encryption_status") +vaults = _load("key_vault_configuration") +storage = _load("storage_encryption_status") +mi_encryption = _load("sql_managed_instance_encryption") +mi_backups = _load("sql_managed_instance_backups") + +SUB = "/subscriptions/s1" +RG = f"{SUB}/resourceGroups/rg-a" +DENIED = PermissionError("(AuthorizationFailed) 403 no read access") +NOT_COUNTS = ("provider_registration_status", "resource_exclusion_patterns", "resource_exclusion_patterns_unmatched") + + +def _denied(*_args, **_kwargs): + raise DENIED + + +def _collector(): + return sys.modules["azure_common"].Collector(logging.getLogger("test")) + + +def _run(module, monkeypatch, tmp_path, filename): + monkeypatch.setattr(module, "load_dotenv", lambda: None) + monkeypatch.setattr(module, "resolve_subscription", lambda c: {"subscription_id": "s1", "subscription_source": "env"}) + monkeypatch.setattr(module, "credential", lambda: object()) + monkeypatch.setattr(module, "provider_registration_status", lambda *a: "registered") + monkeypatch.setenv("EVIDENCE_DIR", str(tmp_path)) + monkeypatch.delenv(ENV, raising=False) + code = module.main() + return code, json.loads((tmp_path / f"{filename}_s1.json").read_text()) + + +def _assert_all_null(summary, keep=()): + counts = {k: v for k, v in summary.items() if k not in NOT_COUNTS and k not in keep} + assert counts and all(v is None for v in counts.values()), {k: v for k, v in counts.items() if v is not None} + + +# --- azure_disk_encryption_status --- + +def test_disks_unread_list_gives_null_counts(monkeypatch, tmp_path, assert_summary_documented): + client = ns(disks=ns(list=_denied)) + monkeypatch.setitem(sys.modules, "azure.mgmt.compute", ns(ComputeManagementClient=lambda **_: client)) + code, evidence = _run(disks, monkeypatch, tmp_path, "azure_disk_encryption_status") + assert code == 1 + _assert_all_null(evidence["summary"]) + assert evidence["results"]["disks"] == [] + assert_summary_documented("azure/disk_encryption_status", evidence["summary"]) + + +def test_disks_read_empty_list_still_counts_zero(): + summary = disks.summarize([]) + assert summary["total_disks"] == 0 and summary["cmk_percentage"] == 0 + + +# --- azure_key_vault_configuration --- + +def _vault(name): + return ns(id=f"{RG}/providers/Microsoft.KeyVault/vaults/{name}", name=name, location="regionwest", + properties=ns(enable_rbac_authorization=True)) + + +def test_key_vaults_unread_list_gives_null_counts(monkeypatch, tmp_path, assert_summary_documented): + client = ns(vaults=ns(list_by_subscription=_denied)) + monkeypatch.setitem(sys.modules, "azure.mgmt.keyvault", ns(KeyVaultManagementClient=lambda **_: client)) + code, evidence = _run(vaults, monkeypatch, tmp_path, "azure_key_vault_configuration") + assert code == 1 + _assert_all_null(evidence["summary"]) + assert evidence["results"]["key_vaults"] == [] + assert_summary_documented("azure/key_vault_configuration", evidence["summary"]) + + +def test_key_vaults_unread_diagnostic_settings_null_the_counts_built_from_them(monkeypatch): + def settings(resource_uri): + if resource_uri.endswith("kv-b"): + raise DENIED + return [] + + client = ns(vaults=ns(list_by_subscription=lambda: [_vault("kv-a"), _vault("kv-b")])) + monitor = ns(diagnostic_settings=ns(list=settings)) + monkeypatch.setitem(sys.modules, "azure.mgmt.keyvault", ns(KeyVaultManagementClient=lambda **_: client)) + monkeypatch.setitem(sys.modules, "azure.mgmt.monitor", ns(MonitorManagementClient=lambda **_: monitor)) + found = vaults.collect_key_vaults("s1", object(), _collector()) + summary = vaults.summarize(found) + for key in vaults.AUDIT_COUNTS: + assert summary[key] is None, key + assert summary["vaults_audit_logging_unknown"] == 1 + assert summary["total_key_vaults"] == 2 and summary["rbac_authorization_vaults"] == 2 + + +# --- azure_storage_encryption_status --- + +def _account(name): + return ns(id=f"{RG}/providers/Microsoft.Storage/storageAccounts/{name}", name=name, location="regionwest", + sku=ns(name="Standard_RAGRS"), encryption=ns(key_source="Microsoft.Storage")) + + +def _storage_client(accounts, containers, get_properties=None): + return ns( + storage_accounts=ns(list=accounts, get_properties=get_properties or (lambda *a, **k: ns(geo_replication_stats=None))), + blob_services=ns(get_service_properties=lambda *a: ns()), + file_services=ns(get_service_properties=lambda *a: ns()), + blob_containers=ns(list=containers), + ) + + +def test_storage_unread_account_list_gives_null_counts(monkeypatch, tmp_path, assert_summary_documented): + client = _storage_client(_denied, lambda *a: []) + monkeypatch.setitem(sys.modules, "azure.mgmt.storage", ns(StorageManagementClient=lambda **_: client)) + code, evidence = _run(storage, monkeypatch, tmp_path, "azure_storage_encryption_status") + assert code == 1 + _assert_all_null(evidence["summary"]) + assert evidence["summary"]["in_scope_containers"] is None + assert_summary_documented("azure/storage_encryption_status", evidence["summary"]) + + +def test_storage_unread_container_list_nulls_container_counts(monkeypatch, tmp_path, assert_summary_documented): + def containers(group, name): + if name == "sa-b": + raise DENIED + return [ns(id=f"{RG}/c1", name="c1", public_access="Blob")] + + client = _storage_client(lambda: iter([_account("sa-a"), _account("sa-b")]), containers) + monkeypatch.setitem(sys.modules, "azure.mgmt.storage", ns(StorageManagementClient=lambda **_: client)) + code, evidence = _run(storage, monkeypatch, tmp_path, "azure_storage_encryption_status") + summary = evidence["summary"] + assert code == 1 + for key in (*storage.CONTAINER_COUNTS, "in_scope_containers", "excluded_containers"): + assert summary[key] is None, key + assert summary["accounts_with_containers_not_collected"] == 1 + assert summary["total_storage_accounts"] == 2 and summary["in_scope_storage_accounts"] == 2 + assert_summary_documented("azure/storage_encryption_status", summary) + + +def test_storage_unread_replication_status_nulls_the_counts_built_from_it(monkeypatch): + client = _storage_client(lambda: iter([_account("sa-a")]), lambda *a: [], _denied) + monkeypatch.setitem(sys.modules, "azure.mgmt.storage", ns(StorageManagementClient=lambda **_: client)) + found = storage.collect_storage_accounts("s1", object(), _collector()) + summary = storage.summarize(found) + assert summary["geo_redundant_accounts"] == 1 + assert all(summary[k] is None for k in storage.REPLICATION_COUNTS) + + +# --- azure_sql_managed_instance_encryption / _backups --- + +MI = f"{RG}/providers/Microsoft.Sql/managedInstances/mi-a" + + +def _instance(): + return ns(id=MI, name="mi-a", location="eastus", state="Ready", current_backup_storage_redundancy="Geo", + requested_backup_storage_redundancy="Geo") + + +def _mi_client(instances, databases, protector=None): + def call(value): + def fn(*args): + if isinstance(value, Exception): + raise value + return value + return fn + + return ns( + managed_instances=ns(list=instances), + managed_databases=ns(list_by_instance=call(databases)), + managed_instance_encryption_protectors=ns(get=call(protector or ns(server_key_type="ServiceManaged"))), + managed_database_transparent_data_encryption=ns(get=call(ns(state="Enabled"))), + managed_backup_short_term_retention_policies=ns(get=call(ns(retention_days=7))), + managed_instance_long_term_retention_policies=ns(get=call(ns(weekly_retention="P4W", monthly_retention="PT0S", + yearly_retention="PT0S", week_of_year=0))), + instance_failover_groups=ns(list_by_location=call([])), + ) + + +def _db(): + return ns(id=f"{MI}/databases/app", name="app", status="Online") + + +def test_mi_encryption_unread_instance_list_gives_null_counts(monkeypatch, tmp_path, assert_summary_documented): + monkeypatch.setattr(mi_encryption, "sql_client", lambda *a: _mi_client(_denied, [])) + code, evidence = _run(mi_encryption, monkeypatch, tmp_path, "azure_sql_managed_instance_encryption") + assert code == 1 + _assert_all_null(evidence["summary"]) + assert_summary_documented("azure/sql_managed_instance_encryption", evidence["summary"]) + + +def test_mi_encryption_unread_database_list_nulls_tde_counts(monkeypatch): + client = _mi_client(lambda: [_instance()], DENIED) + monkeypatch.setattr(mi_encryption, "sql_client", lambda *a: client) + summary = mi_encryption.summarize(mi_encryption.collect("s1", object(), _collector())) + for key in ("total_user_databases", "tde_enabled_user_databases", "tde_disabled_user_databases", + "tde_unknown_user_databases"): + assert summary[key] is None, key + assert summary["total_managed_instances"] == 1 + + +def test_mi_encryption_unread_protector_nulls_key_counts(monkeypatch): + client = _mi_client(lambda: [_instance()], [_db()], protector=DENIED) + monkeypatch.setattr(mi_encryption, "sql_client", lambda *a: client) + summary = mi_encryption.summarize(mi_encryption.collect("s1", object(), _collector())) + assert summary["instances_customer_managed_key"] is None and summary["instances_key_auto_rotation"] is None + assert summary["tde_enabled_user_databases"] == 1 + + +def test_mi_backups_unread_instance_list_gives_null_counts(monkeypatch, tmp_path, assert_summary_documented): + monkeypatch.setattr(mi_backups, "sql_client", lambda *a: _mi_client(_denied, [])) + monkeypatch.setattr(mi_backups, "read_failover_history", lambda *a: ([], {"days": 90, "complete": True})) + code, evidence = _run(mi_backups, monkeypatch, tmp_path, "azure_sql_managed_instance_backups") + summary = evidence["summary"] + assert code == 1 + failover_fields = ("failover_history_days", "failover_history_read", *mi_backups.summarize_failovers( + [], {"days": 90}, mi_backups.utc_now())) + _assert_all_null(summary, keep=(*failover_fields, "excluded_failovers", "in_scope_failovers")) + for key in ("total_managed_instances", "total_user_databases", "instances_by_current_backup_storage_redundancy", + "total_instance_failover_groups", "in_scope_managed_instances", "in_scope_instance_failover_groups"): + assert summary[key] is None, key + assert_summary_documented("azure/sql_managed_instance_backups", summary) + + +def test_mi_backups_unread_database_list_nulls_database_counts(monkeypatch): + client = _mi_client(lambda: [_instance()], DENIED) + monkeypatch.setattr(mi_backups, "sql_client", lambda *a: client) + found, groups, unlisted = mi_backups.collect("s1", object(), _collector()) + summary = mi_backups.summarize(found, groups, not unlisted) + for key in ("total_user_databases", "databases_retention_at_least_7_days", "databases_retention_below_7_days", + "databases_missing_retention_policy", "databases_retention_unknown", + "databases_with_long_term_retention", "databases_long_term_retention_unknown"): + assert summary[key] is None, key + assert summary["total_managed_instances"] == 1 + + +def test_mi_backups_unread_failover_group_list_nulls_membership_count(monkeypatch): + client = _mi_client(lambda: [_instance()], [_db()]) + client.instance_failover_groups = ns(list_by_location=_denied) + monkeypatch.setattr(mi_backups, "sql_client", lambda *a: client) + found, groups, unlisted = mi_backups.collect("s1", object(), _collector()) + summary = mi_backups.summarize(found, groups, not unlisted) + assert summary["instances_in_failover_group"] is None and summary["total_instance_failover_groups"] is None + assert summary["instances_failover_group_unknown"] == 1 From 4e7b217a1f36d6238c269793ec7a44fe03f14b6a Mon Sep 17 00:00:00 2001 From: Connor Dalton Date: Fri, 13 Nov 2026 15:11:16 -0700 Subject: [PATCH 2/2] Storage: unread service properties null the versioning and soft-delete counts; tests for empty lists Each account records whether its blob and file service properties were collected, not supported or failed; a failed read nulls the counts built from it. Key Vault total_access_policies is documented as null when the vault list is unread. Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 6 +- .../key_vault_configuration/fetcher.yaml | 2 +- .../storage_encryption_status/fetcher.py | 20 ++++-- .../storage_encryption_status/fetcher.yaml | 6 +- ...zure_unread_lists_give_null_counts_data.py | 67 ++++++++++++++++++- 5 files changed, 88 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a616dc8e1..21a91dd60 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2405,7 +2405,11 @@ schemas and the `paramify` CLI — not the internal code. container list is not read, the container counts and `in_scope_containers` and `excluded_containers` are null; when any geo-redundant account's replication status is not read, `geo_replication_live_accounts` and - `geo_failover_ready_accounts` are null. + `geo_failover_ready_accounts` are null. When any account's blob or file + service properties are not read, `blob_versioning_accounts`, + `container_soft_delete_accounts` (blob) and `share_soft_delete_accounts` + (file) are null; each account records `blob_properties_status` and + `file_service_properties_status` (collected, not_supported or failed). - `azure_sql_managed_instance_encryption` 0.4.0: when the instance list is not read, every count is null (`total_managed_instances` and `in_scope_managed_instances` included). The key counts are null when any diff --git a/fetchers/azure/key_vault_configuration/fetcher.yaml b/fetchers/azure/key_vault_configuration/fetcher.yaml index 6b19576fe..1a64a6a5e 100644 --- a/fetchers/azure/key_vault_configuration/fetcher.yaml +++ b/fetchers/azure/key_vault_configuration/fetcher.yaml @@ -93,7 +93,7 @@ evidence_set: - rbac_authorization_vaults — Number of vaults that use Azure RBAC role assignments for access to keys, secrets and certificates (the role assignments aren't in this evidence) - access_policy_vaults — Number of vaults that use the older vault access policies instead of Azure RBAC - rbac_authorization_percentage — Percentage of vaults that use Azure RBAC = rbac_authorization_vaults / total_key_vaults × 100 (a whole number, rounded down; reads 0 when total_key_vaults is 0, null when the key vault list could not be read) - - total_access_policies — Number of access policy entries across all vaults, each giving a user, group or app permissions on a vault's keys, secrets, certificates or storage accounts; 0 when the key vault list could not be read + - total_access_policies — Number of access policy entries across all vaults, each giving a user, group or app permissions on a vault's keys, secrets, certificates or storage accounts; null when the key vault list could not be read Can a deleted vault or secret be recovered? - soft_delete_vaults — Number of vaults with soft delete on, so deleted vaults, keys and secrets are kept for the retention period and can be restored; a vault that doesn't report the setting counts as off diff --git a/fetchers/azure/storage_encryption_status/fetcher.py b/fetchers/azure/storage_encryption_status/fetcher.py index d79997c93..4d8e261b3 100644 --- a/fetchers/azure/storage_encryption_status/fetcher.py +++ b/fetchers/azure/storage_encryption_status/fetcher.py @@ -429,7 +429,9 @@ def account_record( "allow_cross_tenant_replication": True if cross_tenant is None else bool(cross_tenant), # Filled in by the blob/file enrichment; None = not collected (no endpoint). "blob_properties": None, + "blob_properties_status": CONTAINERS_NOT_ATTEMPTED, "file_service_properties": None, + "file_service_properties_status": CONTAINERS_NOT_ATTEMPTED, # --- geo-replication (stats filled in by the get_properties enrichment) --- "geo_redundant": bool(GEO_REDUNDANT_SKU.search(sku)), "read_access_secondary": bool(READ_ACCESS_SKU.search(sku)), @@ -492,6 +494,10 @@ def file_service_properties_record(properties: dict) -> dict: "total_containers", "containers_with_public_access_level", "anonymously_accessible_containers", "containers_with_immutability_policy", "containers_with_legal_hold", ) +PROPERTY_COUNTS = { + "blob_properties_status": ("blob_versioning_accounts", "container_soft_delete_accounts"), + "file_service_properties_status": ("share_soft_delete_accounts",), +} REPLICATION_COUNTS = ("geo_replication_live_accounts", "geo_failover_ready_accounts") @@ -549,6 +555,9 @@ def _null_unread(accounts: list[dict], counts: dict) -> dict: """Null the counts built from a container list or replication status that failed for some account.""" if any(a.get("containers_status") == CONTAINERS_FAILED for a in accounts): counts.update({k: None for k in CONTAINER_COUNTS}) + for status, keys in PROPERTY_COUNTS.items(): + if any(a.get(status) == CONTAINERS_FAILED for a in accounts): + counts.update({k: None for k in keys}) if any((a.get("geo_replication_stats") or {}).get("stats_status") == STATS_FAILED for a in accounts): counts.update({k: None for k in REPLICATION_COUNTS}) return counts @@ -680,8 +689,9 @@ def _list(): "storage.blob_services.get_service_properties", RuntimeError(f"storage account {name!r} has no resource group in its id"), ) + account["blob_properties_status"] = account["file_service_properties_status"] = CONTAINERS_FAILED continue - account["blob_properties"] = _service_properties( + account["blob_properties"], account["blob_properties_status"] = _service_properties( collector, "storage.blob_services.get_service_properties", name, @@ -691,7 +701,7 @@ def _list(): ) ), ) - account["file_service_properties"] = _service_properties( + account["file_service_properties"], account["file_service_properties_status"] = _service_properties( collector, "storage.file_services.get_service_properties", name, @@ -762,15 +772,15 @@ def _service_properties(collector: Collector, operation: str, account_name: str, otherwise fail the whole run. Prowler skips it the same way. """ try: - return fn() + return fn(), CONTAINERS_COLLECTED except Exception as exc: # noqa: BLE001 — boundary: classify, don't crash the run if _is_benign_unsupported(exc): logger.warning( "%s: skipping %s — %s", operation, account_name, str(exc).strip().splitlines()[0] ) - return None + return None, CONTAINERS_NOT_SUPPORTED collector.record(f"{operation} ({account_name})", exc) - return None + return None, CONTAINERS_FAILED def main() -> int: diff --git a/fetchers/azure/storage_encryption_status/fetcher.yaml b/fetchers/azure/storage_encryption_status/fetcher.yaml index 6675b9177..8b215773b 100644 --- a/fetchers/azure/storage_encryption_status/fetcher.yaml +++ b/fetchers/azure/storage_encryption_status/fetcher.yaml @@ -145,9 +145,9 @@ evidence_set: - geo_failed_over_accounts — Number of storage accounts that have failed over to their second region at least once Can deleted or changed data be recovered? - - blob_versioning_accounts — Number of storage accounts with blob versioning on - - container_soft_delete_accounts — Number of storage accounts that keep deleted blob containers for a retention period (container soft delete) - - share_soft_delete_accounts — Number of storage accounts that keep deleted file shares for a retention period (share soft delete) + - blob_versioning_accounts — Number of storage accounts with blob versioning on; null when any account's blob service properties could not be read + - container_soft_delete_accounts — Number of storage accounts that keep deleted blob containers for a retention period (container soft delete); null when any account's blob service properties could not be read + - share_soft_delete_accounts — Number of storage accounts that keep deleted file shares for a retention period (share soft delete); null when any account's file service properties could not be read - containers_with_immutability_policy — Number of blob containers with a time-based retention (immutability) policy; null when any account's container list could not be read - containers_with_legal_hold — Number of blob containers under a legal hold; null when any account's container list could not be read diff --git a/tests/test_azure_unread_lists_give_null_counts_data.py b/tests/test_azure_unread_lists_give_null_counts_data.py index 03c57fd39..a9f6796fc 100644 --- a/tests/test_azure_unread_lists_give_null_counts_data.py +++ b/tests/test_azure_unread_lists_give_null_counts_data.py @@ -109,7 +109,8 @@ def settings(resource_uri): monkeypatch.setitem(sys.modules, "azure.mgmt.monitor", ns(MonitorManagementClient=lambda **_: monitor)) found = vaults.collect_key_vaults("s1", object(), _collector()) summary = vaults.summarize(found) - for key in vaults.AUDIT_COUNTS: + for key in ("vaults_with_diagnostic_settings", "vaults_with_audit_logging", "vaults_without_audit_logging", + "audit_logging_percentage"): assert summary[key] is None, key assert summary["vaults_audit_logging_unknown"] == 1 assert summary["total_key_vaults"] == 2 and summary["rbac_authorization_vaults"] == 2 @@ -152,7 +153,9 @@ def containers(group, name): code, evidence = _run(storage, monkeypatch, tmp_path, "azure_storage_encryption_status") summary = evidence["summary"] assert code == 1 - for key in (*storage.CONTAINER_COUNTS, "in_scope_containers", "excluded_containers"): + for key in ("total_containers", "containers_with_public_access_level", "anonymously_accessible_containers", + "containers_with_immutability_policy", "containers_with_legal_hold", "in_scope_containers", + "excluded_containers"): assert summary[key] is None, key assert summary["accounts_with_containers_not_collected"] == 1 assert summary["total_storage_accounts"] == 2 and summary["in_scope_storage_accounts"] == 2 @@ -165,7 +168,7 @@ def test_storage_unread_replication_status_nulls_the_counts_built_from_it(monkey found = storage.collect_storage_accounts("s1", object(), _collector()) summary = storage.summarize(found) assert summary["geo_redundant_accounts"] == 1 - assert all(summary[k] is None for k in storage.REPLICATION_COUNTS) + assert summary["geo_replication_live_accounts"] is None and summary["geo_failover_ready_accounts"] is None # --- azure_sql_managed_instance_encryption / _backups --- @@ -263,3 +266,61 @@ def test_mi_backups_unread_failover_group_list_nulls_membership_count(monkeypatc summary = mi_backups.summarize(found, groups, not unlisted) assert summary["instances_in_failover_group"] is None and summary["total_instance_failover_groups"] is None assert summary["instances_failover_group_unknown"] == 1 + + +def test_storage_unread_service_properties_null_versioning_and_soft_delete_counts(monkeypatch): + def blob_props(group, name): + if name == "sa-b": + raise DENIED + return ns() + + def file_props(group, name): + raise RuntimeError("File is not supported for the account.") + + client = _storage_client(lambda: iter([_account("sa-a"), _account("sa-b")]), lambda *a: []) + client.blob_services = ns(get_service_properties=blob_props) + client.file_services = ns(get_service_properties=file_props) + monkeypatch.setitem(sys.modules, "azure.mgmt.storage", ns(StorageManagementClient=lambda **_: client)) + found = storage.collect_storage_accounts("s1", object(), _collector()) + assert {a["name"]: a["blob_properties_status"] for a in found} == {"sa-a": "collected", "sa-b": "failed"} + assert {a["file_service_properties_status"] for a in found} == {"not_supported"} + summary = storage.summarize(found) + assert summary["blob_versioning_accounts"] is None and summary["container_soft_delete_accounts"] is None + assert summary["share_soft_delete_accounts"] == 0 # not_supported is expected, not a failure + + +# --- a list that was read and is empty still counts 0 --- + +def test_key_vaults_read_empty_list_counts_zero(monkeypatch, tmp_path): + client = ns(vaults=ns(list_by_subscription=lambda: [])) + monkeypatch.setitem(sys.modules, "azure.mgmt.keyvault", ns(KeyVaultManagementClient=lambda **_: client)) + code, evidence = _run(vaults, monkeypatch, tmp_path, "azure_key_vault_configuration") + summary = evidence["summary"] + assert code == 0 and summary["total_key_vaults"] == 0 and summary["in_scope_key_vaults"] == 0 + assert summary["total_access_policies"] == 0 + + +def test_storage_read_empty_list_counts_zero(monkeypatch, tmp_path): + client = _storage_client(lambda: iter([]), lambda *a: []) + monkeypatch.setitem(sys.modules, "azure.mgmt.storage", ns(StorageManagementClient=lambda **_: client)) + code, evidence = _run(storage, monkeypatch, tmp_path, "azure_storage_encryption_status") + summary = evidence["summary"] + assert code == 0 and summary["total_storage_accounts"] == 0 and summary["in_scope_storage_accounts"] == 0 + assert summary["total_containers"] == 0 and summary["in_scope_containers"] == 0 + + +def test_mi_encryption_read_empty_list_counts_zero(monkeypatch, tmp_path): + monkeypatch.setattr(mi_encryption, "sql_client", lambda *a: _mi_client(lambda: [], [])) + code, evidence = _run(mi_encryption, monkeypatch, tmp_path, "azure_sql_managed_instance_encryption") + summary = evidence["summary"] + assert code == 0 and summary["total_managed_instances"] == 0 and summary["in_scope_managed_instances"] == 0 + assert summary["tde_enabled_user_databases"] == 0 + + +def test_mi_backups_read_empty_list_counts_zero(monkeypatch, tmp_path): + monkeypatch.setattr(mi_backups, "sql_client", lambda *a: _mi_client(lambda: [], [])) + monkeypatch.setattr(mi_backups, "read_failover_history", lambda *a: ([], {"days": 90, "complete": True})) + code, evidence = _run(mi_backups, monkeypatch, tmp_path, "azure_sql_managed_instance_backups") + summary = evidence["summary"] + assert code == 0 and summary["total_managed_instances"] == 0 and summary["in_scope_managed_instances"] == 0 + assert summary["total_instance_failover_groups"] == 0 and summary["in_scope_instance_failover_groups"] == 0