From 39b8c43623be4bedd3247e568c2c72f6ffd5c43d Mon Sep 17 00:00:00 2001 From: Connor Dalton Date: Tue, 10 Nov 2026 10:45:21 -0700 Subject: [PATCH 1/6] fix(azure): policy compliance keeps unread remediation tasks unread, fails on unknown scopes, and nulls counts from failed reads Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 15 ++++ fetchers/azure/policy_compliance/fetcher.py | 53 +++++++----- fetchers/azure/policy_compliance/fetcher.yaml | 28 +++---- tests/test_azure_policy_compliance.py | 81 +++++++++++++++++++ 4 files changed, 144 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ad868cf..196b89e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2102,6 +2102,21 @@ schemas and the `paramify` CLI — not the internal code. ### Fixed +- **`azure_policy_compliance` 0.6.0: unread reads give null counts, and an + unread remediation task can no longer read as complete.** + - `deployments_complete` could read true on a run that missed a task's + deployment list: a later task's read replaced the record of unread tasks. + A task whose deployments could not be listed now keeps it false. + - A remediation task at a scope the fetcher doesn't recognise now fails the + run, where before it was skipped with only a log line. + - When the compliance summary could not be read, `non_compliant_resources`, + `non_compliant_policies`, `compliant_resources`, `resources_evaluated`, + `resource_compliance_percentage`, `assignments_evaluated`, + `non_compliant_assignments` and `compliant_assignments` read null, not 0. + When the remediation task list could not be read, `remediation_tasks`, + `remediation_tasks_by_provisioning_state` and the two deployment totals + read null, not 0. With the provider not registered they still read 0. + - **Azure resource exclusions: `resource_exclusion_patterns_unmatched` is null when a resource list was not read.** Before, a list that failed (or had no credential or subscription) left every configured pattern in the field and diff --git a/fetchers/azure/policy_compliance/fetcher.py b/fetchers/azure/policy_compliance/fetcher.py index 6dc0a600..26de50bd 100644 --- a/fetchers/azure/policy_compliance/fetcher.py +++ b/fetchers/azure/policy_compliance/fetcher.py @@ -647,8 +647,10 @@ def summarize( states: list[dict], states_truncated: bool, remediations: list[dict], + summary_read: bool = True, + tasks_read: bool = True, ) -> dict: - """Subscription-wide counts first (from the service), then what was itemized.""" + """Subscription-wide counts first (from the service), then what was itemized; null where the read failed.""" compliant, non_compliant = compliant_resource_counts( subscription_results.get("resource_details") ) @@ -658,19 +660,21 @@ def summarize( by_state[key] = by_state.get(key, 0) + 1 return { # --- the headline: the service's own subscription-wide counts --- - "non_compliant_resources": subscription_results.get("non_compliant_resources") or 0, - "non_compliant_policies": subscription_results.get("non_compliant_policies") or 0, - "compliant_resources": compliant, + "non_compliant_resources": (subscription_results.get("non_compliant_resources") or 0) if summary_read else None, + "non_compliant_policies": (subscription_results.get("non_compliant_policies") or 0) if summary_read else None, + "compliant_resources": compliant if summary_read else None, # Zero here means nothing was evaluated in the window, which is what tells # "0 non-compliant" (a finding) from "0 non-compliant, nothing assessed". - "resources_evaluated": sum((subscription_results.get("resource_details") or {}).values()), - "resource_compliance_percentage": coverage_percentage(compliant, compliant + non_compliant), + "resources_evaluated": sum((subscription_results.get("resource_details") or {}).values()) + if summary_read else None, + "resource_compliance_percentage": coverage_percentage(compliant, compliant + non_compliant) + if summary_read else None, "resources_by_compliance_state": subscription_results.get("resource_details") or {}, "policies_by_compliance_state": subscription_results.get("policy_details") or {}, # --- per assignment --- - "assignments_evaluated": len(assignments), - "non_compliant_assignments": sum(1 for a in assignments if not a["is_compliant"]), - "compliant_assignments": sum(1 for a in assignments if a["is_compliant"]), + "assignments_evaluated": len(assignments) if summary_read else None, + "non_compliant_assignments": sum(1 for a in assignments if not a["is_compliant"]) if summary_read else None, + "compliant_assignments": sum(1 for a in assignments if a["is_compliant"]) if summary_read else None, # --- the itemized list --- "listed_non_compliant_states": len(states), "listed_distinct_non_compliant_resources": len( @@ -678,12 +682,12 @@ def summarize( ), "non_compliant_states_truncated": states_truncated, # --- remediation --- - "remediation_tasks": len(remediations), - "remediation_tasks_by_provisioning_state": dict(sorted(by_state.items())), - "remediation_failed_deployments": sum(r.get("failed_deployments") or 0 for r in remediations), - "remediation_successful_deployments": sum( - r.get("successful_deployments") or 0 for r in remediations - ), + "remediation_tasks": len(remediations) if tasks_read else None, + "remediation_tasks_by_provisioning_state": dict(sorted(by_state.items())) if tasks_read else None, + "remediation_failed_deployments": sum(r.get("failed_deployments") or 0 for r in remediations) + if tasks_read else None, + "remediation_successful_deployments": sum(r.get("successful_deployments") or 0 for r in remediations) + if tasks_read else None, } @@ -888,6 +892,8 @@ def config_limit(env: str, default: int) -> int: def empty_collection() -> dict: return { "subscription_results": project_summary_results(None), + "summary_read": False, + "tasks_read": False, "assignments": [], "states": [], "states_truncated": False, @@ -1123,7 +1129,9 @@ def _current(policy: tuple, resource_ids: list[str]) -> dict: kind, scope = remediation_scope(task.get("id")) name = task.get("name") if kind not in listers: - logger.warning("remediation %s has an unrecognised scope; deployments not read", task.get("id")) + collector.record( + f"policyinsights.remediations.list_deployments (scope of {task.get('id')})", + ValueError("remediation scope not recognised; its deployments were not read")) unread.append(task.get("id")) continue @@ -1154,9 +1162,9 @@ def _list(kind=kind, scope=scope, name=name, remaining=remaining, task_id=task.g arm_id_key(task.get("policy_definition_reference_id"))) read = read_by_policy.setdefault(policy, set()) current = current_by_policy.setdefault(policy, {}) - unread = sorted({arm_id_key(d.get("remediated_resource_id")) for d in rows} - read - {""}) - for at in range(0, len(unread), RESOURCE_FILTER_BATCH): - batch = unread[at:at + RESOURCE_FILTER_BATCH] + to_read = sorted({arm_id_key(d.get("remediated_resource_id")) for d in rows} - read - {""}) + for at in range(0, len(to_read), RESOURCE_FILTER_BATCH): + batch = to_read[at:at + RESOURCE_FILTER_BATCH] read.update(batch) current.update(collector.guard( f"policyinsights.policy_states.list_query_results_for_subscription (remediated {name})", @@ -1203,6 +1211,7 @@ def _summarize(): ) summarized = collector.guard("policyinsights.policy_states.summarize_for_subscription", _summarize) + collected["summary_read"] = summarized is not None subscription_results, assignments = summarized or (collected["subscription_results"], []) # Only assignments made inside an excluded group or resource go; the service's counts can't be split by resource. assignments, excluded = exclusions.split( @@ -1240,6 +1249,7 @@ def _states(): "policyinsights.remediations.list_for_subscription", lambda: [project_remediation(r) for r in client.remediations.list_for_subscription()], ) + collected["tasks_read"] = listed_tasks is not None remediations, excluded_tasks = exclusions.split(listed_tasks or [], "remediation_task") # The service's counts can't be split by resource, so with patterns they are recounted from every latest pair. latest_pairs, latest_status = (collect_latest_pairs(client, subscription_id, collector, exclusions) @@ -1271,6 +1281,8 @@ def _states(): ) return { "subscription_results": subscription_results, + "summary_read": collected["summary_read"], + "tasks_read": collected["tasks_read"], "assignments": sorted(assignments, key=lambda a: a.get("policy_assignment_id") or ""), "states": sorted( states, @@ -1361,6 +1373,9 @@ def main() -> int: collected["states"], collected["states_truncated"], collected["remediations"], + # Nothing is read when the provider is unregistered, which is "not in use": 0, not unknown. + summary_read=collected["summary_read"] or registration == NOT_REGISTERED, + tasks_read=collected["tasks_read"] or registration == NOT_REGISTERED, ), **summarize_evaluation_details(collected["states"], collected["details"]), } diff --git a/fetchers/azure/policy_compliance/fetcher.yaml b/fetchers/azure/policy_compliance/fetcher.yaml index afb5ddc4..9f8fce90 100644 --- a/fetchers/azure/policy_compliance/fetcher.yaml +++ b/fetchers/azure/policy_compliance/fetcher.yaml @@ -1,5 +1,5 @@ name: azure_policy_compliance -version: 0.5.1 +version: 0.6.0 description: >- Shows what Azure Policy found non-compliant on the subscription and why, whether remediation tasks fixed it, and which non-compliance keeps coming back. category: azure @@ -148,7 +148,7 @@ evidence_set: - policy_states.list_query_results_for_subscription("default") grouped per resource, region, policy and state, and a count of the groups — 30 days of evaluation history, checked to be complete - policy_states.list_query_results_for_subscription("default") for one pair, ordered by time (per in-scope part-time non-compliant pair, up to a limit) — counts its separate non-compliant episodes - remediations.list_for_subscription() — remediation tasks and their deployment counts - - remediations.list_deployments_at_subscription(), _at_resource_group(), _at_management_group() or _at_resource() (per in-scope task) — each resource a task remediated + - remediations.list_deployments_at_subscription(), _at_resource_group(), _at_management_group() or _at_resource() (per in-scope task) — each resource a task remediated; a task whose deployments cannot be listed, or made at a scope this fetcher doesn't recognise, fails the run - policy_states.list_query_results_for_subscription("latest") filtered to the task's policy and in-scope remediated resources (per task, 20 resources at a time) — whether each remediated resource is compliant now, and when that was evaluated - policy_states.list_query_results_for_subscription("latest") grouped per resource, region, policy and state, and a count of the groups (only with resource_exclusions patterns set) — every resource's and policy's latest state, to recount the subscription's and each assignment's compliance over in-scope resources @@ -169,7 +169,7 @@ evidence_set: - in_scope_remediation_deployments — Number of remediation deployments read and in scope What was evaluated? - - resources_evaluated — Number of resources with a compliance state in the service's evaluation window (the last day); 0 means nothing was evaluated, not that everything passed; also 0 when the summary read failed (the run fails) or Microsoft.PolicyInsights isn't registered. With a resource_exclusions pattern set, only in-scope resources count ("in scope": no pattern matches the resource by id, resource group or region), recounted from every latest (resource, policy) state; null unless that grouped read came back whole (see in_scope_states_status) + - resources_evaluated — Number of resources with a compliance state in the service's evaluation window (the last day); 0 means nothing was evaluated, not that everything passed; null when the summary read failed (the run fails); 0 when Microsoft.PolicyInsights isn't registered. With a resource_exclusions pattern set, only in-scope resources count ("in scope": no pattern matches the resource by id, resource group or region), recounted from every latest (resource, policy) state; null unless that grouped read came back whole (see in_scope_states_status) - in_scope_states_status — Whether the grouped read of every latest (resource, policy) state behind the in-scope counts came back whole, with a resource_exclusions pattern set; null when no pattern is set, or when the compliance reads didn't run (Microsoft.PolicyInsights not registered, no subscription or credential, or the Policy Insights client couldn't be created): - read (every group came back, so the in-scope counts stand; an empty count with no groups means nothing was evaluated) - truncated (fewer groups came back than the service counted, or it counted more than 50,000) @@ -177,16 +177,16 @@ evidence_set: - failed (the read or its count failed, and the run exits non-zero) - resources_by_compliance_state — Number of resources in each compliance state (for example compliant, noncompliant, exempt, unknown), each resource counted once under its highest-ranked state in the order noncompliant, compliant, error, conflict, protected, exempt, unknown, as the service counts them. With a resource_exclusions pattern set, counted over in-scope resources, keeping every state the service reported (0 when none is in scope); every value is null when resources_evaluated is null - policies_by_compliance_state — Number of policy assignments in each compliance state (an initiative counts once), each under its highest-ranked state across the resources it evaluated, as the service counts them. With a resource_exclusions pattern set, counted over in-scope resources only; every value is null when resources_evaluated is null - - assignments_evaluated — Number of in-scope policy assignments the service summarized + - assignments_evaluated — Number of in-scope policy assignments the service summarized; null when the summary could not be read - provider_registration_status — Whether the policy compliance provider (Microsoft.PolicyInsights) is registered on the subscription: registered, not_registered or unknown; not_registered is valid evidence, not a failure: the run succeeds, nothing else is read, and the counts read 0 (null with a resource_exclusions pattern set) How compliant is the subscription? - - non_compliant_resources — Number of resources non-compliant with at least one policy, the noncompliant entry of resources_by_compliance_state. With no resource_exclusions pattern set, the service's own count; with one set, the in-scope resources with a non-compliant latest state, null when resources_evaluated is null - - non_compliant_policies — Number of policy assignments with at least one non-compliant resource (an initiative counts once), as the service counts them at subscription scope. With a resource_exclusions pattern set, counted over in-scope resources only, null when resources_evaluated is null - - compliant_resources — Number of resources compliant with every policy that applies to them, the compliant entry of resources_by_compliance_state. With a resource_exclusions pattern set, counted over in-scope resources only, null when resources_evaluated is null - - resource_compliance_percentage — Percentage of evaluated resources that are compliant = compliant_resources / (compliant_resources + non_compliant_resources) × 100 (a whole number, 0–100); 0 when that sum is 0. Resources in other states (exempt, unknown, conflict, error, protected) are left out of both. With a resource_exclusions pattern set, over in-scope resources only, null when resources_evaluated is null - - non_compliant_assignments — Number of in-scope policy assignments with at least one non-compliant resource. With no resource_exclusions pattern set, by the service's per-assignment count (0, not null, when that read failed); with one set, by each assignment's in-scope latest states, null when resources_evaluated is null - - compliant_assignments — Number of in-scope policy assignments with no non-compliant resource. With no resource_exclusions pattern set, by the service's per-assignment count (0, not null, when that read failed); with one set, by each assignment's in-scope latest states, null when resources_evaluated is null + - non_compliant_resources — Number of resources non-compliant with at least one policy, the noncompliant entry of resources_by_compliance_state. With no resource_exclusions pattern set, the service's own count; with one set, the in-scope resources with a non-compliant latest state, null when resources_evaluated is null; null when the compliance summary could not be read (the run fails) + - non_compliant_policies — Number of policy assignments with at least one non-compliant resource (an initiative counts once), as the service counts them at subscription scope. With a resource_exclusions pattern set, counted over in-scope resources only, null when resources_evaluated is null; null when the compliance summary could not be read (the run fails) + - compliant_resources — Number of resources compliant with every policy that applies to them, the compliant entry of resources_by_compliance_state. With a resource_exclusions pattern set, counted over in-scope resources only, null when resources_evaluated is null; null when the compliance summary could not be read (the run fails) + - resource_compliance_percentage — Percentage of evaluated resources that are compliant = compliant_resources / (compliant_resources + non_compliant_resources) × 100 (a whole number, 0–100); 0 when that sum is 0. Resources in other states (exempt, unknown, conflict, error, protected) are left out of both. With a resource_exclusions pattern set, over in-scope resources only, null when resources_evaluated is null; null when the compliance summary could not be read (the run fails) + - non_compliant_assignments — Number of in-scope policy assignments with at least one non-compliant resource. With no resource_exclusions pattern set, by the service's per-assignment count (null when the summary could not be read); with one set, by each assignment's in-scope latest states, null when resources_evaluated is null + - compliant_assignments — Number of in-scope policy assignments with no non-compliant resource. With no resource_exclusions pattern set, by the service's per-assignment count (null when the summary could not be read); with one set, by each assignment's in-scope latest states, null when resources_evaluated is null Which resources are non-compliant? - listed_non_compliant_states — Number of in-scope non-compliant (resource, policy) records listed in the evidence (at most the max_non_compliant_resources setting, default 1000; records a resource_exclusions pattern takes out don't count toward it) @@ -209,10 +209,10 @@ evidence_set: - failed (the read for the resource failed) Do remediation tasks fix what they target? - - remediation_tasks — Number of in-scope remediation tasks on the subscription; 0 when the task list could not be read - - remediation_tasks_by_provisioning_state — Number of in-scope remediation tasks in each state (for example Succeeded, Failed, Evaluating); a task's state covers every resource it remediated, including resources a resource_exclusions pattern takes out - - remediation_successful_deployments — Number of remediation deployments that succeeded. With no resource_exclusions pattern set, as the tasks report them; with one set, the listed in-scope deployments whose status is Succeeded, null when remediation_deployments_truncated is true, max_remediation_deployments is 0 (the deployments are skipped), or the task list or any task's deployment list failed - - remediation_failed_deployments — Number of remediation deployments that failed. With no resource_exclusions pattern set, as the tasks report them; with one set, the listed in-scope deployments whose status is Failed, null when remediation_deployments_truncated is true, max_remediation_deployments is 0 (the deployments are skipped), or the task list or any task's deployment list failed + - remediation_tasks — Number of in-scope remediation tasks on the subscription; null when the task list could not be read + - remediation_tasks_by_provisioning_state — Number of in-scope remediation tasks in each state (for example Succeeded, Failed, Evaluating); a task's state covers every resource it remediated, including resources a resource_exclusions pattern takes out; null when the task list could not be read + - remediation_successful_deployments — Number of remediation deployments that succeeded. With no resource_exclusions pattern set, as the tasks report them; with one set, the listed in-scope deployments whose status is Succeeded, null when remediation_deployments_truncated is true, max_remediation_deployments is 0 (the deployments are skipped), or the task list or any task's deployment list failed. With no pattern set, null only when the task list could not be read + - remediation_failed_deployments — Number of remediation deployments that failed. With no resource_exclusions pattern set, as the tasks report them; with one set, the listed in-scope deployments whose status is Failed, null when remediation_deployments_truncated is true, max_remediation_deployments is 0 (the deployments are skipped), or the task list or any task's deployment list failed. With no pattern set, null only when the task list could not be read - remediation_deployments_listed — Number of in-scope per-resource remediation deployments listed in the evidence (at most the max_remediation_deployments setting, default 1000; deployments a resource_exclusions pattern takes out don't count toward it) - remediation_deployments_truncated — True if some in-scope deployments were not listed because of the max_remediation_deployments limit; false if every in-scope one is listed, or the limit is 0 and the deployments are skipped - remediation_deployments_by_status — Number of listed remediation deployments in each status (for example Succeeded, Failed) diff --git a/tests/test_azure_policy_compliance.py b/tests/test_azure_policy_compliance.py index 301dc9c4..2bdc1a7c 100644 --- a/tests/test_azure_policy_compliance.py +++ b/tests/test_azure_policy_compliance.py @@ -862,3 +862,84 @@ def collect(subscription_id, cred, collector, limits, exclusions=None): assert fx.main() == 0 summary = json.loads((tmp_path / "azure_policy_compliance_s1.json").read_text())["summary"] assert_summary_documented("azure/policy_compliance", summary) + + +def _deployment(rid): + return NS(remediated_resource_id=rid, deployment_id=None, status="Succeeded", resource_location=None, + error=None, created_on=None, last_updated_on=None) + + +def _task(name, scope=SUB, ref="r1"): + return {"id": f"{scope}/providers/Microsoft.PolicyInsights/remediations/{name}", "name": name, + "policy_assignment_id": ASSIGN, "policy_definition_reference_id": ref, "total_deployments": 1} + + +@needs_sdk +def test_a_task_whose_deployments_could_not_be_listed_stays_unread_after_later_tasks_are_read(): + def at_subscription(name): + if name == "t2": + raise RuntimeError("throttled") + return [_deployment(VM)] + + client = fake_client( + policy_states={"list_query_results_for_subscription": lambda *a, **k: []}, + remediations={"list_deployments_at_subscription": at_subscription}, + ) + unread: list = [] + collector = fx.Collector(logging.getLogger("t")) + rows, _, _ = fx.collect_remediation_deployments( + client, "s1", [_task("t1"), _task("t2")], 10, collector, unread=unread) + assert len(rows) == 1 + assert unread == [_task("t2")["id"]] + assert not collector.ok + + +@needs_sdk +def test_unrecognised_remediation_scope_fails_the_run(): + client = fake_client(policy_states={}, remediations={}) + odd = {**_task("t1"), "id": "/providers/Odd/remediations/t1"} + unread: list = [] + collector = fx.Collector(logging.getLogger("t")) + fx.collect_remediation_deployments(client, "s1", [odd], 10, collector, unread=unread) + assert unread == [odd["id"]] + assert not collector.ok + + +def _run_main(monkeypatch, tmp_path, collected, registration="registered"): + monkeypatch.setattr(fx, "load_dotenv", lambda: None) + monkeypatch.setenv("EVIDENCE_DIR", str(tmp_path)) + monkeypatch.setattr(fx, "resolve_subscription", lambda c: {"subscription_id": "s1", "subscription_source": "env"}) + monkeypatch.setattr(fx, "credential", lambda: object()) + monkeypatch.setattr(fx, "provider_registration_status", lambda *a: registration) + monkeypatch.setattr(fx, "collect_compliance", lambda *a, **k: collected) + assert fx.main() == 0 + return json.loads((tmp_path / "azure_policy_compliance_s1.json").read_text())["summary"] + + +SERVICE_COUNTS = ("non_compliant_resources", "non_compliant_policies", "compliant_resources", "resources_evaluated", + "resource_compliance_percentage", "assignments_evaluated", "non_compliant_assignments", + "compliant_assignments") +TASK_COUNTS = ("remediation_tasks", "remediation_tasks_by_provisioning_state", "remediation_failed_deployments", + "remediation_successful_deployments") + + +def test_counts_are_null_when_the_summary_and_task_reads_failed(monkeypatch, tmp_path): + summary = _run_main(monkeypatch, tmp_path, fx.empty_collection()) + assert all(summary[k] is None for k in SERVICE_COUNTS + TASK_COUNTS) + + +def test_counts_read_zero_when_both_reads_came_back_empty(monkeypatch, tmp_path): + collected = {**fx.empty_collection(), "summary_read": True, "tasks_read": True} + summary = _run_main(monkeypatch, tmp_path, collected) + assert all(summary[k] == 0 for k in SERVICE_COUNTS + ("remediation_tasks", "remediation_failed_deployments")) + assert summary["remediation_tasks_by_provisioning_state"] == {} + + +def test_each_read_nulls_only_its_own_counts(monkeypatch, tmp_path): + summary = _run_main(monkeypatch, tmp_path, {**fx.empty_collection(), "summary_read": True}) + assert summary["non_compliant_resources"] == 0 and summary["remediation_tasks"] is None + + +def test_unregistered_provider_still_reads_zero(monkeypatch, tmp_path): + summary = _run_main(monkeypatch, tmp_path, fx.empty_collection(), registration=fx.NOT_REGISTERED) + assert summary["non_compliant_resources"] == 0 and summary["remediation_tasks"] == 0 From 6d4a2fd28dd6accc85558b7a0d0c7a484a49cfc2 Mon Sep 17 00:00:00 2001 From: Connor Dalton Date: Tue, 10 Nov 2026 10:46:30 -0700 Subject: [PATCH 2/6] fix(azure): policy compliance listed state counts read null when the states query fails Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 4 ++- fetchers/azure/policy_compliance/fetcher.py | 12 +++++++-- fetchers/azure/policy_compliance/fetcher.yaml | 4 +-- tests/test_azure_policy_compliance.py | 26 +++++++++++++++++++ 4 files changed, 41 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 196b89e1..198db5b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2115,7 +2115,9 @@ schemas and the `paramify` CLI — not the internal code. `non_compliant_assignments` and `compliant_assignments` read null, not 0. When the remediation task list could not be read, `remediation_tasks`, `remediation_tasks_by_provisioning_state` and the two deployment totals - read null, not 0. With the provider not registered they still read 0. + read null, not 0. `listed_non_compliant_states` and + `listed_distinct_non_compliant_resources` read null when the non-compliant + states query failed. With the provider not registered they still read 0. - **Azure resource exclusions: `resource_exclusion_patterns_unmatched` is null when a resource list was not read.** Before, a list that failed (or had no diff --git a/fetchers/azure/policy_compliance/fetcher.py b/fetchers/azure/policy_compliance/fetcher.py index 26de50bd..c1b4b176 100644 --- a/fetchers/azure/policy_compliance/fetcher.py +++ b/fetchers/azure/policy_compliance/fetcher.py @@ -649,6 +649,7 @@ def summarize( remediations: list[dict], summary_read: bool = True, tasks_read: bool = True, + states_read: bool = True, ) -> dict: """Subscription-wide counts first (from the service), then what was itemized; null where the read failed.""" compliant, non_compliant = compliant_resource_counts( @@ -676,10 +677,10 @@ def summarize( "non_compliant_assignments": sum(1 for a in assignments if not a["is_compliant"]) if summary_read else None, "compliant_assignments": sum(1 for a in assignments if a["is_compliant"]) if summary_read else None, # --- the itemized list --- - "listed_non_compliant_states": len(states), + "listed_non_compliant_states": len(states) if states_read else None, "listed_distinct_non_compliant_resources": len( {str(s.get("resource_id") or "").lower() for s in states if s.get("resource_id")} - ), + ) if states_read else None, "non_compliant_states_truncated": states_truncated, # --- remediation --- "remediation_tasks": len(remediations) if tasks_read else None, @@ -894,6 +895,7 @@ def empty_collection() -> dict: "subscription_results": project_summary_results(None), "summary_read": False, "tasks_read": False, + "states_read": False, "assignments": [], "states": [], "states_truncated": False, @@ -1243,7 +1245,11 @@ def _states(): break states.append(record) + failed_before = len(collector.failures) collector.guard("policyinsights.policy_states.list_query_results_for_subscription", _states) + collected["states_read"] = len(collector.failures) == failed_before + else: + collected["states_read"] = True listed_tasks = collector.guard( "policyinsights.remediations.list_for_subscription", @@ -1283,6 +1289,7 @@ def _states(): "subscription_results": subscription_results, "summary_read": collected["summary_read"], "tasks_read": collected["tasks_read"], + "states_read": collected["states_read"], "assignments": sorted(assignments, key=lambda a: a.get("policy_assignment_id") or ""), "states": sorted( states, @@ -1376,6 +1383,7 @@ def main() -> int: # Nothing is read when the provider is unregistered, which is "not in use": 0, not unknown. summary_read=collected["summary_read"] or registration == NOT_REGISTERED, tasks_read=collected["tasks_read"] or registration == NOT_REGISTERED, + states_read=collected["states_read"] or registration == NOT_REGISTERED, ), **summarize_evaluation_details(collected["states"], collected["details"]), } diff --git a/fetchers/azure/policy_compliance/fetcher.yaml b/fetchers/azure/policy_compliance/fetcher.yaml index 9f8fce90..724eb16f 100644 --- a/fetchers/azure/policy_compliance/fetcher.yaml +++ b/fetchers/azure/policy_compliance/fetcher.yaml @@ -189,8 +189,8 @@ evidence_set: - compliant_assignments — Number of in-scope policy assignments with no non-compliant resource. With no resource_exclusions pattern set, by the service's per-assignment count (null when the summary could not be read); with one set, by each assignment's in-scope latest states, null when resources_evaluated is null Which resources are non-compliant? - - listed_non_compliant_states — Number of in-scope non-compliant (resource, policy) records listed in the evidence (at most the max_non_compliant_resources setting, default 1000; records a resource_exclusions pattern takes out don't count toward it) - - listed_distinct_non_compliant_resources — Number of distinct resources among the listed non-compliant records + - listed_non_compliant_states — Number of in-scope non-compliant (resource, policy) records listed in the evidence (at most the max_non_compliant_resources setting, default 1000; records a resource_exclusions pattern takes out don't count toward it); null when the non-compliant states query failed (the run fails) + - listed_distinct_non_compliant_resources — Number of distinct resources among the listed non-compliant records; null when the non-compliant states query failed (the run fails) - non_compliant_states_truncated — True if there were more in-scope non-compliant records than the list holds; false if every one is listed, or max_non_compliant_resources is 0 and the list is skipped Why did they fail? diff --git a/tests/test_azure_policy_compliance.py b/tests/test_azure_policy_compliance.py index 2bdc1a7c..d577802d 100644 --- a/tests/test_azure_policy_compliance.py +++ b/tests/test_azure_policy_compliance.py @@ -928,6 +928,32 @@ def test_counts_are_null_when_the_summary_and_task_reads_failed(monkeypatch, tmp assert all(summary[k] is None for k in SERVICE_COUNTS + TASK_COUNTS) +@needs_sdk +def test_listed_state_counts_are_null_when_the_states_query_fails(monkeypatch, tmp_path): + def boom(*a, **k): + raise RuntimeError("throttled") + + client = fake_client(policy_states={"list_query_results_for_subscription": boom}) + monkeypatch.setattr(fx, "collect_evaluation_details", lambda *a, **k: fx.empty_collection()["details"]) + monkeypatch.setattr(fx, "collect_recurrence", lambda *a, **k: fx.empty_collection()["recurrence"]) + monkeypatch.setattr(fx, "collect_remediation_deployments", lambda *a, **k: ([], False, [])) + import azure.mgmt.policyinsights as pi + monkeypatch.setattr(pi, "PolicyInsightsClient", lambda **k: client) + collector = fx.Collector(logging.getLogger("t")) + limits = {"non_compliant": 10, "detail_resources": 1, "recurrence_pairs": 1, "remediation_deployments": 1} + collected = fx.collect_compliance("s1", object(), collector, limits) + assert collected["states_read"] is False and not collector.ok + summary = _run_main(monkeypatch, tmp_path, collected) + assert summary["listed_non_compliant_states"] is None + assert summary["listed_distinct_non_compliant_resources"] is None + + +def test_listed_state_counts_read_zero_when_the_list_was_read_and_empty(monkeypatch, tmp_path): + summary = _run_main(monkeypatch, tmp_path, {**fx.empty_collection(), "states_read": True}) + assert summary["listed_non_compliant_states"] == 0 + assert summary["listed_distinct_non_compliant_resources"] == 0 + + def test_counts_read_zero_when_both_reads_came_back_empty(monkeypatch, tmp_path): collected = {**fx.empty_collection(), "summary_read": True, "tasks_read": True} summary = _run_main(monkeypatch, tmp_path, collected) From 938f16f8e989989bd7162bd499ee5fad81ca3994 Mon Sep 17 00:00:00 2001 From: Connor Dalton Date: Tue, 10 Nov 2026 10:51:29 -0700 Subject: [PATCH 3/6] fix(azure): policy compliance nulls assignment counts under patterns on an unread summary, and deployment counts on an incomplete list Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 7 +++++ fetchers/azure/policy_compliance/fetcher.py | 28 +++++++++++++------ fetchers/azure/policy_compliance/fetcher.yaml | 20 ++++++------- tests/test_azure_policy_compliance.py | 28 +++++++++++++++++++ 4 files changed, 65 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 198db5b7..875c2916 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2118,6 +2118,13 @@ schemas and the `paramify` CLI — not the internal code. read null, not 0. `listed_non_compliant_states` and `listed_distinct_non_compliant_resources` read null when the non-compliant states query failed. With the provider not registered they still read 0. + - With a `resource_exclusions` pattern set and the summary unread, + `non_compliant_assignments` and `compliant_assignments` read null, not 0. + - When the task list or any task's deployment list was not read, + `remediation_deployments_listed`, `remediation_deployments_by_status` and + the `remediated_resources_*` counts (and + `succeeded_remediations_now_non_compliant`) read null, not counts over the + partial list. - **Azure resource exclusions: `resource_exclusion_patterns_unmatched` is null when a resource list was not read.** Before, a list that failed (or had no diff --git a/fetchers/azure/policy_compliance/fetcher.py b/fetchers/azure/policy_compliance/fetcher.py index c1b4b176..1c853ee1 100644 --- a/fetchers/azure/policy_compliance/fetcher.py +++ b/fetchers/azure/policy_compliance/fetcher.py @@ -785,7 +785,8 @@ def in_scope_compliance(results: dict, assignments: list[dict], pairs: list[dict ) -def in_scope_summary(results: dict, assignments: list[dict], deployments: list[dict] | None) -> dict: +def in_scope_summary(results: dict, assignments: list[dict], deployments: list[dict] | None, + summary_read: bool = True) -> dict: """The summary fields recounted over in-scope records; deployments is None when that list is incomplete.""" known = results.get("non_compliant_resources") is not None details = results.get("resource_details") or {} @@ -799,8 +800,11 @@ def in_scope_summary(results: dict, assignments: list[dict], deployments: list[d "resource_compliance_percentage": coverage_percentage(compliant, compliant + non_compliant) if known else None, "resources_by_compliance_state": details, "policies_by_compliance_state": results.get("policy_details") or {}, - "non_compliant_assignments": sum(1 for a in assignments if a["is_compliant"] is False) if known else None, - "compliant_assignments": sum(1 for a in assignments if a["is_compliant"]) if known else None, + # The assignment list comes from the summary, so an unread summary leaves it empty, not zero. + "non_compliant_assignments": ( + sum(1 for a in assignments if a["is_compliant"] is False) if known and summary_read else None), + "compliant_assignments": ( + sum(1 for a in assignments if a["is_compliant"]) if known and summary_read else None), "remediation_failed_deployments": statuses["failed"] if deployments is not None else None, "remediation_successful_deployments": statuses["succeeded"] if deployments is not None else None, } @@ -845,10 +849,11 @@ def summarize_recurrence( } -def summarize_remediation_deployments(deployments: list[dict], truncated: bool) -> dict: - """Deployments by status; the remediated_* counts are per resource, its latest deployment deciding.""" +def summarize_remediation_deployments(deployments: list[dict], truncated: bool, complete: bool = True) -> dict: + """Deployments by status; the remediated_* counts are per resource, its latest deployment deciding. + Null where the task list or a task's deployment list was not read, so a partial list can't read as 0.""" resources = latest_deployment_per_resource(deployments) - return { + summary = { "remediation_deployments_listed": len(deployments), "remediation_deployments_truncated": truncated, "remediation_deployments_by_status": sorted_counts( @@ -873,6 +878,11 @@ def summarize_remediation_deployments(deployments: list[dict], truncated: bool) if d["now_compliant"] is None and d.get("reevaluated_since_deployment") is not False ), } + if not complete: + for key in summary: + if key != "remediation_deployments_truncated": + summary[key] = None + return summary def config_limit(env: str, default: int) -> int: @@ -1395,7 +1405,8 @@ def main() -> int: deployments_known = (limits["remediation_deployments"] > 0 and collected["deployments_complete"] and not collected["deployments_truncated"]) summary.update(in_scope_summary(subscription_results, assignments, - collected["deployments"] if deployments_known else None)) + collected["deployments"] if deployments_known else None, + summary_read=collected["summary_read"])) summary["in_scope_states_status"] = collected["latest_pairs_status"] recurrence = collected["recurrence"] evidence = build_payload( @@ -1426,7 +1437,8 @@ def main() -> int: recurrence["pairs_truncated"], recurrence["history_collected"], ), **summarize_remediation_deployments( - collected["deployments"], collected["deployments_truncated"] + collected["deployments"], collected["deployments_truncated"], + complete=collected["deployments_complete"] or registration == NOT_REGISTERED, ), "provider_registration_status": registration, **exclusions.summary(collected["excluded"], EXCLUDED_KINDS, { diff --git a/fetchers/azure/policy_compliance/fetcher.yaml b/fetchers/azure/policy_compliance/fetcher.yaml index 724eb16f..24da95c1 100644 --- a/fetchers/azure/policy_compliance/fetcher.yaml +++ b/fetchers/azure/policy_compliance/fetcher.yaml @@ -185,8 +185,8 @@ evidence_set: - non_compliant_policies — Number of policy assignments with at least one non-compliant resource (an initiative counts once), as the service counts them at subscription scope. With a resource_exclusions pattern set, counted over in-scope resources only, null when resources_evaluated is null; null when the compliance summary could not be read (the run fails) - compliant_resources — Number of resources compliant with every policy that applies to them, the compliant entry of resources_by_compliance_state. With a resource_exclusions pattern set, counted over in-scope resources only, null when resources_evaluated is null; null when the compliance summary could not be read (the run fails) - resource_compliance_percentage — Percentage of evaluated resources that are compliant = compliant_resources / (compliant_resources + non_compliant_resources) × 100 (a whole number, 0–100); 0 when that sum is 0. Resources in other states (exempt, unknown, conflict, error, protected) are left out of both. With a resource_exclusions pattern set, over in-scope resources only, null when resources_evaluated is null; null when the compliance summary could not be read (the run fails) - - non_compliant_assignments — Number of in-scope policy assignments with at least one non-compliant resource. With no resource_exclusions pattern set, by the service's per-assignment count (null when the summary could not be read); with one set, by each assignment's in-scope latest states, null when resources_evaluated is null - - compliant_assignments — Number of in-scope policy assignments with no non-compliant resource. With no resource_exclusions pattern set, by the service's per-assignment count (null when the summary could not be read); with one set, by each assignment's in-scope latest states, null when resources_evaluated is null + - non_compliant_assignments — Number of in-scope policy assignments with at least one non-compliant resource. With no resource_exclusions pattern set, by the service's per-assignment count (null when the summary could not be read); with one set, by each assignment's in-scope latest states, null when resources_evaluated is null or the summary could not be read + - compliant_assignments — Number of in-scope policy assignments with no non-compliant resource. With no resource_exclusions pattern set, by the service's per-assignment count (null when the summary could not be read); with one set, by each assignment's in-scope latest states, null when resources_evaluated is null or the summary could not be read Which resources are non-compliant? - listed_non_compliant_states — Number of in-scope non-compliant (resource, policy) records listed in the evidence (at most the max_non_compliant_resources setting, default 1000; records a resource_exclusions pattern takes out don't count toward it); null when the non-compliant states query failed (the run fails) @@ -213,15 +213,15 @@ evidence_set: - remediation_tasks_by_provisioning_state — Number of in-scope remediation tasks in each state (for example Succeeded, Failed, Evaluating); a task's state covers every resource it remediated, including resources a resource_exclusions pattern takes out; null when the task list could not be read - remediation_successful_deployments — Number of remediation deployments that succeeded. With no resource_exclusions pattern set, as the tasks report them; with one set, the listed in-scope deployments whose status is Succeeded, null when remediation_deployments_truncated is true, max_remediation_deployments is 0 (the deployments are skipped), or the task list or any task's deployment list failed. With no pattern set, null only when the task list could not be read - remediation_failed_deployments — Number of remediation deployments that failed. With no resource_exclusions pattern set, as the tasks report them; with one set, the listed in-scope deployments whose status is Failed, null when remediation_deployments_truncated is true, max_remediation_deployments is 0 (the deployments are skipped), or the task list or any task's deployment list failed. With no pattern set, null only when the task list could not be read - - remediation_deployments_listed — Number of in-scope per-resource remediation deployments listed in the evidence (at most the max_remediation_deployments setting, default 1000; deployments a resource_exclusions pattern takes out don't count toward it) + - remediation_deployments_listed — Number of in-scope per-resource remediation deployments listed in the evidence (at most the max_remediation_deployments setting, default 1000; deployments a resource_exclusions pattern takes out don't count toward it); null when the task list or a task's deployment list could not be read (the run fails) - remediation_deployments_truncated — True if some in-scope deployments were not listed because of the max_remediation_deployments limit; false if every in-scope one is listed, or the limit is 0 and the deployments are skipped - - remediation_deployments_by_status — Number of listed remediation deployments in each status (for example Succeeded, Failed) - - remediated_resources_listed — Number of distinct resources among the listed remediation deployments (remediated resources). A resource deployed to more than once is judged by its newest deployment, against the policy of that deployment's task, so remediated_resources_now_compliant, remediated_resources_now_non_compliant, remediated_resources_not_yet_reevaluated and remediated_resources_compliance_unknown add up to it - - remediated_resources_now_compliant — Number of remediated resources whose current compliance state for the task's policy is Compliant and wasn't evaluated before their newest deployment - - remediated_resources_now_non_compliant — Number of remediated resources whose current compliance state for the task's policy is NonCompliant and wasn't evaluated before their newest deployment, including resources fixed and later drifted back - - succeeded_remediations_now_non_compliant — Number of remediated resources whose newest deployment succeeded but which are non-compliant with the task's policy now, so the fix didn't hold - - remediated_resources_not_yet_reevaluated — Number of remediated resources whose current compliance state was evaluated before their newest deployment last changed, so it doesn't yet show whether the fix worked (Azure re-evaluates every 24 hours) - - remediated_resources_compliance_unknown — Number of remediated resources with no current Compliant or NonCompliant state for the task's policy (not evaluated in the last day, exempt, deleted, or the state read failed); resources not yet re-evaluated are counted apart + - remediation_deployments_by_status — Number of listed remediation deployments in each status (for example Succeeded, Failed); null when the task list or a task's deployment list could not be read (the run fails) + - remediated_resources_listed — Number of distinct resources among the listed remediation deployments (remediated resources). A resource deployed to more than once is judged by its newest deployment, against the policy of that deployment's task, so remediated_resources_now_compliant, remediated_resources_now_non_compliant, remediated_resources_not_yet_reevaluated and remediated_resources_compliance_unknown add up to it; null when the task list or a task's deployment list could not be read (the run fails) + - remediated_resources_now_compliant — Number of remediated resources whose current compliance state for the task's policy is Compliant and wasn't evaluated before their newest deployment; null when the task list or a task's deployment list could not be read (the run fails) + - remediated_resources_now_non_compliant — Number of remediated resources whose current compliance state for the task's policy is NonCompliant and wasn't evaluated before their newest deployment, including resources fixed and later drifted back; null when the task list or a task's deployment list could not be read (the run fails) + - succeeded_remediations_now_non_compliant — Number of remediated resources whose newest deployment succeeded but which are non-compliant with the task's policy now, so the fix didn't hold; null when the task list or a task's deployment list could not be read (the run fails) + - remediated_resources_not_yet_reevaluated — Number of remediated resources whose current compliance state was evaluated before their newest deployment last changed, so it doesn't yet show whether the fix worked (Azure re-evaluates every 24 hours); null when the task list or a task's deployment list could not be read (the run fails) + - remediated_resources_compliance_unknown — Number of remediated resources with no current Compliant or NonCompliant state for the task's policy (not evaluated in the last day, exempt, deleted, or the state read failed); resources not yet re-evaluated are counted apart; null when the task list or a task's deployment list could not be read (the run fails) Does non-compliance keep coming back? - recurrence_window_days — Days of evaluation history read (30) diff --git a/tests/test_azure_policy_compliance.py b/tests/test_azure_policy_compliance.py index d577802d..8bf6aa8e 100644 --- a/tests/test_azure_policy_compliance.py +++ b/tests/test_azure_policy_compliance.py @@ -948,6 +948,34 @@ def boom(*a, **k): assert summary["listed_distinct_non_compliant_resources"] is None +DEPLOYMENT_COUNTS = ("remediation_deployments_listed", "remediation_deployments_by_status", + "remediated_resources_listed", "remediated_resources_now_compliant", + "remediated_resources_now_non_compliant", "succeeded_remediations_now_non_compliant", + "remediated_resources_not_yet_reevaluated", "remediated_resources_compliance_unknown") + + +def test_deployment_counts_are_null_when_a_deployment_list_was_not_read(monkeypatch, tmp_path): + summary = _run_main(monkeypatch, tmp_path, {**fx.empty_collection(), "tasks_read": True}) + assert all(summary[k] is None for k in DEPLOYMENT_COUNTS) + assert summary["remediation_deployments_truncated"] is False + + +def test_deployment_counts_read_zero_when_every_list_was_read(monkeypatch, tmp_path): + summary = _run_main(monkeypatch, tmp_path, {**fx.empty_collection(), "deployments_complete": True}) + assert all(summary[k] in (0, {}) for k in DEPLOYMENT_COUNTS) + summary = _run_main(monkeypatch, tmp_path, fx.empty_collection(), registration=fx.NOT_REGISTERED) + assert all(summary[k] in (0, {}) for k in DEPLOYMENT_COUNTS) + + +def test_assignment_counts_are_null_with_patterns_when_the_summary_was_not_read(monkeypatch, tmp_path): + monkeypatch.setenv("AZURE_RESOURCE_EXCLUSIONS", "rg:other") + pair = {"resource_id": VM, "policy_assignment_id": ASSIGN, "policy_definition_id": DEF, + "policy_definition_reference_id": "r1", "compliance_state": "noncompliant"} + collected = {**fx.empty_collection(), "latest_pairs": [pair], "latest_pairs_status": fx.LATEST_READ} + summary = _run_main(monkeypatch, tmp_path, collected) + assert summary["non_compliant_assignments"] is None and summary["compliant_assignments"] is None + + def test_listed_state_counts_read_zero_when_the_list_was_read_and_empty(monkeypatch, tmp_path): summary = _run_main(monkeypatch, tmp_path, {**fx.empty_collection(), "states_read": True}) assert summary["listed_non_compliant_states"] == 0 From 41bd3d97318a07a468dbcce04eaf276b5b62b96d Mon Sep 17 00:00:00 2001 From: Connor Dalton Date: Tue, 17 Nov 2026 12:57:00 -0700 Subject: [PATCH 4/6] fix(azure_policy_compliance): null state maps and truncated flags when their read failed The two compliance-state maps read {} when the summary was not read, and the two _truncated flags read false when their list was not read. All four now read null, and the field text says so. Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 5 +++++ fetchers/azure/policy_compliance/fetcher.py | 15 ++++++++------- fetchers/azure/policy_compliance/fetcher.yaml | 8 ++++---- tests/test_azure_policy_compliance.py | 9 ++++++++- 4 files changed, 25 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e81ed60..404526dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2773,6 +2773,11 @@ schemas and the `paramify` CLI — not the internal code. the `remediated_resources_*` counts (and `succeeded_remediations_now_non_compliant`) read null, not counts over the partial list. + - `resources_by_compliance_state` and `policies_by_compliance_state` read null, + not `{}`, when the compliance summary could not be read. + `non_compliant_states_truncated` reads null when the non-compliant states + query failed, and `remediation_deployments_truncated` reads null when the + task list or a task's deployment list was not read, not false. - **`azure_managed_identity_configuration` 0.3.0: an unread list gives null counts, not 0, and an unread federated-credential list no longer counts an diff --git a/fetchers/azure/policy_compliance/fetcher.py b/fetchers/azure/policy_compliance/fetcher.py index 1c853ee1..d97c3754 100644 --- a/fetchers/azure/policy_compliance/fetcher.py +++ b/fetchers/azure/policy_compliance/fetcher.py @@ -670,8 +670,10 @@ def summarize( if summary_read else None, "resource_compliance_percentage": coverage_percentage(compliant, compliant + non_compliant) if summary_read else None, - "resources_by_compliance_state": subscription_results.get("resource_details") or {}, - "policies_by_compliance_state": subscription_results.get("policy_details") or {}, + "resources_by_compliance_state": (subscription_results.get("resource_details") or {}) + if summary_read else None, + "policies_by_compliance_state": (subscription_results.get("policy_details") or {}) + if summary_read else None, # --- per assignment --- "assignments_evaluated": len(assignments) if summary_read else None, "non_compliant_assignments": sum(1 for a in assignments if not a["is_compliant"]) if summary_read else None, @@ -681,7 +683,7 @@ def summarize( "listed_distinct_non_compliant_resources": len( {str(s.get("resource_id") or "").lower() for s in states if s.get("resource_id")} ) if states_read else None, - "non_compliant_states_truncated": states_truncated, + "non_compliant_states_truncated": states_truncated if states_read else None, # --- remediation --- "remediation_tasks": len(remediations) if tasks_read else None, "remediation_tasks_by_provisioning_state": dict(sorted(by_state.items())) if tasks_read else None, @@ -798,8 +800,8 @@ def in_scope_summary(results: dict, assignments: list[dict], deployments: list[d "compliant_resources": compliant if known else None, "resources_evaluated": sum(details.values()) if known else None, "resource_compliance_percentage": coverage_percentage(compliant, compliant + non_compliant) if known else None, - "resources_by_compliance_state": details, - "policies_by_compliance_state": results.get("policy_details") or {}, + "resources_by_compliance_state": details if summary_read else None, + "policies_by_compliance_state": (results.get("policy_details") or {}) if summary_read else None, # The assignment list comes from the summary, so an unread summary leaves it empty, not zero. "non_compliant_assignments": ( sum(1 for a in assignments if a["is_compliant"] is False) if known and summary_read else None), @@ -880,8 +882,7 @@ def summarize_remediation_deployments(deployments: list[dict], truncated: bool, } if not complete: for key in summary: - if key != "remediation_deployments_truncated": - summary[key] = None + summary[key] = None return summary diff --git a/fetchers/azure/policy_compliance/fetcher.yaml b/fetchers/azure/policy_compliance/fetcher.yaml index 24da95c1..d2078b3b 100644 --- a/fetchers/azure/policy_compliance/fetcher.yaml +++ b/fetchers/azure/policy_compliance/fetcher.yaml @@ -175,8 +175,8 @@ evidence_set: - truncated (fewer groups came back than the service counted, or it counted more than 50,000) - uncounted (groups came back but their count didn't, so they can't be shown complete) - failed (the read or its count failed, and the run exits non-zero) - - resources_by_compliance_state — Number of resources in each compliance state (for example compliant, noncompliant, exempt, unknown), each resource counted once under its highest-ranked state in the order noncompliant, compliant, error, conflict, protected, exempt, unknown, as the service counts them. With a resource_exclusions pattern set, counted over in-scope resources, keeping every state the service reported (0 when none is in scope); every value is null when resources_evaluated is null - - policies_by_compliance_state — Number of policy assignments in each compliance state (an initiative counts once), each under its highest-ranked state across the resources it evaluated, as the service counts them. With a resource_exclusions pattern set, counted over in-scope resources only; every value is null when resources_evaluated is null + - resources_by_compliance_state — Number of resources in each compliance state (for example compliant, noncompliant, exempt, unknown), each resource counted once under its highest-ranked state in the order noncompliant, compliant, error, conflict, protected, exempt, unknown, as the service counts them. With a resource_exclusions pattern set, counted over in-scope resources, keeping every state the service reported (0 when none is in scope); every value is null when resources_evaluated is null; null when the compliance summary could not be read (the run fails) + - policies_by_compliance_state — Number of policy assignments in each compliance state (an initiative counts once), each under its highest-ranked state across the resources it evaluated, as the service counts them. With a resource_exclusions pattern set, counted over in-scope resources only; every value is null when resources_evaluated is null; null when the compliance summary could not be read (the run fails) - assignments_evaluated — Number of in-scope policy assignments the service summarized; null when the summary could not be read - provider_registration_status — Whether the policy compliance provider (Microsoft.PolicyInsights) is registered on the subscription: registered, not_registered or unknown; not_registered is valid evidence, not a failure: the run succeeds, nothing else is read, and the counts read 0 (null with a resource_exclusions pattern set) @@ -191,7 +191,7 @@ evidence_set: Which resources are non-compliant? - listed_non_compliant_states — Number of in-scope non-compliant (resource, policy) records listed in the evidence (at most the max_non_compliant_resources setting, default 1000; records a resource_exclusions pattern takes out don't count toward it); null when the non-compliant states query failed (the run fails) - listed_distinct_non_compliant_resources — Number of distinct resources among the listed non-compliant records; null when the non-compliant states query failed (the run fails) - - non_compliant_states_truncated — True if there were more in-scope non-compliant records than the list holds; false if every one is listed, or max_non_compliant_resources is 0 and the list is skipped + - non_compliant_states_truncated — True if there were more in-scope non-compliant records than the list holds; false if every one is listed, or max_non_compliant_resources is 0 and the list is skipped; null when the list could not be read Why did they fail? - evaluation_details_eligible_resources — Number of distinct resources among the listed non-compliant records whose evaluation details can be read (subscriptions and resource groups can't be) @@ -214,7 +214,7 @@ evidence_set: - remediation_successful_deployments — Number of remediation deployments that succeeded. With no resource_exclusions pattern set, as the tasks report them; with one set, the listed in-scope deployments whose status is Succeeded, null when remediation_deployments_truncated is true, max_remediation_deployments is 0 (the deployments are skipped), or the task list or any task's deployment list failed. With no pattern set, null only when the task list could not be read - remediation_failed_deployments — Number of remediation deployments that failed. With no resource_exclusions pattern set, as the tasks report them; with one set, the listed in-scope deployments whose status is Failed, null when remediation_deployments_truncated is true, max_remediation_deployments is 0 (the deployments are skipped), or the task list or any task's deployment list failed. With no pattern set, null only when the task list could not be read - remediation_deployments_listed — Number of in-scope per-resource remediation deployments listed in the evidence (at most the max_remediation_deployments setting, default 1000; deployments a resource_exclusions pattern takes out don't count toward it); null when the task list or a task's deployment list could not be read (the run fails) - - remediation_deployments_truncated — True if some in-scope deployments were not listed because of the max_remediation_deployments limit; false if every in-scope one is listed, or the limit is 0 and the deployments are skipped + - remediation_deployments_truncated — True if some in-scope deployments were not listed because of the max_remediation_deployments limit; false if every in-scope one is listed, or the limit is 0 and the deployments are skipped; null when the list could not be read - remediation_deployments_by_status — Number of listed remediation deployments in each status (for example Succeeded, Failed); null when the task list or a task's deployment list could not be read (the run fails) - remediated_resources_listed — Number of distinct resources among the listed remediation deployments (remediated resources). A resource deployed to more than once is judged by its newest deployment, against the policy of that deployment's task, so remediated_resources_now_compliant, remediated_resources_now_non_compliant, remediated_resources_not_yet_reevaluated and remediated_resources_compliance_unknown add up to it; null when the task list or a task's deployment list could not be read (the run fails) - remediated_resources_now_compliant — Number of remediated resources whose current compliance state for the task's policy is Compliant and wasn't evaluated before their newest deployment; null when the task list or a task's deployment list could not be read (the run fails) diff --git a/tests/test_azure_policy_compliance.py b/tests/test_azure_policy_compliance.py index d99cda4a..dfb80d54 100644 --- a/tests/test_azure_policy_compliance.py +++ b/tests/test_azure_policy_compliance.py @@ -944,6 +944,8 @@ def _run_main(monkeypatch, tmp_path, collected, registration="registered"): def test_counts_are_null_when_the_summary_and_task_reads_failed(monkeypatch, tmp_path): summary = _run_main(monkeypatch, tmp_path, fx.empty_collection()) assert all(summary[k] is None for k in SERVICE_COUNTS + TASK_COUNTS) + assert summary["resources_by_compliance_state"] is None and summary["policies_by_compliance_state"] is None + assert summary["non_compliant_states_truncated"] is None @needs_sdk @@ -964,6 +966,7 @@ def boom(*a, **k): summary = _run_main(monkeypatch, tmp_path, collected) assert summary["listed_non_compliant_states"] is None assert summary["listed_distinct_non_compliant_resources"] is None + assert summary["non_compliant_states_truncated"] is None DEPLOYMENT_COUNTS = ("remediation_deployments_listed", "remediation_deployments_by_status", @@ -975,12 +978,13 @@ def boom(*a, **k): def test_deployment_counts_are_null_when_a_deployment_list_was_not_read(monkeypatch, tmp_path): summary = _run_main(monkeypatch, tmp_path, {**fx.empty_collection(), "tasks_read": True}) assert all(summary[k] is None for k in DEPLOYMENT_COUNTS) - assert summary["remediation_deployments_truncated"] is False + assert summary["remediation_deployments_truncated"] is None def test_deployment_counts_read_zero_when_every_list_was_read(monkeypatch, tmp_path): summary = _run_main(monkeypatch, tmp_path, {**fx.empty_collection(), "deployments_complete": True}) assert all(summary[k] in (0, {}) for k in DEPLOYMENT_COUNTS) + assert summary["remediation_deployments_truncated"] is False summary = _run_main(monkeypatch, tmp_path, fx.empty_collection(), registration=fx.NOT_REGISTERED) assert all(summary[k] in (0, {}) for k in DEPLOYMENT_COUNTS) @@ -992,12 +996,14 @@ def test_assignment_counts_are_null_with_patterns_when_the_summary_was_not_read( collected = {**fx.empty_collection(), "latest_pairs": [pair], "latest_pairs_status": fx.LATEST_READ} summary = _run_main(monkeypatch, tmp_path, collected) assert summary["non_compliant_assignments"] is None and summary["compliant_assignments"] is None + assert summary["resources_by_compliance_state"] is None and summary["policies_by_compliance_state"] is None def test_listed_state_counts_read_zero_when_the_list_was_read_and_empty(monkeypatch, tmp_path): summary = _run_main(monkeypatch, tmp_path, {**fx.empty_collection(), "states_read": True}) assert summary["listed_non_compliant_states"] == 0 assert summary["listed_distinct_non_compliant_resources"] == 0 + assert summary["non_compliant_states_truncated"] is False def test_counts_read_zero_when_both_reads_came_back_empty(monkeypatch, tmp_path): @@ -1005,6 +1011,7 @@ def test_counts_read_zero_when_both_reads_came_back_empty(monkeypatch, tmp_path) summary = _run_main(monkeypatch, tmp_path, collected) assert all(summary[k] == 0 for k in SERVICE_COUNTS + ("remediation_tasks", "remediation_failed_deployments")) assert summary["remediation_tasks_by_provisioning_state"] == {} + assert summary["resources_by_compliance_state"] == {} and summary["policies_by_compliance_state"] == {} def test_each_read_nulls_only_its_own_counts(monkeypatch, tmp_path): From 56518532da653d28c9bc36411c3cb1e5784f9e02 Mon Sep 17 00:00:00 2001 From: Connor Dalton Date: Tue, 17 Nov 2026 12:58:53 -0700 Subject: [PATCH 5/6] test(azure_policy_compliance): truncated flags read null after a failed list read Co-Authored-By: Claude Sonnet 5.5 --- tests/test_azure_resource_exclusions_inventory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_azure_resource_exclusions_inventory.py b/tests/test_azure_resource_exclusions_inventory.py index 550fdd9b..0c34e050 100644 --- a/tests/test_azure_resource_exclusions_inventory.py +++ b/tests/test_azure_resource_exclusions_inventory.py @@ -703,7 +703,7 @@ def test_policy_list_limits_leave_the_recount_known_but_cut_deployments_null_the def test_policy_recount_survives_a_states_read_that_fails_partway(monkeypatch, tmp_path): code, evidence, _ = run_policy(monkeypatch, tmp_path, PATTERNS, fail=("states",)) summary, results = evidence["summary"], evidence["results"] - assert code == 1 and summary["non_compliant_states_truncated"] is False + assert code == 1 and summary["non_compliant_states_truncated"] is None assert (summary["non_compliant_resources"], summary["non_compliant_assignments"]) == (2, 1) assert [a["is_compliant"] for a in results["assignment_compliance"]] == [True, False] assert summary["remediation_failed_deployments"] == 0 From 44ad5dcd9ccb2bbea3f1d34bb42ade61d47d32aa Mon Sep 17 00:00:00 2001 From: Connor Dalton Date: Tue, 17 Nov 2026 12:59:02 -0700 Subject: [PATCH 6/6] test(azure_policy_compliance): deployments truncated flag reads null when a list was not read Co-Authored-By: Claude Sonnet 5.5 --- tests/test_azure_resource_exclusions_inventory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_azure_resource_exclusions_inventory.py b/tests/test_azure_resource_exclusions_inventory.py index 0c34e050..43d653a6 100644 --- a/tests/test_azure_resource_exclusions_inventory.py +++ b/tests/test_azure_resource_exclusions_inventory.py @@ -747,7 +747,7 @@ def test_policy_remediation_totals_are_null_when_a_task_or_deployment_list_fails for fail in ("deployments", "tasks"): code, evidence, _ = run_policy(monkeypatch, tmp_path, PATTERNS, fail=(fail,)) summary = evidence["summary"] - assert code == 1 and summary["remediation_deployments_truncated"] is False, fail + assert code == 1 and summary["remediation_deployments_truncated"] is None, fail assert (summary["remediation_failed_deployments"], summary["remediation_successful_deployments"]) == ( None, None), fail assert summary["non_compliant_resources"] == 2, fail