diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ee79711..ad86db90 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2806,6 +2806,35 @@ schemas and the `paramify` CLI — not the internal code. explicitly enabled. Existing security-contact reads and failure handling are unchanged. +- **`azure_policy_compliance` 0.6.0: unread reads give null counts, and an + unread remediation task can no longer read as complete.** + - `deployments_complete` could read true on a run that missed a task's + deployment list: a later task's read replaced the record of unread tasks. + A task whose deployments could not be listed now keeps it false. + - A remediation task at a scope the fetcher doesn't recognise now fails the + run, where before it was skipped with only a log line. + - When the compliance summary could not be read, `non_compliant_resources`, + `non_compliant_policies`, `compliant_resources`, `resources_evaluated`, + `resource_compliance_percentage`, `assignments_evaluated`, + `non_compliant_assignments` and `compliant_assignments` read null, not 0. + When the remediation task list could not be read, `remediation_tasks`, + `remediation_tasks_by_provisioning_state` and the two deployment totals + read null, not 0. `listed_non_compliant_states` and + `listed_distinct_non_compliant_resources` read null when the non-compliant + states query failed. With the provider not registered they still read 0. + - With a `resource_exclusions` pattern set and the summary unread, + `non_compliant_assignments` and `compliant_assignments` read null, not 0. + - When the task list or any task's deployment list was not read, + `remediation_deployments_listed`, `remediation_deployments_by_status` and + the `remediated_resources_*` counts (and + `succeeded_remediations_now_non_compliant`) read null, not counts over the + partial list. + - `resources_by_compliance_state` and `policies_by_compliance_state` read null, + not `{}`, when the compliance summary could not be read. + `non_compliant_states_truncated` reads null when the non-compliant states + query failed, and `remediation_deployments_truncated` reads null when the + task list or a task's deployment list was not read, not false. + - **`azure_managed_identity_configuration` 0.3.0: an unread list gives null counts, not 0, and an unread federated-credential list no longer counts an identity as unattached.** diff --git a/fetchers/azure/policy_compliance/fetcher.py b/fetchers/azure/policy_compliance/fetcher.py index 9ba97e6a..d97c3754 100644 --- a/fetchers/azure/policy_compliance/fetcher.py +++ b/fetchers/azure/policy_compliance/fetcher.py @@ -647,8 +647,11 @@ def summarize( states: list[dict], states_truncated: bool, remediations: list[dict], + summary_read: bool = True, + tasks_read: bool = True, + states_read: bool = True, ) -> dict: - """Subscription-wide counts first (from the service), then what was itemized.""" + """Subscription-wide counts first (from the service), then what was itemized; null where the read failed.""" compliant, non_compliant = compliant_resource_counts( subscription_results.get("resource_details") ) @@ -658,32 +661,36 @@ def summarize( by_state[key] = by_state.get(key, 0) + 1 return { # --- the headline: the service's own subscription-wide counts --- - "non_compliant_resources": subscription_results.get("non_compliant_resources") or 0, - "non_compliant_policies": subscription_results.get("non_compliant_policies") or 0, - "compliant_resources": compliant, + "non_compliant_resources": (subscription_results.get("non_compliant_resources") or 0) if summary_read else None, + "non_compliant_policies": (subscription_results.get("non_compliant_policies") or 0) if summary_read else None, + "compliant_resources": compliant if summary_read else None, # Zero here means nothing was evaluated in the window, which is what tells # "0 non-compliant" (a finding) from "0 non-compliant, nothing assessed". - "resources_evaluated": sum((subscription_results.get("resource_details") or {}).values()), - "resource_compliance_percentage": coverage_percentage(compliant, compliant + non_compliant), - "resources_by_compliance_state": subscription_results.get("resource_details") or {}, - "policies_by_compliance_state": subscription_results.get("policy_details") or {}, + "resources_evaluated": sum((subscription_results.get("resource_details") or {}).values()) + if summary_read else None, + "resource_compliance_percentage": coverage_percentage(compliant, compliant + non_compliant) + if summary_read else None, + "resources_by_compliance_state": (subscription_results.get("resource_details") or {}) + if summary_read else None, + "policies_by_compliance_state": (subscription_results.get("policy_details") or {}) + if summary_read else None, # --- per assignment --- - "assignments_evaluated": len(assignments), - "non_compliant_assignments": sum(1 for a in assignments if not a["is_compliant"]), - "compliant_assignments": sum(1 for a in assignments if a["is_compliant"]), + "assignments_evaluated": len(assignments) if summary_read else None, + "non_compliant_assignments": sum(1 for a in assignments if not a["is_compliant"]) if summary_read else None, + "compliant_assignments": sum(1 for a in assignments if a["is_compliant"]) if summary_read else None, # --- the itemized list --- - "listed_non_compliant_states": len(states), + "listed_non_compliant_states": len(states) if states_read else None, "listed_distinct_non_compliant_resources": len( {str(s.get("resource_id") or "").lower() for s in states if s.get("resource_id")} - ), - "non_compliant_states_truncated": states_truncated, + ) if states_read else None, + "non_compliant_states_truncated": states_truncated if states_read else None, # --- remediation --- - "remediation_tasks": len(remediations), - "remediation_tasks_by_provisioning_state": dict(sorted(by_state.items())), - "remediation_failed_deployments": sum(r.get("failed_deployments") or 0 for r in remediations), - "remediation_successful_deployments": sum( - r.get("successful_deployments") or 0 for r in remediations - ), + "remediation_tasks": len(remediations) if tasks_read else None, + "remediation_tasks_by_provisioning_state": dict(sorted(by_state.items())) if tasks_read else None, + "remediation_failed_deployments": sum(r.get("failed_deployments") or 0 for r in remediations) + if tasks_read else None, + "remediation_successful_deployments": sum(r.get("successful_deployments") or 0 for r in remediations) + if tasks_read else None, } @@ -780,7 +787,8 @@ def in_scope_compliance(results: dict, assignments: list[dict], pairs: list[dict ) -def in_scope_summary(results: dict, assignments: list[dict], deployments: list[dict] | None) -> dict: +def in_scope_summary(results: dict, assignments: list[dict], deployments: list[dict] | None, + summary_read: bool = True) -> dict: """The summary fields recounted over in-scope records; deployments is None when that list is incomplete.""" known = results.get("non_compliant_resources") is not None details = results.get("resource_details") or {} @@ -792,10 +800,13 @@ def in_scope_summary(results: dict, assignments: list[dict], deployments: list[d "compliant_resources": compliant if known else None, "resources_evaluated": sum(details.values()) if known else None, "resource_compliance_percentage": coverage_percentage(compliant, compliant + non_compliant) if known else None, - "resources_by_compliance_state": details, - "policies_by_compliance_state": results.get("policy_details") or {}, - "non_compliant_assignments": sum(1 for a in assignments if a["is_compliant"] is False) if known else None, - "compliant_assignments": sum(1 for a in assignments if a["is_compliant"]) if known else None, + "resources_by_compliance_state": details if summary_read else None, + "policies_by_compliance_state": (results.get("policy_details") or {}) if summary_read else None, + # The assignment list comes from the summary, so an unread summary leaves it empty, not zero. + "non_compliant_assignments": ( + sum(1 for a in assignments if a["is_compliant"] is False) if known and summary_read else None), + "compliant_assignments": ( + sum(1 for a in assignments if a["is_compliant"]) if known and summary_read else None), "remediation_failed_deployments": statuses["failed"] if deployments is not None else None, "remediation_successful_deployments": statuses["succeeded"] if deployments is not None else None, } @@ -840,10 +851,11 @@ def summarize_recurrence( } -def summarize_remediation_deployments(deployments: list[dict], truncated: bool) -> dict: - """Deployments by status; the remediated_* counts are per resource, its latest deployment deciding.""" +def summarize_remediation_deployments(deployments: list[dict], truncated: bool, complete: bool = True) -> dict: + """Deployments by status; the remediated_* counts are per resource, its latest deployment deciding. + Null where the task list or a task's deployment list was not read, so a partial list can't read as 0.""" resources = latest_deployment_per_resource(deployments) - return { + summary = { "remediation_deployments_listed": len(deployments), "remediation_deployments_truncated": truncated, "remediation_deployments_by_status": sorted_counts( @@ -868,6 +880,10 @@ def summarize_remediation_deployments(deployments: list[dict], truncated: bool) if d["now_compliant"] is None and d.get("reevaluated_since_deployment") is not False ), } + if not complete: + for key in summary: + summary[key] = None + return summary def config_limit(env: str, default: int) -> int: @@ -888,6 +904,9 @@ def config_limit(env: str, default: int) -> int: def empty_collection() -> dict: return { "subscription_results": project_summary_results(None), + "summary_read": False, + "tasks_read": False, + "states_read": False, "assignments": [], "states": [], "states_truncated": False, @@ -1123,7 +1142,9 @@ def _current(policy: tuple, resource_ids: list[str]) -> dict: kind, scope = remediation_scope(task.get("id")) name = task.get("name") if kind not in listers: - logger.warning("remediation %s has an unrecognised scope; deployments not read", task.get("id")) + collector.record( + f"policyinsights.remediations.list_deployments (scope of {task.get('id')})", + ValueError("remediation scope not recognised; its deployments were not read")) unread.append(task.get("id")) continue @@ -1203,6 +1224,7 @@ def _summarize(): ) summarized = collector.guard("policyinsights.policy_states.summarize_for_subscription", _summarize) + collected["summary_read"] = summarized is not None subscription_results, assignments = summarized or (collected["subscription_results"], []) # Only assignments made inside an excluded group or resource go; the service's counts can't be split by resource. assignments, excluded = exclusions.split( @@ -1234,12 +1256,17 @@ def _states(): break states.append(record) + failed_before = len(collector.failures) collector.guard("policyinsights.policy_states.list_query_results_for_subscription", _states) + collected["states_read"] = len(collector.failures) == failed_before + else: + collected["states_read"] = True listed_tasks = collector.guard( "policyinsights.remediations.list_for_subscription", lambda: [project_remediation(r) for r in client.remediations.list_for_subscription()], ) + collected["tasks_read"] = listed_tasks is not None remediations, excluded_tasks = exclusions.split(listed_tasks or [], "remediation_task") # The service's counts can't be split by resource, so with patterns they are recounted from every latest pair. latest_pairs, latest_status = (collect_latest_pairs(client, subscription_id, collector, exclusions) @@ -1271,6 +1298,9 @@ def _states(): ) return { "subscription_results": subscription_results, + "summary_read": collected["summary_read"], + "tasks_read": collected["tasks_read"], + "states_read": collected["states_read"], "assignments": sorted(assignments, key=lambda a: a.get("policy_assignment_id") or ""), "states": sorted( states, @@ -1361,6 +1391,10 @@ def main() -> int: collected["states"], collected["states_truncated"], collected["remediations"], + # Nothing is read when the provider is unregistered, which is "not in use": 0, not unknown. + summary_read=collected["summary_read"] or registration == NOT_REGISTERED, + tasks_read=collected["tasks_read"] or registration == NOT_REGISTERED, + states_read=collected["states_read"] or registration == NOT_REGISTERED, ), **summarize_evaluation_details(collected["states"], collected["details"]), } @@ -1372,7 +1406,8 @@ def main() -> int: deployments_known = (limits["remediation_deployments"] > 0 and collected["deployments_complete"] and not collected["deployments_truncated"]) summary.update(in_scope_summary(subscription_results, assignments, - collected["deployments"] if deployments_known else None)) + collected["deployments"] if deployments_known else None, + summary_read=collected["summary_read"])) summary["in_scope_states_status"] = collected["latest_pairs_status"] recurrence = collected["recurrence"] evidence = build_payload( @@ -1403,7 +1438,8 @@ def main() -> int: recurrence["pairs_truncated"], recurrence["history_collected"], ), **summarize_remediation_deployments( - collected["deployments"], collected["deployments_truncated"] + collected["deployments"], collected["deployments_truncated"], + complete=collected["deployments_complete"] or registration == NOT_REGISTERED, ), "provider_registration_status": registration, **exclusions.summary(collected["excluded"], EXCLUDED_KINDS, { diff --git a/fetchers/azure/policy_compliance/fetcher.yaml b/fetchers/azure/policy_compliance/fetcher.yaml index afb5ddc4..d2078b3b 100644 --- a/fetchers/azure/policy_compliance/fetcher.yaml +++ b/fetchers/azure/policy_compliance/fetcher.yaml @@ -1,5 +1,5 @@ name: azure_policy_compliance -version: 0.5.1 +version: 0.6.0 description: >- Shows what Azure Policy found non-compliant on the subscription and why, whether remediation tasks fixed it, and which non-compliance keeps coming back. category: azure @@ -148,7 +148,7 @@ evidence_set: - policy_states.list_query_results_for_subscription("default") grouped per resource, region, policy and state, and a count of the groups — 30 days of evaluation history, checked to be complete - policy_states.list_query_results_for_subscription("default") for one pair, ordered by time (per in-scope part-time non-compliant pair, up to a limit) — counts its separate non-compliant episodes - remediations.list_for_subscription() — remediation tasks and their deployment counts - - remediations.list_deployments_at_subscription(), _at_resource_group(), _at_management_group() or _at_resource() (per in-scope task) — each resource a task remediated + - remediations.list_deployments_at_subscription(), _at_resource_group(), _at_management_group() or _at_resource() (per in-scope task) — each resource a task remediated; a task whose deployments cannot be listed, or made at a scope this fetcher doesn't recognise, fails the run - policy_states.list_query_results_for_subscription("latest") filtered to the task's policy and in-scope remediated resources (per task, 20 resources at a time) — whether each remediated resource is compliant now, and when that was evaluated - policy_states.list_query_results_for_subscription("latest") grouped per resource, region, policy and state, and a count of the groups (only with resource_exclusions patterns set) — every resource's and policy's latest state, to recount the subscription's and each assignment's compliance over in-scope resources @@ -169,29 +169,29 @@ evidence_set: - in_scope_remediation_deployments — Number of remediation deployments read and in scope What was evaluated? - - resources_evaluated — Number of resources with a compliance state in the service's evaluation window (the last day); 0 means nothing was evaluated, not that everything passed; also 0 when the summary read failed (the run fails) or Microsoft.PolicyInsights isn't registered. With a resource_exclusions pattern set, only in-scope resources count ("in scope": no pattern matches the resource by id, resource group or region), recounted from every latest (resource, policy) state; null unless that grouped read came back whole (see in_scope_states_status) + - resources_evaluated — Number of resources with a compliance state in the service's evaluation window (the last day); 0 means nothing was evaluated, not that everything passed; null when the summary read failed (the run fails); 0 when Microsoft.PolicyInsights isn't registered. With a resource_exclusions pattern set, only in-scope resources count ("in scope": no pattern matches the resource by id, resource group or region), recounted from every latest (resource, policy) state; null unless that grouped read came back whole (see in_scope_states_status) - in_scope_states_status — Whether the grouped read of every latest (resource, policy) state behind the in-scope counts came back whole, with a resource_exclusions pattern set; null when no pattern is set, or when the compliance reads didn't run (Microsoft.PolicyInsights not registered, no subscription or credential, or the Policy Insights client couldn't be created): - read (every group came back, so the in-scope counts stand; an empty count with no groups means nothing was evaluated) - truncated (fewer groups came back than the service counted, or it counted more than 50,000) - uncounted (groups came back but their count didn't, so they can't be shown complete) - failed (the read or its count failed, and the run exits non-zero) - - resources_by_compliance_state — Number of resources in each compliance state (for example compliant, noncompliant, exempt, unknown), each resource counted once under its highest-ranked state in the order noncompliant, compliant, error, conflict, protected, exempt, unknown, as the service counts them. With a resource_exclusions pattern set, counted over in-scope resources, keeping every state the service reported (0 when none is in scope); every value is null when resources_evaluated is null - - policies_by_compliance_state — Number of policy assignments in each compliance state (an initiative counts once), each under its highest-ranked state across the resources it evaluated, as the service counts them. With a resource_exclusions pattern set, counted over in-scope resources only; every value is null when resources_evaluated is null - - assignments_evaluated — Number of in-scope policy assignments the service summarized + - resources_by_compliance_state — Number of resources in each compliance state (for example compliant, noncompliant, exempt, unknown), each resource counted once under its highest-ranked state in the order noncompliant, compliant, error, conflict, protected, exempt, unknown, as the service counts them. With a resource_exclusions pattern set, counted over in-scope resources, keeping every state the service reported (0 when none is in scope); every value is null when resources_evaluated is null; null when the compliance summary could not be read (the run fails) + - policies_by_compliance_state — Number of policy assignments in each compliance state (an initiative counts once), each under its highest-ranked state across the resources it evaluated, as the service counts them. With a resource_exclusions pattern set, counted over in-scope resources only; every value is null when resources_evaluated is null; null when the compliance summary could not be read (the run fails) + - assignments_evaluated — Number of in-scope policy assignments the service summarized; null when the summary could not be read - provider_registration_status — Whether the policy compliance provider (Microsoft.PolicyInsights) is registered on the subscription: registered, not_registered or unknown; not_registered is valid evidence, not a failure: the run succeeds, nothing else is read, and the counts read 0 (null with a resource_exclusions pattern set) How compliant is the subscription? - - non_compliant_resources — Number of resources non-compliant with at least one policy, the noncompliant entry of resources_by_compliance_state. With no resource_exclusions pattern set, the service's own count; with one set, the in-scope resources with a non-compliant latest state, null when resources_evaluated is null - - non_compliant_policies — Number of policy assignments with at least one non-compliant resource (an initiative counts once), as the service counts them at subscription scope. With a resource_exclusions pattern set, counted over in-scope resources only, null when resources_evaluated is null - - compliant_resources — Number of resources compliant with every policy that applies to them, the compliant entry of resources_by_compliance_state. With a resource_exclusions pattern set, counted over in-scope resources only, null when resources_evaluated is null - - resource_compliance_percentage — Percentage of evaluated resources that are compliant = compliant_resources / (compliant_resources + non_compliant_resources) × 100 (a whole number, 0–100); 0 when that sum is 0. Resources in other states (exempt, unknown, conflict, error, protected) are left out of both. With a resource_exclusions pattern set, over in-scope resources only, null when resources_evaluated is null - - non_compliant_assignments — Number of in-scope policy assignments with at least one non-compliant resource. With no resource_exclusions pattern set, by the service's per-assignment count (0, not null, when that read failed); with one set, by each assignment's in-scope latest states, null when resources_evaluated is null - - compliant_assignments — Number of in-scope policy assignments with no non-compliant resource. With no resource_exclusions pattern set, by the service's per-assignment count (0, not null, when that read failed); with one set, by each assignment's in-scope latest states, null when resources_evaluated is null + - non_compliant_resources — Number of resources non-compliant with at least one policy, the noncompliant entry of resources_by_compliance_state. With no resource_exclusions pattern set, the service's own count; with one set, the in-scope resources with a non-compliant latest state, null when resources_evaluated is null; null when the compliance summary could not be read (the run fails) + - non_compliant_policies — Number of policy assignments with at least one non-compliant resource (an initiative counts once), as the service counts them at subscription scope. With a resource_exclusions pattern set, counted over in-scope resources only, null when resources_evaluated is null; null when the compliance summary could not be read (the run fails) + - compliant_resources — Number of resources compliant with every policy that applies to them, the compliant entry of resources_by_compliance_state. With a resource_exclusions pattern set, counted over in-scope resources only, null when resources_evaluated is null; null when the compliance summary could not be read (the run fails) + - resource_compliance_percentage — Percentage of evaluated resources that are compliant = compliant_resources / (compliant_resources + non_compliant_resources) × 100 (a whole number, 0–100); 0 when that sum is 0. Resources in other states (exempt, unknown, conflict, error, protected) are left out of both. With a resource_exclusions pattern set, over in-scope resources only, null when resources_evaluated is null; null when the compliance summary could not be read (the run fails) + - non_compliant_assignments — Number of in-scope policy assignments with at least one non-compliant resource. With no resource_exclusions pattern set, by the service's per-assignment count (null when the summary could not be read); with one set, by each assignment's in-scope latest states, null when resources_evaluated is null or the summary could not be read + - compliant_assignments — Number of in-scope policy assignments with no non-compliant resource. With no resource_exclusions pattern set, by the service's per-assignment count (null when the summary could not be read); with one set, by each assignment's in-scope latest states, null when resources_evaluated is null or the summary could not be read Which resources are non-compliant? - - listed_non_compliant_states — Number of in-scope non-compliant (resource, policy) records listed in the evidence (at most the max_non_compliant_resources setting, default 1000; records a resource_exclusions pattern takes out don't count toward it) - - listed_distinct_non_compliant_resources — Number of distinct resources among the listed non-compliant records - - non_compliant_states_truncated — True if there were more in-scope non-compliant records than the list holds; false if every one is listed, or max_non_compliant_resources is 0 and the list is skipped + - listed_non_compliant_states — Number of in-scope non-compliant (resource, policy) records listed in the evidence (at most the max_non_compliant_resources setting, default 1000; records a resource_exclusions pattern takes out don't count toward it); null when the non-compliant states query failed (the run fails) + - listed_distinct_non_compliant_resources — Number of distinct resources among the listed non-compliant records; null when the non-compliant states query failed (the run fails) + - non_compliant_states_truncated — True if there were more in-scope non-compliant records than the list holds; false if every one is listed, or max_non_compliant_resources is 0 and the list is skipped; null when the list could not be read Why did they fail? - evaluation_details_eligible_resources — Number of distinct resources among the listed non-compliant records whose evaluation details can be read (subscriptions and resource groups can't be) @@ -209,19 +209,19 @@ evidence_set: - failed (the read for the resource failed) Do remediation tasks fix what they target? - - remediation_tasks — Number of in-scope remediation tasks on the subscription; 0 when the task list could not be read - - remediation_tasks_by_provisioning_state — Number of in-scope remediation tasks in each state (for example Succeeded, Failed, Evaluating); a task's state covers every resource it remediated, including resources a resource_exclusions pattern takes out - - remediation_successful_deployments — Number of remediation deployments that succeeded. With no resource_exclusions pattern set, as the tasks report them; with one set, the listed in-scope deployments whose status is Succeeded, null when remediation_deployments_truncated is true, max_remediation_deployments is 0 (the deployments are skipped), or the task list or any task's deployment list failed - - remediation_failed_deployments — Number of remediation deployments that failed. With no resource_exclusions pattern set, as the tasks report them; with one set, the listed in-scope deployments whose status is Failed, null when remediation_deployments_truncated is true, max_remediation_deployments is 0 (the deployments are skipped), or the task list or any task's deployment list failed - - remediation_deployments_listed — Number of in-scope per-resource remediation deployments listed in the evidence (at most the max_remediation_deployments setting, default 1000; deployments a resource_exclusions pattern takes out don't count toward it) - - remediation_deployments_truncated — True if some in-scope deployments were not listed because of the max_remediation_deployments limit; false if every in-scope one is listed, or the limit is 0 and the deployments are skipped - - remediation_deployments_by_status — Number of listed remediation deployments in each status (for example Succeeded, Failed) - - remediated_resources_listed — Number of distinct resources among the listed remediation deployments (remediated resources). A resource deployed to more than once is judged by its newest deployment, against the policy of that deployment's task, so remediated_resources_now_compliant, remediated_resources_now_non_compliant, remediated_resources_not_yet_reevaluated and remediated_resources_compliance_unknown add up to it - - remediated_resources_now_compliant — Number of remediated resources whose current compliance state for the task's policy is Compliant and wasn't evaluated before their newest deployment - - remediated_resources_now_non_compliant — Number of remediated resources whose current compliance state for the task's policy is NonCompliant and wasn't evaluated before their newest deployment, including resources fixed and later drifted back - - succeeded_remediations_now_non_compliant — Number of remediated resources whose newest deployment succeeded but which are non-compliant with the task's policy now, so the fix didn't hold - - remediated_resources_not_yet_reevaluated — Number of remediated resources whose current compliance state was evaluated before their newest deployment last changed, so it doesn't yet show whether the fix worked (Azure re-evaluates every 24 hours) - - remediated_resources_compliance_unknown — Number of remediated resources with no current Compliant or NonCompliant state for the task's policy (not evaluated in the last day, exempt, deleted, or the state read failed); resources not yet re-evaluated are counted apart + - remediation_tasks — Number of in-scope remediation tasks on the subscription; null when the task list could not be read + - remediation_tasks_by_provisioning_state — Number of in-scope remediation tasks in each state (for example Succeeded, Failed, Evaluating); a task's state covers every resource it remediated, including resources a resource_exclusions pattern takes out; null when the task list could not be read + - remediation_successful_deployments — Number of remediation deployments that succeeded. With no resource_exclusions pattern set, as the tasks report them; with one set, the listed in-scope deployments whose status is Succeeded, null when remediation_deployments_truncated is true, max_remediation_deployments is 0 (the deployments are skipped), or the task list or any task's deployment list failed. With no pattern set, null only when the task list could not be read + - remediation_failed_deployments — Number of remediation deployments that failed. With no resource_exclusions pattern set, as the tasks report them; with one set, the listed in-scope deployments whose status is Failed, null when remediation_deployments_truncated is true, max_remediation_deployments is 0 (the deployments are skipped), or the task list or any task's deployment list failed. With no pattern set, null only when the task list could not be read + - remediation_deployments_listed — Number of in-scope per-resource remediation deployments listed in the evidence (at most the max_remediation_deployments setting, default 1000; deployments a resource_exclusions pattern takes out don't count toward it); null when the task list or a task's deployment list could not be read (the run fails) + - remediation_deployments_truncated — True if some in-scope deployments were not listed because of the max_remediation_deployments limit; false if every in-scope one is listed, or the limit is 0 and the deployments are skipped; null when the list could not be read + - remediation_deployments_by_status — Number of listed remediation deployments in each status (for example Succeeded, Failed); null when the task list or a task's deployment list could not be read (the run fails) + - remediated_resources_listed — Number of distinct resources among the listed remediation deployments (remediated resources). A resource deployed to more than once is judged by its newest deployment, against the policy of that deployment's task, so remediated_resources_now_compliant, remediated_resources_now_non_compliant, remediated_resources_not_yet_reevaluated and remediated_resources_compliance_unknown add up to it; null when the task list or a task's deployment list could not be read (the run fails) + - remediated_resources_now_compliant — Number of remediated resources whose current compliance state for the task's policy is Compliant and wasn't evaluated before their newest deployment; null when the task list or a task's deployment list could not be read (the run fails) + - remediated_resources_now_non_compliant — Number of remediated resources whose current compliance state for the task's policy is NonCompliant and wasn't evaluated before their newest deployment, including resources fixed and later drifted back; null when the task list or a task's deployment list could not be read (the run fails) + - succeeded_remediations_now_non_compliant — Number of remediated resources whose newest deployment succeeded but which are non-compliant with the task's policy now, so the fix didn't hold; null when the task list or a task's deployment list could not be read (the run fails) + - remediated_resources_not_yet_reevaluated — Number of remediated resources whose current compliance state was evaluated before their newest deployment last changed, so it doesn't yet show whether the fix worked (Azure re-evaluates every 24 hours); null when the task list or a task's deployment list could not be read (the run fails) + - remediated_resources_compliance_unknown — Number of remediated resources with no current Compliant or NonCompliant state for the task's policy (not evaluated in the last day, exempt, deleted, or the state read failed); resources not yet re-evaluated are counted apart; null when the task list or a task's deployment list could not be read (the run fails) Does non-compliance keep coming back? - recurrence_window_days — Days of evaluation history read (30) diff --git a/tests/test_azure_policy_compliance.py b/tests/test_azure_policy_compliance.py index d7d569a0..dfb80d54 100644 --- a/tests/test_azure_policy_compliance.py +++ b/tests/test_azure_policy_compliance.py @@ -880,3 +880,145 @@ def collect(subscription_id, cred, collector, limits, exclusions=None): assert fx.main() == 0 summary = json.loads((tmp_path / "azure_policy_compliance_s1.json").read_text())["summary"] assert_summary_documented("azure/policy_compliance", summary) + + +def _deployment(rid): + return NS(remediated_resource_id=rid, deployment_id=None, status="Succeeded", resource_location=None, + error=None, created_on=None, last_updated_on=None) + + +def _task(name, scope=SUB, ref="r1"): + return {"id": f"{scope}/providers/Microsoft.PolicyInsights/remediations/{name}", "name": name, + "policy_assignment_id": ASSIGN, "policy_definition_reference_id": ref, "total_deployments": 1} + + +@needs_sdk +def test_a_task_whose_deployments_could_not_be_listed_stays_unread_after_later_tasks_are_read(): + def at_subscription(name): + if name == "t2": + raise RuntimeError("throttled") + return [_deployment(VM)] + + client = fake_client( + policy_states={"list_query_results_for_subscription": lambda *a, **k: []}, + remediations={"list_deployments_at_subscription": at_subscription}, + ) + unread: list = [] + collector = fx.Collector(logging.getLogger("t")) + rows, _, _ = fx.collect_remediation_deployments( + client, "s1", [_task("t1"), _task("t2")], 10, collector, unread=unread) + assert len(rows) == 1 + assert unread == [_task("t2")["id"]] + assert not collector.ok + + +@needs_sdk +def test_unrecognised_remediation_scope_fails_the_run(): + client = fake_client(policy_states={}, remediations={}) + odd = {**_task("t1"), "id": "/providers/Odd/remediations/t1"} + unread: list = [] + collector = fx.Collector(logging.getLogger("t")) + fx.collect_remediation_deployments(client, "s1", [odd], 10, collector, unread=unread) + assert unread == [odd["id"]] + assert not collector.ok + + +def _run_main(monkeypatch, tmp_path, collected, registration="registered"): + monkeypatch.setattr(fx, "load_dotenv", lambda: None) + monkeypatch.setenv("EVIDENCE_DIR", str(tmp_path)) + monkeypatch.setattr(fx, "resolve_subscription", lambda c: {"subscription_id": "s1", "subscription_source": "env"}) + monkeypatch.setattr(fx, "credential", lambda: object()) + monkeypatch.setattr(fx, "provider_registration_status", lambda *a: registration) + monkeypatch.setattr(fx, "collect_compliance", lambda *a, **k: collected) + assert fx.main() == 0 + return json.loads((tmp_path / "azure_policy_compliance_s1.json").read_text())["summary"] + + +SERVICE_COUNTS = ("non_compliant_resources", "non_compliant_policies", "compliant_resources", "resources_evaluated", + "resource_compliance_percentage", "assignments_evaluated", "non_compliant_assignments", + "compliant_assignments") +TASK_COUNTS = ("remediation_tasks", "remediation_tasks_by_provisioning_state", "remediation_failed_deployments", + "remediation_successful_deployments") + + +def test_counts_are_null_when_the_summary_and_task_reads_failed(monkeypatch, tmp_path): + summary = _run_main(monkeypatch, tmp_path, fx.empty_collection()) + assert all(summary[k] is None for k in SERVICE_COUNTS + TASK_COUNTS) + assert summary["resources_by_compliance_state"] is None and summary["policies_by_compliance_state"] is None + assert summary["non_compliant_states_truncated"] is None + + +@needs_sdk +def test_listed_state_counts_are_null_when_the_states_query_fails(monkeypatch, tmp_path): + def boom(*a, **k): + raise RuntimeError("throttled") + + client = fake_client(policy_states={"list_query_results_for_subscription": boom}) + monkeypatch.setattr(fx, "collect_evaluation_details", lambda *a, **k: fx.empty_collection()["details"]) + monkeypatch.setattr(fx, "collect_recurrence", lambda *a, **k: fx.empty_collection()["recurrence"]) + monkeypatch.setattr(fx, "collect_remediation_deployments", lambda *a, **k: ([], False, [])) + import azure.mgmt.policyinsights as pi + monkeypatch.setattr(pi, "PolicyInsightsClient", lambda **k: client) + collector = fx.Collector(logging.getLogger("t")) + limits = {"non_compliant": 10, "detail_resources": 1, "recurrence_pairs": 1, "remediation_deployments": 1} + collected = fx.collect_compliance("s1", object(), collector, limits) + assert collected["states_read"] is False and not collector.ok + summary = _run_main(monkeypatch, tmp_path, collected) + assert summary["listed_non_compliant_states"] is None + assert summary["listed_distinct_non_compliant_resources"] is None + assert summary["non_compliant_states_truncated"] is None + + +DEPLOYMENT_COUNTS = ("remediation_deployments_listed", "remediation_deployments_by_status", + "remediated_resources_listed", "remediated_resources_now_compliant", + "remediated_resources_now_non_compliant", "succeeded_remediations_now_non_compliant", + "remediated_resources_not_yet_reevaluated", "remediated_resources_compliance_unknown") + + +def test_deployment_counts_are_null_when_a_deployment_list_was_not_read(monkeypatch, tmp_path): + summary = _run_main(monkeypatch, tmp_path, {**fx.empty_collection(), "tasks_read": True}) + assert all(summary[k] is None for k in DEPLOYMENT_COUNTS) + assert summary["remediation_deployments_truncated"] is None + + +def test_deployment_counts_read_zero_when_every_list_was_read(monkeypatch, tmp_path): + summary = _run_main(monkeypatch, tmp_path, {**fx.empty_collection(), "deployments_complete": True}) + assert all(summary[k] in (0, {}) for k in DEPLOYMENT_COUNTS) + assert summary["remediation_deployments_truncated"] is False + summary = _run_main(monkeypatch, tmp_path, fx.empty_collection(), registration=fx.NOT_REGISTERED) + assert all(summary[k] in (0, {}) for k in DEPLOYMENT_COUNTS) + + +def test_assignment_counts_are_null_with_patterns_when_the_summary_was_not_read(monkeypatch, tmp_path): + monkeypatch.setenv("AZURE_RESOURCE_EXCLUSIONS", "rg:other") + pair = {"resource_id": VM, "policy_assignment_id": ASSIGN, "policy_definition_id": DEF, + "policy_definition_reference_id": "r1", "compliance_state": "noncompliant"} + collected = {**fx.empty_collection(), "latest_pairs": [pair], "latest_pairs_status": fx.LATEST_READ} + summary = _run_main(monkeypatch, tmp_path, collected) + assert summary["non_compliant_assignments"] is None and summary["compliant_assignments"] is None + assert summary["resources_by_compliance_state"] is None and summary["policies_by_compliance_state"] is None + + +def test_listed_state_counts_read_zero_when_the_list_was_read_and_empty(monkeypatch, tmp_path): + summary = _run_main(monkeypatch, tmp_path, {**fx.empty_collection(), "states_read": True}) + assert summary["listed_non_compliant_states"] == 0 + assert summary["listed_distinct_non_compliant_resources"] == 0 + assert summary["non_compliant_states_truncated"] is False + + +def test_counts_read_zero_when_both_reads_came_back_empty(monkeypatch, tmp_path): + collected = {**fx.empty_collection(), "summary_read": True, "tasks_read": True} + summary = _run_main(monkeypatch, tmp_path, collected) + assert all(summary[k] == 0 for k in SERVICE_COUNTS + ("remediation_tasks", "remediation_failed_deployments")) + assert summary["remediation_tasks_by_provisioning_state"] == {} + assert summary["resources_by_compliance_state"] == {} and summary["policies_by_compliance_state"] == {} + + +def test_each_read_nulls_only_its_own_counts(monkeypatch, tmp_path): + summary = _run_main(monkeypatch, tmp_path, {**fx.empty_collection(), "summary_read": True}) + assert summary["non_compliant_resources"] == 0 and summary["remediation_tasks"] is None + + +def test_unregistered_provider_still_reads_zero(monkeypatch, tmp_path): + summary = _run_main(monkeypatch, tmp_path, fx.empty_collection(), registration=fx.NOT_REGISTERED) + assert summary["non_compliant_resources"] == 0 and summary["remediation_tasks"] == 0 diff --git a/tests/test_azure_resource_exclusions_inventory.py b/tests/test_azure_resource_exclusions_inventory.py index 550fdd9b..43d653a6 100644 --- a/tests/test_azure_resource_exclusions_inventory.py +++ b/tests/test_azure_resource_exclusions_inventory.py @@ -703,7 +703,7 @@ def test_policy_list_limits_leave_the_recount_known_but_cut_deployments_null_the def test_policy_recount_survives_a_states_read_that_fails_partway(monkeypatch, tmp_path): code, evidence, _ = run_policy(monkeypatch, tmp_path, PATTERNS, fail=("states",)) summary, results = evidence["summary"], evidence["results"] - assert code == 1 and summary["non_compliant_states_truncated"] is False + assert code == 1 and summary["non_compliant_states_truncated"] is None assert (summary["non_compliant_resources"], summary["non_compliant_assignments"]) == (2, 1) assert [a["is_compliant"] for a in results["assignment_compliance"]] == [True, False] assert summary["remediation_failed_deployments"] == 0 @@ -747,7 +747,7 @@ def test_policy_remediation_totals_are_null_when_a_task_or_deployment_list_fails for fail in ("deployments", "tasks"): code, evidence, _ = run_policy(monkeypatch, tmp_path, PATTERNS, fail=(fail,)) summary = evidence["summary"] - assert code == 1 and summary["remediation_deployments_truncated"] is False, fail + assert code == 1 and summary["remediation_deployments_truncated"] is None, fail assert (summary["remediation_failed_deployments"], summary["remediation_successful_deployments"]) == ( None, None), fail assert summary["non_compliant_resources"] == 2, fail