diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ee79711..3407cdf1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2682,6 +2682,24 @@ schemas and the `paramify` CLI — not the internal code. ### Fixed +- **`azure_alert_rules` 0.4.0: suppression fields are null when the alert + processing rule list could not be read.** The run still exits 1 as + `partial_failure`. Before, the refused list read as empty, so every rule showed + `suppressed: false` and counted as reaching a person. Now each enabled rule's + `suppressed` and `reaches_human` are null, and `enabled_rules_suppressed`, + `enabled_rules_possibly_suppressed`, `enabled_rules_reaching_human`, + `enabled_rules_reaching_human_unknown`, `enabled_rules_not_reaching_human`, + `reaching_human_percentage`, `all_enabled_rules_reach_human`, + `suppressing_processing_rules_active`, `processing_rules`, + `in_scope_processing_rules` and `excluded_processing_rules` are null. A list that was read and is + empty still gives 0 and false. An enabled rule with no deliverable action + group of its own is unknown too, since a processing rule may add one: + `routes_to_enabled_receiver` and `routes_to_person` read null, not false, so + `reaches_human` is null and the rule counts in + `enabled_rules_routing_unknown`. `processing_rule_matches` and + `suppressed_by_processing_rules` are null, not `[]`, and + `enabled_rules_without_action_groups` is null. + - **Disk, Key Vault, storage and SQL Managed Instance fetchers: an unread list gives null counts, not 0.** When a list the run needs is refused (403), throttled or times out, every total, count and percentage built from it is diff --git a/fetchers/azure/_shared/alert_routing.py b/fetchers/azure/_shared/alert_routing.py index a6e14cfc..65d1be14 100644 --- a/fetchers/azure/_shared/alert_routing.py +++ b/fetchers/azure/_shared/alert_routing.py @@ -195,9 +195,11 @@ def _tri(groups: list[dict], has: Callable[[dict], bool]) -> Optional[bool]: return False -def route(enabled: bool, group_ids: list[str], alert_scopes: list[str], groups: dict, rules: list[dict]) -> dict: - """Resolve an alert rule's notification path; `groups` is keyed by lower-cased action group id.""" - matches = matching_processing_rules(alert_scopes, rules) +def route(enabled: bool, group_ids: list[str], alert_scopes: list[str], groups: dict, rules: list[dict] | None) -> dict: + """Resolve an alert rule's notification path; `groups` is keyed by lower-cased action group id. + + rules=None means the processing rule list was not read, so whether anything suppresses the alert is unknown.""" + matches = matching_processing_rules(alert_scopes, rules or []) own = {} for gid in group_ids: if gid: @@ -224,8 +226,14 @@ def route(enabled: bool, group_ids: list[str], alert_scopes: list[str], groups: }) routes = _tri(resolved, lambda g: g["deliverable"]) if enabled else False to_person = _tri(resolved, lambda g: g["notifies_people"]) if enabled else False + if rules is None and enabled: + # A processing rule may add a group, so a "no" from the own groups is unknown. + routes = None if routes is False else routes + to_person = None if to_person is False else to_person suppressors = [m for m in matches if m["rule"]["suppresses_notifications"]] - if any(m["applies_to_every_fire"] for m in suppressors): + if rules is None and enabled: + suppressed = None + elif any(m["applies_to_every_fire"] for m in suppressors): suppressed = True elif suppressors: suppressed = None @@ -242,11 +250,11 @@ def route(enabled: bool, group_ids: list[str], alert_scopes: list[str], groups: "unresolved_action_group_ids": [g["id"] for g in resolved if not g["found"]], "routes_to_enabled_receiver": routes, "routes_to_person": to_person, - "processing_rule_matches": [ + "processing_rule_matches": None if rules is None else [ {"id": m["rule"]["id"], "scope_match": m["scope_match"], "applies_to_every_fire": m["applies_to_every_fire"]} for m in matches ], - "suppressed_by_processing_rules": [m["rule"]["id"] for m in suppressors], + "suppressed_by_processing_rules": None if rules is None else [m["rule"]["id"] for m in suppressors], "suppressed": suppressed, "reaches_human": reaches, } diff --git a/fetchers/azure/alert_rules/fetcher.py b/fetchers/azure/alert_rules/fetcher.py index 802d9350..130ecaec 100644 --- a/fetchers/azure/alert_rules/fetcher.py +++ b/fetchers/azure/alert_rules/fetcher.py @@ -129,7 +129,7 @@ def smart_detector_rule(raw: dict) -> dict: PROJECT = {RULE_METRIC: metric_rule, RULE_LOG_SEARCH: log_search_rule, RULE_SMART_DETECTOR: smart_detector_rule} -def rule_record(rule_type: str, raw: dict, groups: dict, processing_rules: list[dict]) -> dict: +def rule_record(rule_type: str, raw: dict, groups: dict, processing_rules: list[dict] | None) -> dict: rule = {"id": raw.get("id"), "name": raw.get("name"), **PROJECT[rule_type](raw)} return {**rule, **route(rule["enabled"], rule["action_group_ids"], rule["scopes"], groups, processing_rules)} @@ -138,8 +138,26 @@ def _count(rules: list[dict], field: str, value) -> int: return sum(1 for r in rules if r[field] is value) -def summarize(rules: list[dict], groups: list[dict], processing_rules: list[dict]) -> dict: - """Unknown (null) verdicts are counted apart, never folded into "no".""" +# Counts and verdicts that read suppression; null when the processing rule list was not read. +SUPPRESSION_FIELDS = ( + "enabled_rules_suppressed", "enabled_rules_possibly_suppressed", "enabled_rules_reaching_human", + "enabled_rules_reaching_human_unknown", "enabled_rules_not_reaching_human", "reaching_human_percentage", + "all_enabled_rules_reach_human", "suppressing_processing_rules_active", "processing_rules", + "enabled_rules_without_action_groups", +) + + +def summarize(rules: list[dict], groups: list[dict], processing_rules: list[dict], processing_rules_read: bool = True) -> dict: + """Unknown (null) verdicts are counted apart, never folded into "no". + + processing_rules_read=False (that list was not read) gives null for what depends on suppression.""" + summary = _summarize(rules, groups, processing_rules) + if not processing_rules_read: + summary.update(dict.fromkeys(SUPPRESSION_FIELDS)) + return summary + + +def _summarize(rules: list[dict], groups: list[dict], processing_rules: list[dict]) -> dict: alerting = [r for r in rules if r["is_alerting_rule"]] enabled = [r for r in alerting if r["enabled"]] reaching = _count(enabled, "reaches_human", True) @@ -201,6 +219,7 @@ def main() -> int: excluded: list[dict] = [] excluded_processing: list[dict] = [] unread = True + processing_read = True registration = {ns: REGISTRATION_UNKNOWN for ns in ("Microsoft.Insights", "Microsoft.AlertsManagement")} if subscription_id and cred is not None: for ns in registration: @@ -232,6 +251,7 @@ def main() -> int: lambda: collect_processing_rules(rest, subscription_id, now), ) processing_rules = apr["rules"] if apr else [] + processing_read = apr is not None unread = unread or apr is None raw_rules, excluded = exclusions.split( raw_rules, "alert_rule", resource_id=lambda t: t[1].get("id"), location=lambda t: t[1].get("location"), @@ -249,7 +269,7 @@ def main() -> int: outside_scope = sorted({g for g in referenced if not g.lower().startswith(prefix)}) referenced = [g for g in referenced if g.lower().startswith(prefix)] add_referenced_groups(rest, referenced, by_id, collector) - rules = [rule_record(t, r, by_id, processing_rules) for t, r in raw_rules] + rules = [rule_record(t, r, by_id, processing_rules if processing_read else None) for t, r in raw_rules] processing_rules = in_scope_processing elif not subscription_id: collector.record( @@ -284,10 +304,10 @@ def main() -> int: "excluded": excluded, }, summary={ - **summarize(rules, group_records, processing_rules), + **summarize(rules, group_records, processing_rules, processing_read), "provider_registration_status": registration, **exclusions.summary(excluded, ("alert_rule", "action_group", "processing_rule"), { - "alert_rules": len(rules), "action_groups": len(group_records), "processing_rules": len(processing_rules)}), + "alert_rules": len(rules), "action_groups": len(group_records), "processing_rules": len(processing_rules) if processing_read else None}), }, ) diff --git a/fetchers/azure/alert_rules/fetcher.yaml b/fetchers/azure/alert_rules/fetcher.yaml index dc338da4..08a2a91b 100644 --- a/fetchers/azure/alert_rules/fetcher.yaml +++ b/fetchers/azure/alert_rules/fetcher.yaml @@ -1,5 +1,5 @@ name: azure_alert_rules -version: 0.3.1 +version: 0.4.0 description: >- Shows every metric, log search and smart detector alert rule and whether each one actually reaches a person. @@ -101,29 +101,29 @@ evidence_set: - resource_exclusion_patterns_unmatched — List of configured patterns that matched no alert rule, action group or processing rule, such as a mistyped name; null when a list could not be read - excluded_alert_rules — Number of alert rules left out of scope because a pattern matched their resource group, region or id; each is listed under excluded with the pattern, and no other count includes them - excluded_action_groups — Number of action groups left out of scope the same way - - excluded_processing_rules — Number of alert processing rules left out of scope the same way + - excluded_processing_rules — Number of alert processing rules left out of scope the same way; null when the alert processing rule list could not be read - in_scope_alert_rules — Number of alert rules read and in scope; 0 means none was read or the patterns took out every rule, which a check can catch by requiring more than 0 - in_scope_action_groups — Number of action groups read and in scope - - in_scope_processing_rules — Number of alert processing rules read and in scope + - in_scope_processing_rules — Number of alert processing rules read and in scope; null when the alert processing rule list could not be read Do enabled rules notify anyone? - - enabled_rules_without_action_groups — Number of enabled rules with no action group to notify, either their own or one added by a processing rule + - enabled_rules_without_action_groups — Number of enabled rules with no action group to notify, either their own or one added by a processing rule; null when the alert processing rule list could not be read - enabled_rules_routing_to_enabled_receiver — Number of enabled rules that always notify a deliverable action group: one that is enabled and has at least one enabled receiver (email and SMS receivers count only when their status is Enabled; other receiver types always count) - - enabled_rules_routing_unknown — Number of enabled rules where it's unknown whether they notify anyone: no action group is known to deliver on every notification, and one couldn't be read, or a deliverable one is added only some of the time + - enabled_rules_routing_unknown — Number of enabled rules where it's unknown whether they notify anyone: no action group is known to deliver on every notification, and one couldn't be read, or a deliverable one is added only some of the time, or the alert processing rule list could not be read and the rule has no deliverable action group of its own - enabled_rules_routing_to_person — Number of enabled rules that always notify a person (email, SMS, voice call, app push, Azure role or ITSM ticket) - action_groups — Number of action groups in the subscription - deliverable_action_groups — Number of deliverable action groups in the subscription - action_groups_notifying_people — Number of deliverable action groups that notify a person Are notifications being suppressed? - - enabled_rules_suppressed — Number of enabled rules whose notifications an active suppression rule always switches off - - enabled_rules_possibly_suppressed — Number of enabled rules whose notifications a suppression rule might switch off (it covers only part of the scope, has conditions, or has a schedule that can't be evaluated) - - processing_rules — Number of alert processing rules in the subscription - - suppressing_processing_rules_active — Number of enabled suppression rules that are active now, or whose schedule can't rule it out + - enabled_rules_suppressed — Number of enabled rules whose notifications an active suppression rule always switches off; null when the alert processing rule list could not be read + - enabled_rules_possibly_suppressed — Number of enabled rules whose notifications a suppression rule might switch off (it covers only part of the scope, has conditions, or has a schedule that can't be evaluated); null when the alert processing rule list could not be read + - processing_rules — Number of alert processing rules in the subscription; null when the alert processing rule list could not be read + - suppressing_processing_rules_active — Number of enabled suppression rules that are active now, or whose schedule can't rule it out; null when the alert processing rule list could not be read Does every enabled rule reach a person? - - enabled_rules_reaching_human — Number of enabled rules that notify a person and aren't suppressed - - enabled_rules_reaching_human_unknown — Number of enabled rules where it's unknown whether they reach a person: they notify only webhooks, Logic Apps, Functions, runbooks or Event Hubs (code that may or may not tell anyone), an action group couldn't be read, one that notifies a person is added only some of the time, or a suppression rule might apply - - enabled_rules_not_reaching_human — Number of enabled rules that notify no deliverable action group, or that a suppression rule always switches off - - reaching_human_percentage — Percentage of enabled alerting rules that reach a person = enabled_rules_reaching_human / enabled_alerting_rules × 100 (a whole number, 0–100); 0 when enabled_alerting_rules is 0. Rules where reaching a person is unknown (enabled_rules_reaching_human_unknown) count in the total but not as reaching, so they lower it - - all_enabled_rules_reach_human — True if every enabled alerting rule reaches a person; false if any doesn't, or none is enabled; null if none fails but some are unknown + - enabled_rules_reaching_human — Number of enabled rules that notify a person and aren't suppressed; null when the alert processing rule list could not be read + - enabled_rules_reaching_human_unknown — Number of enabled rules where it's unknown whether they reach a person: they notify only webhooks, Logic Apps, Functions, runbooks or Event Hubs (code that may or may not tell anyone), an action group couldn't be read, one that notifies a person is added only some of the time, or a suppression rule might apply; null when the alert processing rule list could not be read + - enabled_rules_not_reaching_human — Number of enabled rules that notify no deliverable action group, or that a suppression rule always switches off; null when the alert processing rule list could not be read + - reaching_human_percentage — Percentage of enabled alerting rules that reach a person = enabled_rules_reaching_human / enabled_alerting_rules × 100 (a whole number, 0–100); 0 when enabled_alerting_rules is 0. Rules where reaching a person is unknown (enabled_rules_reaching_human_unknown) count in the total but not as reaching, so they lower it; null when the alert processing rule list could not be read + - all_enabled_rules_reach_human — True if every enabled alerting rule reaches a person; false if any doesn't, or none is enabled; null if none fails but some are unknown; null when the alert processing rule list could not be read diff --git a/tests/test_azure_alert_rules_null_counts.py b/tests/test_azure_alert_rules_null_counts.py new file mode 100644 index 00000000..696781e1 --- /dev/null +++ b/tests/test_azure_alert_rules_null_counts.py @@ -0,0 +1,123 @@ +"""A processing rule list that could not be read gives null suppression fields, never "not suppressed".""" + +from __future__ import annotations + +import importlib.util +import json +import sys +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parent.parent +AZURE = REPO_ROOT / "fetchers" / "azure" +sys.path.insert(0, str(REPO_ROOT / "fetchers" / "_lib")) +sys.path.insert(0, str(AZURE / "_shared")) + + +def _load(name): + spec = importlib.util.spec_from_file_location(f"azure_{name}_null_counts", AZURE / name / "fetcher.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +rules_fetcher = _load("alert_rules") +routing = sys.modules["alert_routing"] +RG = "/subscriptions/s1/resourceGroups/rg1" +GROUP = f"{RG}/providers/microsoft.insights/actionGroups/ag1" +RULE = {"id": f"{RG}/providers/Microsoft.Insights/metricAlerts/r1", "name": "r1", "location": "global", + "properties": {"enabled": True, "severity": 2, "scopes": [f"{RG}/providers/x/y/z"], + "actions": [{"actionGroupId": GROUP}]}} +SUPPRESS_ALL = {"id": f"{RG}/providers/Microsoft.AlertsManagement/actionRules/apr", "name": "apr", + "properties": {"enabled": True, "scopes": [RG], "actions": [{"actionType": "RemoveAllActionGroups"}]}} +SUPPRESSION_FIELDS = rules_fetcher.SUPPRESSION_FIELDS + + +def _run(monkeypatch, tmp_path, processing, refuse_processing=False, rule=RULE): + group = {"id": GROUP, "name": "ag1", "enabled": True, "deliverable": True, "notifies_people": True, + "receivers": [], "receiver_kinds": ["email"]} + m = rules_fetcher + + def processing_rules(rest, sub, now): + if refuse_processing: + raise PermissionError("(AuthorizationFailed) 403") + return {"rules": [routing.processing_rule_record(p, now) for p in processing]} + + monkeypatch.delenv("AZURE_RESOURCE_EXCLUSIONS", raising=False) + monkeypatch.setattr(m, "load_dotenv", lambda: None) + monkeypatch.setattr(m, "resolve_subscription", lambda c: {"subscription_id": "s1", "subscription_source": "env"}) + monkeypatch.setattr(m, "credential", lambda: object()) + monkeypatch.setattr(m, "provider_registration_status", lambda *a: "registered") + monkeypatch.setattr(m, "arm_rest_client", lambda cred: object()) + monkeypatch.setattr(m, "list_all", lambda rest, path, api: {"items": [rule] if path.endswith("metricAlerts") else [], "pages": 1}) + monkeypatch.setattr(m, "collect_action_groups", lambda rest, sub: {"groups": [group]}) + monkeypatch.setattr(m, "collect_processing_rules", processing_rules) + monkeypatch.setenv("EVIDENCE_DIR", str(tmp_path)) + code = m.main() + evidence = json.loads((tmp_path / "azure_alert_rules_s1.json").read_text()) + return code, evidence["results"]["alert_rules"], evidence["summary"] + + +def test_refused_processing_rule_list_gives_null_suppression_fields(monkeypatch, tmp_path): + code, rules, summary = _run(monkeypatch, tmp_path, [], refuse_processing=True) + assert code == 1 + assert (rules[0]["suppressed"], rules[0]["reaches_human"]) == (None, None) + assert {k: summary[k] for k in SUPPRESSION_FIELDS if summary[k] is not None} == {} + for key in ("processing_rules", "in_scope_processing_rules", "excluded_processing_rules"): + assert summary[key] is None, key + # What does not depend on the processing rules still reads. + assert summary["total_alert_rules"] == 1 and summary["enabled_alerting_rules"] == 1 + + +def test_read_empty_processing_rule_list_still_counts_zero_and_false(monkeypatch, tmp_path): + code, rules, summary = _run(monkeypatch, tmp_path, []) + assert code == 0 + assert (rules[0]["suppressed"], rules[0]["reaches_human"]) == (False, True) + assert summary["enabled_rules_suppressed"] == 0 and summary["enabled_rules_possibly_suppressed"] == 0 + assert summary["suppressing_processing_rules_active"] == 0 + assert (summary["processing_rules"], summary["in_scope_processing_rules"], + summary["excluded_processing_rules"]) == (0, 0, 0) + assert summary["enabled_rules_reaching_human"] == 1 and summary["all_enabled_rules_reach_human"] is True + + +def test_read_suppression_rule_still_suppresses(monkeypatch, tmp_path): + code, rules, summary = _run(monkeypatch, tmp_path, [SUPPRESS_ALL]) + assert code == 0 + assert (rules[0]["suppressed"], rules[0]["reaches_human"]) == (True, False) + assert summary["enabled_rules_suppressed"] == 1 and summary["enabled_rules_not_reaching_human"] == 1 + + +@pytest.mark.parametrize("enabled, expected", [(True, None), (False, False)]) +def test_route_with_unread_processing_rules_is_unknown_only_for_enabled_rules(enabled, expected): + r = routing.route(enabled, [GROUP], [RG], {GROUP.lower(): {"enabled": True, "deliverable": True, "notifies_people": True, + "receivers": []}}, None) + assert r["suppressed"] is expected + assert r["reaches_human"] is (None if enabled else False) + + +def test_unread_processing_rules_leave_a_rule_without_an_own_group_unknown(monkeypatch, tmp_path): + no_group = {**RULE, "properties": {**RULE["properties"], "actions": []}} + code, rules, summary = _run(monkeypatch, tmp_path, [], refuse_processing=True, rule=no_group) + assert code == 1 + assert (rules[0]["routes_to_enabled_receiver"], rules[0]["routes_to_person"], rules[0]["reaches_human"]) == ( + None, None, None) + assert rules[0]["processing_rule_matches"] is None and rules[0]["suppressed_by_processing_rules"] is None + assert summary["enabled_rules_without_action_groups"] is None + assert summary["enabled_rules_routing_unknown"] == 1 and summary["enabled_rules_routing_to_enabled_receiver"] == 0 + + +def test_read_empty_processing_rule_list_still_counts_a_rule_without_a_group(monkeypatch, tmp_path): + no_group = {**RULE, "properties": {**RULE["properties"], "actions": []}} + code, rules, summary = _run(monkeypatch, tmp_path, [], rule=no_group) + assert code == 0 and rules[0]["reaches_human"] is False + assert summary["enabled_rules_without_action_groups"] == 1 and summary["enabled_rules_routing_unknown"] == 0 + + +def test_route_with_unread_processing_rules_and_no_own_group_is_unknown(): + r = routing.route(True, [], [RG], {}, None) + assert (r["routes_to_enabled_receiver"], r["routes_to_person"], r["reaches_human"]) == (None, None, None) + assert r["processing_rule_matches"] is None and r["suppressed_by_processing_rules"] is None + r = routing.route(True, [], [RG], {}, []) + assert (r["routes_to_enabled_receiver"], r["reaches_human"]) == (False, False) + assert r["processing_rule_matches"] == [] and r["suppressed_by_processing_rules"] == []