From df179e7c910932ff6f0e7f0cc2d4eb5b2a065f2e Mon Sep 17 00:00:00 2001 From: Shubham Date: Fri, 30 Oct 2026 10:48:24 +1300 Subject: [PATCH] Show which interfaces and subnets no policy rule talks about (closes #421) PC-049 reports devices no rule covers. That is too coarse: a router can be "covered" by one rule about one subnet while most of its interfaces carry traffic no rule mentions. analysis/policy_coverage.py goes to each interface Batfish reports and asks whether any rule's source or destination overlaps its subnet. Only rules about a device that is in the snapshot count: a rule about a missing device never runs (PC-050), so it covers nothing. Three answers, as in the rest of the policy code: covered, uncovered, and unknown when the only candidate rules use address syntax policy_conflicts does not parse. A device whose interfaces cannot be listed is unknown, never covered. An uncovered subnet is coverage information, not a violation. coverage_map() returns rows for the dashboard to draw; `python -m analysis.policy_coverage [policy.json]` prints them. Live on routing-secure with a policy about the HQ LAN: both LANs covered, the two point-to-point links uncovered. Stacked on #387 for policy_conflicts' address parser. Mutation-tested: rules about absent devices counted, unsure reported as uncovered, the destination ignored, an unconstrained query ignored, and an unreadable device dropped -- each fails the new tests. 1944 passed, 13 skipped with Batfish running. ruff clean. --- analysis/policy_coverage.py | 160 ++++++++++++++++++++++++++++++++++ docs/policy-rules.md | 22 +++++ tests/test_policy_coverage.py | 86 ++++++++++++++++++ 3 files changed, 268 insertions(+) create mode 100644 analysis/policy_coverage.py create mode 100644 tests/test_policy_coverage.py diff --git a/analysis/policy_coverage.py b/analysis/policy_coverage.py new file mode 100644 index 0000000..15f59be --- /dev/null +++ b/analysis/policy_coverage.py @@ -0,0 +1,160 @@ +"""Which parts of my network does no policy rule talk about? (#421) + +WHY THIS EXISTS + PC-049 (#228/#229, #366) already reports devices no rule covers. That is + too coarse: a router can be "covered" by one rule about one subnet while + three of its four interfaces carry traffic no rule mentions. This goes + one level down, to each interface and the subnet on it. + +WHAT "COVERED" MEANS HERE + A subnet is covered when some policy_compliance rule's source or + destination overlaps it -- the rule says something about traffic from or + to that subnet. A query with no address constraint talks about all + traffic, so it covers every subnet. + + Only rules about a device that IS in the snapshot count. A rule about a + missing device never runs (PC-050), so it covers nothing, however its + addresses read. + +THREE ANSWERS, AS EVERYWHERE ELSE IN THIS POLICY CODE + covered some rule provably overlaps the subnet + uncovered no rule does + unknown no rule provably does, and at least one uses address syntax + analysis/policy_conflicts does not parse -- so it might + + And a device whose interfaces Batfish cannot list is UNKNOWN, never + covered. An uncovered subnet is coverage information, not a violation: + nothing here says the network is wrong, only where the policy is silent. + + Drawing the map is the dashboard's job; this returns the data. +""" + +from __future__ import annotations + +import ipaddress +from typing import Any, Dict, List, Optional, Sequence, Set + +from analysis.policy_conflicts import _Undecidable, _networks + + +def interfaces(bf: Any) -> Optional[Dict[str, List[Dict[str, Any]]]]: + """{device: [{interface, subnet, active}]} from Batfish, or None if unreadable. + + Subnets are the NETWORK of each interface address (10.10.10.1/24 -> + 10.10.10.0/24), because that is what a rule's addresses are compared with. + """ + frame = bf.q.interfaceProperties(properties="All_Prefixes,Active").answer().frame() + out: Dict[str, List[Dict[str, Any]]] = {} + for _, row in frame.iterrows(): + iface = row["Interface"] + for prefix in row["All_Prefixes"] or []: + out.setdefault(str(iface.hostname), []).append({ + "interface": str(iface.interface), + "subnet": ipaddress.ip_interface(str(prefix)).network, + "active": bool(row["Active"]), + }) + return out + + +def _touches(rule: Dict[str, Any], subnet: Any) -> Optional[bool]: + """Does this rule's source or destination overlap `subnet`? True/False/None.""" + unsure = False + for query in rule.get("queries", []) or []: + if not isinstance(query, dict): + unsure = True + continue + if "srcIps" not in query or "dstIps" not in query: + return True # an unconstrained side talks about everything + for field in ("srcIps", "dstIps"): + try: + if any(net.overlaps(subnet) for net in _networks(field, query[field])): + return True + except _Undecidable: + unsure = True + return None if unsure else False + + +def coverage_map(rules: Sequence[Dict[str, Any]], + by_device: Optional[Dict[str, List[Dict[str, Any]]]], + devices: Optional[Set[str]] = None) -> List[Dict[str, Any]]: + """One row per interface subnet, saying whether any rule talks about it. + + `devices` is every device in the snapshot. A device in it whose interfaces + could not be read gets one row with `covered` None -- unknown, not covered. + """ + present = set(devices or []) | set(by_device or {}) + live_rules = [r for r in rules if r.get("node") in present] + rows: List[Dict[str, Any]] = [] + for device in sorted(present): + listed = (by_device or {}).get(device) + if not listed: + rows.append({"device": device, "interface": None, "subnet": None, "active": None, + "covered": None, "rules": [], "unsure": [], + "reason": "Batfish listed no addressed interfaces for this device"}) + continue + for entry in listed: + sure, unsure = [], [] + for rule in live_rules: + verdict = _touches(rule, entry["subnet"]) + if verdict is True: + sure.append(rule.get("number")) + elif verdict is None: + unsure.append(rule.get("number")) + rows.append({"device": device, "interface": entry["interface"], + "subnet": str(entry["subnet"]), "active": entry["active"], + "covered": True if sure else (None if unsure else False), + "rules": sure, "unsure": unsure, "reason": ""}) + return rows + + +def summary(rows: Sequence[Dict[str, Any]]) -> str: + known = [r for r in rows if r["subnet"]] + covered = sum(1 for r in known if r["covered"] is True) + unknown = sum(1 for r in rows if r["covered"] is None) + text = f"{covered} of {len(known)} interface subnet(s) are covered by a policy rule" + if unknown: + text += f"; {unknown} could not be decided" + return text + + +def main(argv: Optional[Sequence[str]] = None) -> int: + """`python -m analysis.policy_coverage [policy.json]`""" + import sys + from pathlib import Path + + from analysis import snapshot + from analysis.checks import policy_compliance as pc + from analysis.pipeline import connect, load_snapshot + from analysis.policy import PolicyError, load_policy_file + + args = list(sys.argv[1:] if argv is None else argv) + if len(args) not in (1, 2): + print("usage: python -m analysis.policy_coverage [policy.json]") + return 2 + rules = pc.POLICY_RULES + if len(args) == 2: + try: + rules = load_policy_file(args[1]).entries_for("policy_compliance") + except PolicyError as error: + print(f"{args[1]}: not loaded -- {error}") + return 2 + bf = connect(None) + load_snapshot(bf, Path(args[0]), "netwise-coverage", "snapshot") + try: + by_device = interfaces(bf) + except Exception as error: # noqa: BLE001 -- said, not swallowed + print(f"could not list interfaces, so coverage is unknown: {error}") + by_device = None + rows = coverage_map(rules, by_device, snapshot.device_names(bf)) + label = {True: "covered", False: "UNCOVERED", None: "unknown"} + for r in rows: + who = (f"rule(s) {', '.join(map(str, r['rules']))}" if r["rules"] + else (f"maybe rule(s) {', '.join(map(str, r['unsure']))}" if r["unsure"] else r["reason"])) + print(f" {r['device']:<12} {str(r['interface'] or '-'):<22} {str(r['subnet'] or '-'):<18} " + f"{label[r['covered']]:<10} {who}") + print(summary(rows)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/docs/policy-rules.md b/docs/policy-rules.md index 82eb4fc..b8cde86 100644 --- a/docs/policy-rules.md +++ b/docs/policy-rules.md @@ -367,6 +367,28 @@ leaves the user diffing two header specifiers by eye. This needs no Batfish, no Docker and no config. It is a property of the policy file alone. +### Which subnets does no rule talk about? (#421) + +PC-049 reports devices no rule covers. `analysis/policy_coverage.py` goes one +level down: for every interface Batfish reports, is there a rule whose source +or destination overlaps its subnet? + +``` +$ python -m analysis.policy_coverage tests/fixtures/routing-secure my-policy.json + rtr-branch GigabitEthernet0/0 10.1.1.0/30 UNCOVERED + rtr-branch GigabitEthernet0/1 10.20.20.0/24 covered rule(s) 1 + rtr-hq GigabitEthernet0/0 10.10.10.0/24 covered rule(s) 1 + rtr-hq GigabitEthernet0/1 10.1.1.0/30 UNCOVERED +2 of 4 interface subnet(s) are covered by a policy rule +``` + +Only rules about a device that is in the snapshot count — a rule about a +missing device never runs (PC-050), so it covers nothing. Address syntax this +code cannot parse makes a subnet **unknown**, not uncovered, and a device +whose interfaces cannot be listed is unknown, never covered. An uncovered +subnet is coverage information, not a violation. Drawing it is the +dashboard's job; `coverage_map()` returns the rows. + #### What is not a contradiction Two prohibitions over the same traffic are consistent — denying twice is diff --git a/tests/test_policy_coverage.py b/tests/test_policy_coverage.py new file mode 100644 index 0000000..8a952eb --- /dev/null +++ b/tests/test_policy_coverage.py @@ -0,0 +1,86 @@ +"""Which interfaces and subnets does no policy rule talk about? (#421)""" + +import ipaddress +from pathlib import Path + +from analysis import policy_coverage as pcov +from conftest import needs_batfish + +FIXTURES = Path(__file__).parent / "fixtures" + + +def rule(n, queries, node="rtr-hq"): + return {"number": n, "node": node, "kind": "prohibition", "filter": "acl_in", "queries": queries} + + +def iface(name, subnet): + return {"interface": name, "subnet": ipaddress.ip_network(subnet), "active": True} + + +NET = {"rtr-hq": [iface("Gi0/0", "10.10.10.0/24"), iface("Gi0/1", "10.1.1.0/30")]} + + +def covered(rows): + return {r["interface"]: r["covered"] for r in rows} + + +def test_a_subnet_a_rule_names_is_covered_and_one_it_does_not_is_not(): + rows = pcov.coverage_map([rule(1, [{"srcIps": "10.10.10.0/24", "dstIps": "10.20.0.5"}])], NET) + assert covered(rows) == {"Gi0/0": True, "Gi0/1": False} + assert rows[0]["rules"] == [1] + + +def test_a_destination_counts_as_much_as_a_source(): + rows = pcov.coverage_map([rule(1, [{"srcIps": "192.168.0.0/16", "dstIps": "10.1.1.0/30"}])], NET) + assert covered(rows) == {"Gi0/0": False, "Gi0/1": True} + + +def test_a_query_with_no_address_constraint_covers_everything(): + rows = pcov.coverage_map([rule(1, [{"dstPorts": "23"}])], NET) + assert set(covered(rows).values()) == {True} + + +def test_a_rule_about_a_device_not_in_the_snapshot_covers_nothing(): + """It never runs (PC-050), so its addresses cannot count.""" + rows = pcov.coverage_map([rule(1, [{"srcIps": "10.10.10.0/24", "dstIps": "10.20.0.5"}], node="rtr-us5")], NET) + assert set(covered(rows).values()) == {False} + + +def test_syntax_we_cannot_parse_is_unknown_not_uncovered(): + rows = pcov.coverage_map([rule(1, [{"srcIps": "0.0.0.0/0 \\ (1.1.1.1)", "dstIps": "10.20.0.5"}])], NET) + assert set(covered(rows).values()) == {None} + assert rows[0]["unsure"] == [1] + + +def test_a_definite_rule_wins_over_an_unsure_one(): + rows = pcov.coverage_map([rule(1, [{"srcIps": "0.0.0.0/0 \\ (1.1.1.1)", "dstIps": "10.20.0.5"}]), + rule(2, [{"srcIps": "10.10.10.0/24", "dstIps": "10.20.0.5"}])], NET) + assert covered(rows)["Gi0/0"] is True + + +def test_a_device_whose_interfaces_cannot_be_read_is_unknown_never_covered(): + rows = pcov.coverage_map([rule(1, [{"dstPorts": "23"}])], {}, devices={"rtr-hq"}) + assert len(rows) == 1 and rows[0]["covered"] is None and rows[0]["reason"] + + +def test_interfaces_unreadable_altogether_is_unknown(): + rows = pcov.coverage_map([rule(1, [{"dstPorts": "23"}])], None, devices={"rtr-hq", "rtr-b"}) + assert {r["covered"] for r in rows} == {None} + + +def test_the_summary_counts_unknowns_separately(): + rows = pcov.coverage_map([rule(1, [{"srcIps": "10.10.10.0/24", "dstIps": "10.20.0.5"}])], NET, devices={"rtr-hq", "rtr-x"}) + assert pcov.summary(rows) == "1 of 2 interface subnet(s) are covered by a policy rule; 1 could not be decided" + + +@needs_batfish +def test_live_routing_fixture_with_a_policy_about_the_hq_lan(tmp_path, capsys): + p = tmp_path / "p.json" + p.write_text('{"device": "rtr-hq", "policy_compliance": [{"number": 1, "description": "d", ' + '"kind": "prohibition", "filter": "acl_in", "violation_severity": "high", ' + '"violation_summary": "s", "queries": [{"srcIps": "10.10.10.0/24", ' + '"dstIps": "10.20.20.0/24", "ipProtocols": ["tcp"], "dstPorts": "23"}]}]}') + assert pcov.main([str(FIXTURES / "routing-secure"), str(p)]) == 0 + out = capsys.readouterr().out + assert "2 of 4 interface subnet(s) are covered" in out + assert out.count("UNCOVERED") == 2, "the two point-to-point links"