Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
160 changes: 160 additions & 0 deletions analysis/policy_coverage.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,160 @@
"""Which parts of my network does no policy rule talk about? (#421)

WHY THIS EXISTS
PC-049 (#228/#229, #366) already reports devices no rule covers. That is
too coarse: a router can be "covered" by one rule about one subnet while
three of its four interfaces carry traffic no rule mentions. This goes
one level down, to each interface and the subnet on it.

WHAT "COVERED" MEANS HERE
A subnet is covered when some policy_compliance rule's source or
destination overlaps it -- the rule says something about traffic from or
to that subnet. A query with no address constraint talks about all
traffic, so it covers every subnet.

Only rules about a device that IS in the snapshot count. A rule about a
missing device never runs (PC-050), so it covers nothing, however its
addresses read.

THREE ANSWERS, AS EVERYWHERE ELSE IN THIS POLICY CODE
covered some rule provably overlaps the subnet
uncovered no rule does
unknown no rule provably does, and at least one uses address syntax
analysis/policy_conflicts does not parse -- so it might

And a device whose interfaces Batfish cannot list is UNKNOWN, never
covered. An uncovered subnet is coverage information, not a violation:
nothing here says the network is wrong, only where the policy is silent.

Drawing the map is the dashboard's job; this returns the data.
"""

from __future__ import annotations

import ipaddress
from typing import Any, Dict, List, Optional, Sequence, Set

from analysis.policy_conflicts import _Undecidable, _networks


def interfaces(bf: Any) -> Optional[Dict[str, List[Dict[str, Any]]]]:
"""{device: [{interface, subnet, active}]} from Batfish, or None if unreadable.

Subnets are the NETWORK of each interface address (10.10.10.1/24 ->
10.10.10.0/24), because that is what a rule's addresses are compared with.
"""
frame = bf.q.interfaceProperties(properties="All_Prefixes,Active").answer().frame()
out: Dict[str, List[Dict[str, Any]]] = {}
for _, row in frame.iterrows():
iface = row["Interface"]
for prefix in row["All_Prefixes"] or []:
out.setdefault(str(iface.hostname), []).append({
"interface": str(iface.interface),
"subnet": ipaddress.ip_interface(str(prefix)).network,
"active": bool(row["Active"]),
})
return out


def _touches(rule: Dict[str, Any], subnet: Any) -> Optional[bool]:
"""Does this rule's source or destination overlap `subnet`? True/False/None."""
unsure = False
for query in rule.get("queries", []) or []:
if not isinstance(query, dict):
unsure = True
continue
if "srcIps" not in query or "dstIps" not in query:
return True # an unconstrained side talks about everything
for field in ("srcIps", "dstIps"):
try:
if any(net.overlaps(subnet) for net in _networks(field, query[field])):
return True
except _Undecidable:
unsure = True
return None if unsure else False


def coverage_map(rules: Sequence[Dict[str, Any]],
by_device: Optional[Dict[str, List[Dict[str, Any]]]],
devices: Optional[Set[str]] = None) -> List[Dict[str, Any]]:
"""One row per interface subnet, saying whether any rule talks about it.

`devices` is every device in the snapshot. A device in it whose interfaces
could not be read gets one row with `covered` None -- unknown, not covered.
"""
present = set(devices or []) | set(by_device or {})
live_rules = [r for r in rules if r.get("node") in present]
rows: List[Dict[str, Any]] = []
for device in sorted(present):
listed = (by_device or {}).get(device)
if not listed:
rows.append({"device": device, "interface": None, "subnet": None, "active": None,
"covered": None, "rules": [], "unsure": [],
"reason": "Batfish listed no addressed interfaces for this device"})
continue
for entry in listed:
sure, unsure = [], []
for rule in live_rules:
verdict = _touches(rule, entry["subnet"])
if verdict is True:
sure.append(rule.get("number"))
elif verdict is None:
unsure.append(rule.get("number"))
rows.append({"device": device, "interface": entry["interface"],
"subnet": str(entry["subnet"]), "active": entry["active"],
"covered": True if sure else (None if unsure else False),
"rules": sure, "unsure": unsure, "reason": ""})
return rows


def summary(rows: Sequence[Dict[str, Any]]) -> str:
known = [r for r in rows if r["subnet"]]
covered = sum(1 for r in known if r["covered"] is True)
unknown = sum(1 for r in rows if r["covered"] is None)
text = f"{covered} of {len(known)} interface subnet(s) are covered by a policy rule"
if unknown:
text += f"; {unknown} could not be decided"
return text


def main(argv: Optional[Sequence[str]] = None) -> int:
"""`python -m analysis.policy_coverage <config-dir> [policy.json]`"""
import sys
from pathlib import Path

from analysis import snapshot
from analysis.checks import policy_compliance as pc
from analysis.pipeline import connect, load_snapshot
from analysis.policy import PolicyError, load_policy_file

args = list(sys.argv[1:] if argv is None else argv)
if len(args) not in (1, 2):
print("usage: python -m analysis.policy_coverage <config-dir> [policy.json]")
return 2
rules = pc.POLICY_RULES
if len(args) == 2:
try:
rules = load_policy_file(args[1]).entries_for("policy_compliance")
except PolicyError as error:
print(f"{args[1]}: not loaded -- {error}")
return 2
bf = connect(None)
load_snapshot(bf, Path(args[0]), "netwise-coverage", "snapshot")
try:
by_device = interfaces(bf)
except Exception as error: # noqa: BLE001 -- said, not swallowed
print(f"could not list interfaces, so coverage is unknown: {error}")
by_device = None
rows = coverage_map(rules, by_device, snapshot.device_names(bf))
label = {True: "covered", False: "UNCOVERED", None: "unknown"}
for r in rows:
who = (f"rule(s) {', '.join(map(str, r['rules']))}" if r["rules"]
else (f"maybe rule(s) {', '.join(map(str, r['unsure']))}" if r["unsure"] else r["reason"]))
print(f" {r['device']:<12} {str(r['interface'] or '-'):<22} {str(r['subnet'] or '-'):<18} "
f"{label[r['covered']]:<10} {who}")
print(summary(rows))
return 0


if __name__ == "__main__":
raise SystemExit(main())
22 changes: 22 additions & 0 deletions docs/policy-rules.md
Original file line number Diff line number Diff line change
Expand Up @@ -508,6 +508,28 @@ Rules are paired by `number`, and the command line warns when a file relies
on numbers the loader assigned, because inserting one rule shifts every
number after it.

### Which subnets does no rule talk about? (#421)

PC-049 reports devices no rule covers. `analysis/policy_coverage.py` goes one
level down: for every interface Batfish reports, is there a rule whose source
or destination overlaps its subnet?

```
$ python -m analysis.policy_coverage tests/fixtures/routing-secure my-policy.json
rtr-branch GigabitEthernet0/0 10.1.1.0/30 UNCOVERED
rtr-branch GigabitEthernet0/1 10.20.20.0/24 covered rule(s) 1
rtr-hq GigabitEthernet0/0 10.10.10.0/24 covered rule(s) 1
rtr-hq GigabitEthernet0/1 10.1.1.0/30 UNCOVERED
2 of 4 interface subnet(s) are covered by a policy rule
```

Only rules about a device that is in the snapshot count — a rule about a
missing device never runs (PC-050), so it covers nothing. Address syntax this
code cannot parse makes a subnet **unknown**, not uncovered, and a device
whose interfaces cannot be listed is unknown, never covered. An uncovered
subnet is coverage information, not a violation. Drawing it is the
dashboard's job; `coverage_map()` returns the rows.


## 4. Finding IDs

Expand Down
86 changes: 86 additions & 0 deletions tests/test_policy_coverage.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
"""Which interfaces and subnets does no policy rule talk about? (#421)"""

import ipaddress
from pathlib import Path

from analysis import policy_coverage as pcov
from conftest import needs_batfish

FIXTURES = Path(__file__).parent / "fixtures"


def rule(n, queries, node="rtr-hq"):
return {"number": n, "node": node, "kind": "prohibition", "filter": "acl_in", "queries": queries}


def iface(name, subnet):
return {"interface": name, "subnet": ipaddress.ip_network(subnet), "active": True}


NET = {"rtr-hq": [iface("Gi0/0", "10.10.10.0/24"), iface("Gi0/1", "10.1.1.0/30")]}


def covered(rows):
return {r["interface"]: r["covered"] for r in rows}


def test_a_subnet_a_rule_names_is_covered_and_one_it_does_not_is_not():
rows = pcov.coverage_map([rule(1, [{"srcIps": "10.10.10.0/24", "dstIps": "10.20.0.5"}])], NET)
assert covered(rows) == {"Gi0/0": True, "Gi0/1": False}
assert rows[0]["rules"] == [1]


def test_a_destination_counts_as_much_as_a_source():
rows = pcov.coverage_map([rule(1, [{"srcIps": "192.168.0.0/16", "dstIps": "10.1.1.0/30"}])], NET)
assert covered(rows) == {"Gi0/0": False, "Gi0/1": True}


def test_a_query_with_no_address_constraint_covers_everything():
rows = pcov.coverage_map([rule(1, [{"dstPorts": "23"}])], NET)
assert set(covered(rows).values()) == {True}


def test_a_rule_about_a_device_not_in_the_snapshot_covers_nothing():
"""It never runs (PC-050), so its addresses cannot count."""
rows = pcov.coverage_map([rule(1, [{"srcIps": "10.10.10.0/24", "dstIps": "10.20.0.5"}], node="rtr-us5")], NET)
assert set(covered(rows).values()) == {False}


def test_syntax_we_cannot_parse_is_unknown_not_uncovered():
rows = pcov.coverage_map([rule(1, [{"srcIps": "0.0.0.0/0 \\ (1.1.1.1)", "dstIps": "10.20.0.5"}])], NET)
assert set(covered(rows).values()) == {None}
assert rows[0]["unsure"] == [1]


def test_a_definite_rule_wins_over_an_unsure_one():
rows = pcov.coverage_map([rule(1, [{"srcIps": "0.0.0.0/0 \\ (1.1.1.1)", "dstIps": "10.20.0.5"}]),
rule(2, [{"srcIps": "10.10.10.0/24", "dstIps": "10.20.0.5"}])], NET)
assert covered(rows)["Gi0/0"] is True


def test_a_device_whose_interfaces_cannot_be_read_is_unknown_never_covered():
rows = pcov.coverage_map([rule(1, [{"dstPorts": "23"}])], {}, devices={"rtr-hq"})
assert len(rows) == 1 and rows[0]["covered"] is None and rows[0]["reason"]


def test_interfaces_unreadable_altogether_is_unknown():
rows = pcov.coverage_map([rule(1, [{"dstPorts": "23"}])], None, devices={"rtr-hq", "rtr-b"})
assert {r["covered"] for r in rows} == {None}


def test_the_summary_counts_unknowns_separately():
rows = pcov.coverage_map([rule(1, [{"srcIps": "10.10.10.0/24", "dstIps": "10.20.0.5"}])], NET, devices={"rtr-hq", "rtr-x"})
assert pcov.summary(rows) == "1 of 2 interface subnet(s) are covered by a policy rule; 1 could not be decided"


@needs_batfish
def test_live_routing_fixture_with_a_policy_about_the_hq_lan(tmp_path, capsys):
p = tmp_path / "p.json"
p.write_text('{"device": "rtr-hq", "policy_compliance": [{"number": 1, "description": "d", '
'"kind": "prohibition", "filter": "acl_in", "violation_severity": "high", '
'"violation_summary": "s", "queries": [{"srcIps": "10.10.10.0/24", '
'"dstIps": "10.20.20.0/24", "ipProtocols": ["tcp"], "dstPorts": "23"}]}]}')
assert pcov.main([str(FIXTURES / "routing-secure"), str(p)]) == 0
out = capsys.readouterr().out
assert "2 of 4 interface subnet(s) are covered" in out
assert out.count("UNCOVERED") == 2, "the two point-to-point links"
Loading