From ab9a7841fd6e00c542d59ce734509f71ee972bb4 Mon Sep 17 00:00:00 2001 From: Shubham Date: Mon, 12 Oct 2026 13:32:26 +1300 Subject: [PATCH 1/2] Refuse a policy that forbids what it requires (closes #322) US-30. A `prohibition` says traffic must be DENIED; a `requirement` says it must be PERMITTED. Two such rules over overlapping traffic on the same device and filter cannot both be satisfied -- whichever way the network is configured, one reports a violation, and fixing it creates the other. Before this, that policy loaded cleanly and the scan ran. The user got a real finding, fixed it, and got the opposite finding. Nothing said the rules were at fault. `load_policy()` now refuses it, naming both rules and the intersection: PolicyError: this policy contradicts itself, so no configuration could satisfy it: rule 3 ('The LAN must not reach the finance server') forbids what rule 7 ('Accounts must reach the finance server') requires on rtr-us5 filter acl_in: both describe src 10.10.10.64/26, dst 10.20.0.5/32, dst ports 443 The overlap is in the message because US-30 asks for it: naming only the pair leaves the user diffing two header specifiers by eye. Pure logic over the policy file. No Batfish, no Docker, no config. PROVEN, NOT PRESUMED -- AND THE THIRD ANSWER Batfish header specifiers are a language, not a list: `dstIps` can read `0.0.0.0/0 \ (218.8.104.58, 10.20.0.5)` and `ipProtocols` can read `!tcp`. analysis/policy_conflicts.py parses a deliberate subset and refuses to guess at the rest, so every comparison has three answers: overlap proven to share traffic -> policy refused no overlap proven disjoint -> loads undecidable outside the parsed subset -> loads, and is REPORTED The third one is the point. Undecidable pairs come out on `Policy.undecidable` and the pipeline prints them as `warning:`, never drops them. Treating "we could not compare these" as "these are fine" would be F-4 arriving through the policy file. An unmodelled query field makes a pair undecidable rather than being skipped. That direction is deliberate: skipping a field widens both header spaces and could invent an overlap, refusing a policy that is consistent. The shipped docs/examples/policy.example.json demonstrates it -- it uses both set difference and a negated protocol, so it loads and reports two pairs it could not compare, rather than a clean bill of health it has not earned. MUTATION-TESTED RATHER THAN ASSUMED Each of the five mutations that matter fails the new tests: undecidable collapses to "no overlap" 9 failed unmodelled query fields ignored 2 failed node/filter scope check dropped 2 failed two prohibitions counted as a conflict 8 failed overlap detail dropped from the message 1 failed Scoped to `policy_compliance`, because `kind` is its key and it is the only section whose rules can disagree about the same traffic this way. `contradictions()` takes entries and knows nothing about the section, so another section gaining that shape is a call, not a rewrite. 1848 passed, 42 skipped. ruff clean (the 8 pre-existing errors in docs/document-builds/ are unchanged and also present on main). --- analysis/pipeline.py | 5 + analysis/policy.py | 39 +++- analysis/policy_conflicts.py | 293 ++++++++++++++++++++++++++++ docs/policy-rules.md | 77 ++++++++ tests/test_policy_contradictions.py | 247 +++++++++++++++++++++++ 5 files changed, 660 insertions(+), 1 deletion(-) create mode 100644 analysis/policy_conflicts.py create mode 100644 tests/test_policy_contradictions.py diff --git a/analysis/pipeline.py b/analysis/pipeline.py index 80d6c65..17cc967 100644 --- a/analysis/pipeline.py +++ b/analysis/pipeline.py @@ -879,6 +879,11 @@ def main() -> None: # printing only half of what we changed would be exactly that. for note in list(user_policy.renamed) + list(user_policy.assigned): print(f"note: {note}") + # A pair we could not compare is not a pair we cleared (#322). It is + # louder than a note because an undetected contradiction makes every + # finding from those two rules suspect. + for note in user_policy.undecidable: + print(f"warning: {note}") config_dir = argv[0] requested = argv[1:] or None diff --git a/analysis/policy.py b/analysis/policy.py index 40285f7..50d15bd 100644 --- a/analysis/policy.py +++ b/analysis/policy.py @@ -65,6 +65,8 @@ from pathlib import Path from typing import Any, Dict, List, Mapping, Optional, Tuple +from analysis import policy_conflicts + from analysis.findings import PREFIX_BY_CHECK #: The checks a policy file may configure. `risk` and `change_impact` are @@ -231,6 +233,7 @@ def __init__( sections: Dict[str, List[Dict[str, Any]]], renamed: Optional[List[str]] = None, assigned: Optional[List[str]] = None, + undecidable: Optional[List[str]] = None, ) -> None: self.sections = sections #: Legacy key names that were corrected, as human-readable notes. @@ -242,6 +245,13 @@ def __init__( #: this module forbids SILENT defaults, not defaults. A user whose #: finding ids were chosen for them is entitled to know (#181). self.assigned = assigned or [] + #: Rule pairs that MIGHT contradict each other and could not be + #: decided, because a header specifier is outside the subset + #: `policy_conflicts` parses (#322). Same contract as the two above: + #: silence here would mean "we checked and found nothing" standing + #: in for "we could not check", which is F-4 arriving through the + #: policy file. A contradiction we can PROVE raises instead. + self.undecidable = undecidable or [] @property def is_empty(self) -> bool: @@ -526,6 +536,30 @@ def _check_numbers(sections: Dict[str, List[Dict[str, Any]]]) -> None: raise PolicyError("; ".join(problems)) +def _check_contradictions(sections: Dict[str, List[Dict[str, Any]]]) -> List[str]: + """Refuse a policy that forbids what it requires (#322, US-30). + + A `prohibition` and a `requirement` over overlapping traffic on the same + device and filter cannot both be satisfied, so a scan against them + produces a violation whichever way the network is configured, and fixing + one creates the other. The rules are at fault, and the scan is the wrong + place to find that out. + + Raises on anything PROVEN to contradict. Returns the pairs that could not + be decided, for the caller to carry on `Policy.undecidable` -- see + `policy_conflicts` for why those are never silently treated as fine. + """ + conflicts, undecided = policy_conflicts.contradictions( + sections.get("policy_compliance", []) + ) + if conflicts: + raise PolicyError( + "this policy contradicts itself, so no configuration could satisfy " + "it: " + "; ".join(conflicts) + ) + return undecided + + def _assign_missing_numbers( sections: Dict[str, List[Dict[str, Any]]] ) -> List[str]: @@ -648,7 +682,10 @@ def load_policy(data: Any) -> Policy: _check_numbers(sections) assigned = _assign_missing_numbers(sections) - return Policy(sections, renamed, assigned) + # Last, because it reasons about numbers and nodes that the passes above + # fill in: an entry whose number was assigned here is still named by it. + undecidable = _check_contradictions(sections) + return Policy(sections, renamed, assigned, undecidable) def load_policy_file(path: Any) -> Policy: diff --git a/analysis/policy_conflicts.py b/analysis/policy_conflicts.py new file mode 100644 index 0000000..4a586fa --- /dev/null +++ b/analysis/policy_conflicts.py @@ -0,0 +1,293 @@ +"""Does the user's own policy contradict itself? (#322, US-30) + +WHY THIS EXISTS + A `prohibition` says traffic must be DENIED. A `requirement` says traffic + must be PERMITTED. If two such rules describe overlapping traffic on the + same device and the same filter, **no configuration can satisfy both** -- + whatever the network does, one of them reports a violation. + + Before this, that policy loaded cleanly and the scan ran. The user got a + finding, it was real, and fixing it created the other one. The rules were + at fault and nothing said so. + + This is #240's problem D, and the only one in that list needing no network: + it is a property of the policy file alone. No Batfish, no config, no + Docker. + +WHAT COUNTS AS A CONTRADICTION + One `prohibition` and one `requirement`, same `node`, same `filter`, whose + header spaces intersect. Two prohibitions that overlap are consistent -- + both deny, and denying twice is denying. Two requirements likewise. + + A rule's `queries` are a union: `policy_compliance` fails the rule if ANY + query fails, so the rule covers every query it lists. So a contradiction + is any query of the prohibition meeting any query of the requirement. + +THE DESIGN DECISION THAT MATTERS: PROVEN, NOT PRESUMED + Batfish header specifiers are a language, not a list. `dstIps` can read + `0.0.0.0/0 \\ (218.8.104.58, 10.20.0.5)`; `ipProtocols` can read `!tcp`. + This module does not implement that language and must not pretend to. + + So every field parses to one of three answers -- a set, "matches + everything" (the field was absent), or **undecidable**. A pair of queries + is only a contradiction when every field is decidable and every field + intersects. If any field is undecidable the pair is reported as + undecidable and is NOT called a contradiction. + + Undecidable is not the same as safe, and this module never lets it be + silently treated as such -- `load_policy` carries those pairs out on + `Policy.undecidable`, which the pipeline reports the same way it reports + renamed keys. Deciding we cannot tell, and saying nothing, is F-4 arriving + through the policy file: "we checked and found nothing" standing in for + "we could not check". + +SCOPE + `policy_compliance` only, because `kind` is a `policy_compliance` key -- + it is the only section whose rules can disagree about the same traffic in + this way. `routing` expresses expectations per route rather than per + header space, and `access_control` statements are not paired as + permit/deny over a shared space. If either gains that shape, the entry + point is `contradictions()`, which takes entries and knows nothing about + which section produced them. +""" + +from __future__ import annotations + +import ipaddress +import re +from typing import Any, Dict, List, Optional, Sequence, Set, Tuple + +#: The header fields this module can reason about. A query key outside this +#: set makes the query undecidable rather than ignored: a field we do not +#: understand may be the one that keeps two rules apart, and treating it as +#: absent would widen the space and invent an overlap. +KNOWN_FIELDS = ("srcIps", "dstIps", "srcPorts", "dstPorts", "ipProtocols") + +#: Characters that put a specifier outside the plain-list subset parsed here. +#: `\` is set difference, `!` negation, `(`/`)` grouping, `@` and `/` with +#: letters are named specifiers like `@enter(...)` or `ref.Interface(...)`. +_COMPLEX = re.compile(r"[\\!()@]|\bref\.", re.IGNORECASE) + +ANY = "ANY" #: the field was absent, so it constrains nothing + + +class _Undecidable(Exception): + """Raised internally when a specifier is outside the parsed subset.""" + + def __init__(self, field: str, value: Any, why: str) -> None: + super().__init__(f"{field}={value!r} ({why})") + self.field = field + self.value = value + self.why = why + + +def _networks(field: str, spec: Any) -> List[ipaddress.IPv4Network]: + """A comma-separated list of addresses or CIDRs, as networks.""" + if not isinstance(spec, str): + raise _Undecidable(field, spec, "not a string") + if _COMPLEX.search(spec): + raise _Undecidable(field, spec, "uses Batfish specifier syntax this check does not parse") + out: List[ipaddress.IPv4Network] = [] + for part in (p.strip() for p in spec.split(",")): + if not part: + continue + try: + out.append(ipaddress.ip_network(part, strict=False)) + except ValueError: + raise _Undecidable(field, spec, f"{part!r} is not an address or CIDR") + if not out: + raise _Undecidable(field, spec, "empty") + return out + + +def _ports(field: str, spec: Any) -> List[Tuple[int, int]]: + """`"443"`, `"0-442,444-65535"` -> inclusive integer ranges.""" + if isinstance(spec, int): + return [(spec, spec)] + if not isinstance(spec, str): + raise _Undecidable(field, spec, "not a string or integer") + if _COMPLEX.search(spec): + raise _Undecidable(field, spec, "uses Batfish specifier syntax this check does not parse") + out: List[Tuple[int, int]] = [] + for part in (p.strip() for p in spec.split(",")): + if not part: + continue + bounds = part.split("-") + try: + if len(bounds) == 1: + lo = hi = int(bounds[0]) + elif len(bounds) == 2: + lo, hi = int(bounds[0]), int(bounds[1]) + else: + raise ValueError + except ValueError: + raise _Undecidable(field, spec, f"{part!r} is not a port or port range") + if not (0 <= lo <= hi <= 65535): + raise _Undecidable(field, spec, f"{part!r} is not a port range from 0 to 65535") + out.append((lo, hi)) + if not out: + raise _Undecidable(field, spec, "empty") + return out + + +def _protocols(field: str, spec: Any) -> Set[str]: + if isinstance(spec, str): + spec = [spec] + if not isinstance(spec, (list, tuple)): + raise _Undecidable(field, spec, "not a list") + out = set() + for item in spec: + text = str(item).strip().lower() + if not text or _COMPLEX.search(text): + raise _Undecidable(field, spec, f"{item!r} is negated or not a plain protocol name") + out.add(text) + if not out: + raise _Undecidable(field, spec, "empty") + return out + + +def _parse(query: Any) -> Dict[str, Any]: + """One query to {field: parsed value or ANY}. Raises _Undecidable.""" + if not isinstance(query, dict): + raise _Undecidable("query", query, "not an object") + unknown = sorted(set(query) - set(KNOWN_FIELDS)) + if unknown: + raise _Undecidable("query", query, f"has field(s) this check does not model: {', '.join(unknown)}") + parsed: Dict[str, Any] = {} + for field in KNOWN_FIELDS: + if field not in query: + parsed[field] = ANY + elif field in ("srcIps", "dstIps"): + parsed[field] = _networks(field, query[field]) + elif field in ("srcPorts", "dstPorts"): + parsed[field] = _ports(field, query[field]) + else: + parsed[field] = _protocols(field, query[field]) + return parsed + + +def _nets_meet(a: Sequence[ipaddress.IPv4Network], + b: Sequence[ipaddress.IPv4Network]) -> List[ipaddress.IPv4Network]: + """Shared addresses. Two CIDR blocks are disjoint or one holds the other, + so the intersection of a pair is simply the more specific of the two.""" + shared = [] + for x in a: + for y in b: + if x.overlaps(y): + shared.append(x if x.prefixlen >= y.prefixlen else y) + return shared + + +def _ranges_meet(a: Sequence[Tuple[int, int]], + b: Sequence[Tuple[int, int]]) -> List[Tuple[int, int]]: + shared = [] + for lo1, hi1 in a: + for lo2, hi2 in b: + lo, hi = max(lo1, lo2), min(hi1, hi2) + if lo <= hi: + shared.append((lo, hi)) + return shared + + +def _field_meet(field: str, x: Any, y: Any) -> Tuple[bool, Optional[str]]: + """Do two parsed constraints share anything? -> (yes, description).""" + if x is ANY and y is ANY: + return True, None # both unconstrained: no detail worth printing + if x is ANY: + return True, _describe(field, y) + if y is ANY: + return True, _describe(field, x) + if field in ("srcIps", "dstIps"): + shared = _nets_meet(x, y) + elif field in ("srcPorts", "dstPorts"): + shared = _ranges_meet(x, y) + else: + shared = sorted(x & y) + return (bool(shared), _describe(field, shared) if shared else None) + + +_LABEL = {"srcIps": "src", "dstIps": "dst", "srcPorts": "src ports", + "dstPorts": "dst ports", "ipProtocols": "protocol"} + + +def _describe(field: str, value: Any) -> str: + if field in ("srcPorts", "dstPorts"): + text = ",".join(str(lo) if lo == hi else f"{lo}-{hi}" for lo, hi in value) + else: + text = ",".join(str(v) for v in value) + return f"{_LABEL[field]} {text}" + + +def overlap(query_a: Any, query_b: Any) -> Tuple[Optional[bool], str]: + """Do two queries describe any traffic in common? + + Returns `(True, "src ..., dst ...")` when they provably do, `(False, "")` + when they provably do not, and `(None, reason)` when a specifier is + outside the subset this module parses. Never guesses. + """ + try: + a, b = _parse(query_a), _parse(query_b) + except _Undecidable as exc: + return None, str(exc) + parts = [] + for field in KNOWN_FIELDS: + meets, detail = _field_meet(field, a[field], b[field]) + if not meets: + return False, "" + if detail: + parts.append(detail) + return True, ", ".join(parts) if parts else "all traffic" + + +def _scope(entry: Any) -> Tuple[str, str]: + return (str(entry.get("node", "")), str(entry.get("filter", ""))) + + +def _ref(entry: Any) -> str: + number = entry.get("number") + label = f"rule {number}" if number is not None else "an unnumbered rule" + description = entry.get("description") + return f"{label} ({description!r})" if description else label + + +def contradictions(entries: Sequence[Any]) -> Tuple[List[str], List[str]]: + """Find prohibition/requirement pairs that cannot both hold. + + Returns `(contradictions, undecidable)`, both lists of human-readable + sentences naming each pair. A caller that refuses the policy should use + the first; it must not treat the second as nothing -- see the module + docstring. + """ + prohibitions = [e for e in entries if isinstance(e, dict) and e.get("kind") == "prohibition"] + requirements = [e for e in entries if isinstance(e, dict) and e.get("kind") == "requirement"] + + conflicts: List[str] = [] + undecided: List[str] = [] + for forbid in prohibitions: + for require in requirements: + if _scope(forbid) != _scope(require): + continue # different device or filter: cannot collide + node, filter_name = _scope(forbid) + where = f"on {node} filter {filter_name}" if node and filter_name else "" + unknown_here: List[str] = [] + found = False + for qa in forbid.get("queries", []) or []: + for qb in require.get("queries", []) or []: + verdict, detail = overlap(qa, qb) + if verdict is True: + conflicts.append( + f"{_ref(forbid)} forbids what {_ref(require)} requires {where}" + f": both describe {detail}".replace(" ", " ") + ) + found = True + break + if verdict is None: + unknown_here.append(detail) + if found: + break + if not found and unknown_here: + undecided.append( + f"{_ref(forbid)} and {_ref(require)} {where} could not be compared" + f": {unknown_here[0]}".replace(" ", " ") + ) + return conflicts, undecided diff --git a/docs/policy-rules.md b/docs/policy-rules.md index 43ec36e..7feabd9 100644 --- a/docs/policy-rules.md +++ b/docs/policy-rules.md @@ -336,6 +336,83 @@ consistently with POL-4. --- +### A policy that forbids what it requires is refused (#322, US-30) + +A `prohibition` says traffic must be DENIED. A `requirement` says traffic must +be PERMITTED. If two such rules describe overlapping traffic **on the same +device and the same filter**, no configuration can satisfy both: whichever way +the network is set up, one of them reports a violation, and fixing that one +creates the other. + +Before #322 such a policy loaded cleanly and the scan ran. The user got a real +finding, fixed it, and got the opposite finding. Nothing said the rules were at +fault. + +`analysis/policy_conflicts.py` now compares every prohibition against every +requirement in the same scope, and `load_policy()` **refuses** the policy when +it can prove an overlap: + +``` +PolicyError: this policy contradicts itself, so no configuration could +satisfy it: rule 3 ('The LAN must not reach the finance server') forbids +what rule 7 ('Accounts must reach the finance server') requires on rtr-us5 +filter acl_in: both describe src 10.10.10.64/26, dst 10.20.0.5/32, +dst ports 443 +``` + +The message names both rules and **the intersection**, not either rule +restated. US-30 asks for the overlap because a message naming only the pair +leaves the user diffing two header specifiers by eye. + +This needs no Batfish, no Docker and no config. It is a property of the policy +file alone. + +#### What is not a contradiction + +Two prohibitions over the same traffic are consistent — denying twice is +denying — and so are two requirements. Only permit-against-deny is +unsatisfiable. Rules on different devices or different filters never collide. + +A rule's `queries` are a union, because `policy_compliance` fails the rule if +any one of them fails, so a contradiction is any query of the prohibition +meeting any query of the requirement. + +#### Proven, not presumed — and the third answer + +Batfish header specifiers are a language, not a list. `dstIps` can read +`0.0.0.0/0 \ (218.8.104.58, 10.20.0.5)`; `ipProtocols` can read `!tcp`. This +module parses a deliberate subset — plain addresses, CIDRs, port ranges and +protocol names — and **refuses to guess at the rest**. + +So every comparison has three answers, not two: + +| answer | meaning | effect | +|---|---|---| +| overlap | proven to share traffic | the policy is **refused** | +| no overlap | proven disjoint | loads normally | +| **undecidable** | a specifier outside the parsed subset | loads, and is **reported** | + +The third is the one that matters. An undecidable pair is carried out on +`Policy.undecidable` and printed by the pipeline as a `warning:`, never +dropped. Treating "we could not compare these" as "these are fine" is F-4 +arriving through the policy file: *we checked and found nothing* standing in +for *we could not check*. + +An unmodelled query field makes the pair undecidable rather than being +skipped, and that direction is deliberate. Skipping a field would widen both +header spaces and could invent an overlap that is not there — refusing a +policy that is perfectly consistent. + +The shipped `docs/examples/policy.example.json` demonstrates this: it uses +both set difference and a negated protocol, so it loads and reports two pairs +it could not compare. + +`tests/test_policy_contradictions.py` pins all of it, and the five mutations +that matter — undecidable collapsing to "no overlap", unmodelled fields being +ignored, the scope check being dropped, two prohibitions counting as a +conflict, and the overlap detail falling out of the message — each fail it. + + ## 4. Finding IDs Agreed with Arsh, and requiring no change to `docs/finding-format.md`: diff --git a/tests/test_policy_contradictions.py b/tests/test_policy_contradictions.py new file mode 100644 index 0000000..df71094 --- /dev/null +++ b/tests/test_policy_contradictions.py @@ -0,0 +1,247 @@ +"""A policy that forbids what it requires must be refused (#322, US-30). + +WHAT THESE PIN + Two things, and the second is the one that will rot first. + + 1. A PROVEN contradiction raises, naming both rules and the traffic they + disagree about. Without the overlap in the message the user is told + two rules fight and left to work out where. + + 2. A pair this module CANNOT decide is never silently treated as clean. + `analysis/policy_conflicts.py` parses a deliberate subset of Batfish + header specifiers; everything else is undecidable and surfaces on + `Policy.undecidable`. A future change that widens the parser must keep + that third answer, because losing it turns "we could not check" into + "we checked and found nothing", which is F-4 reached through the + policy file. +""" + +import pytest + +from analysis import policy_conflicts +from analysis.policy import PolicyError, load_policy, load_policy_file + + +def rule(number, kind, queries, node="rtr-us5", filter_name="acl_in", description=None): + return { + "number": number, + "description": description or f"rule {number}", + "node": node, + "kind": kind, + "filter": filter_name, + "violation_severity": "high", + "violation_summary": "something was not as the policy requires", + "queries": queries, + } + + +def policy(*entries): + return {"device": "rtr-us5", "policy_compliance": list(entries)} + + +LAN_TO_SERVER = [{"srcIps": "10.10.10.0/24", "dstIps": "10.20.0.5"}] +ACCOUNTS_TO_SERVER = [{"srcIps": "10.10.10.64/26", "dstIps": "10.20.0.0/16", "dstPorts": "443"}] + + +# --- 1. the contradiction itself ---------------------------------------- + +def test_a_prohibition_that_forbids_what_a_requirement_requires_is_refused(): + with pytest.raises(PolicyError) as caught: + load_policy(policy( + rule(3, "prohibition", LAN_TO_SERVER), + rule(7, "requirement", ACCOUNTS_TO_SERVER), + )) + assert "contradicts itself" in str(caught.value) + + +def test_the_message_names_both_rule_numbers_and_the_overlap(): + """US-30 asks for the pair AND the overlap. A message naming only the + rules sends the user to diff two header specifiers by eye.""" + with pytest.raises(PolicyError) as caught: + load_policy(policy( + rule(3, "prohibition", LAN_TO_SERVER), + rule(7, "requirement", ACCOUNTS_TO_SERVER), + )) + message = str(caught.value) + assert "rule 3" in message and "rule 7" in message + # the overlap is the INTERSECTION, not either rule restated + assert "10.10.10.64/26" in message + assert "10.20.0.5/32" in message + assert "443" in message + + +def test_the_scan_is_blocked_rather_than_run_on_contradictory_rules(): + """`load_policy` is the gate every entry point goes through, so raising + here is what stops a scan producing a violation either way round.""" + with pytest.raises(PolicyError): + load_policy(policy( + rule(1, "prohibition", [{"dstIps": "10.20.0.5"}]), + rule(2, "requirement", [{"dstIps": "10.20.0.5"}]), + )) + + +# --- 2. what is NOT a contradiction -------------------------------------- + +def test_two_prohibitions_over_the_same_traffic_are_consistent(): + """Denying twice is denying. Only permit-versus-deny cannot be satisfied.""" + loaded = load_policy(policy( + rule(1, "prohibition", LAN_TO_SERVER), + rule(2, "prohibition", LAN_TO_SERVER), + )) + assert loaded.undecidable == [] + + +def test_two_requirements_over_the_same_traffic_are_consistent(): + loaded = load_policy(policy( + rule(1, "requirement", LAN_TO_SERVER), + rule(2, "requirement", LAN_TO_SERVER), + )) + assert loaded.undecidable == [] + + +def test_rules_on_different_devices_cannot_contradict(): + loaded = load_policy(policy( + rule(1, "prohibition", LAN_TO_SERVER, node="rtr-us5"), + rule(2, "requirement", LAN_TO_SERVER, node="rtr-hq"), + )) + assert len(loaded.entries_for("policy_compliance")) == 2 + assert loaded.undecidable == [] + + +def test_rules_on_different_filters_cannot_contradict(): + loaded = load_policy(policy( + rule(1, "prohibition", LAN_TO_SERVER, filter_name="acl_in"), + rule(2, "requirement", LAN_TO_SERVER, filter_name="acl_out"), + )) + assert len(loaded.entries_for("policy_compliance")) == 2 + assert loaded.undecidable == [] + + +def test_disjoint_traffic_is_not_a_contradiction(): + loaded = load_policy(policy( + rule(1, "prohibition", [{"srcIps": "10.10.10.0/24"}]), + rule(2, "requirement", [{"srcIps": "192.168.1.0/24"}]), + )) + assert loaded.undecidable == [] + assert policy_conflicts.overlap( + {"srcIps": "10.10.10.0/24"}, {"srcIps": "192.168.1.0/24"})[0] is False + + +def test_a_port_range_that_excludes_the_required_port_does_not_collide(): + """The real shape in docs/examples/policy.example.json: "everything except + 443" against "443". Treating the ranges as "both mention ports" would + refuse a policy that is perfectly consistent.""" + loaded = load_policy(policy( + rule(1, "prohibition", [{"dstIps": "10.20.0.5", "dstPorts": "0-442,444-65535"}]), + rule(2, "requirement", [{"dstIps": "10.20.0.5", "dstPorts": "443"}]), + )) + assert loaded.undecidable == [], "the ports were parsed, so nothing is undecided" + assert policy_conflicts.overlap( + {"dstPorts": "0-442,444-65535"}, {"dstPorts": "443"})[0] is False + + +def test_a_protocol_that_does_not_match_does_not_collide(): + loaded = load_policy(policy( + rule(1, "prohibition", [{"dstIps": "10.20.0.5", "ipProtocols": ["udp"]}]), + rule(2, "requirement", [{"dstIps": "10.20.0.5", "ipProtocols": ["tcp"]}]), + )) + assert loaded.undecidable == [] + + +# --- 3. undecidable is its own answer ------------------------------------ + +def test_a_specifier_we_cannot_parse_is_recorded_rather_than_cleared(): + """The whole safety argument. `\\` is Batfish set difference, which this + module does not implement. The pair MUST NOT load silently.""" + loaded = load_policy(policy( + rule(1, "prohibition", [{"dstIps": "0.0.0.0/0 \\ (10.20.0.5)"}]), + rule(2, "requirement", [{"dstIps": "10.20.0.5"}]), + )) + assert len(loaded.undecidable) == 1 + note = loaded.undecidable[0] + assert "rule 1" in note and "rule 2" in note + assert "could not be compared" in note + + +def test_a_negated_protocol_is_undecidable_rather_than_ignored(): + loaded = load_policy(policy( + rule(1, "prohibition", [{"dstIps": "10.20.0.5", "ipProtocols": ["!tcp"]}]), + rule(2, "requirement", [{"dstIps": "10.20.0.5"}]), + )) + assert loaded.undecidable, "a negated protocol must not read as no constraint" + + +def test_a_query_field_we_do_not_model_makes_the_pair_undecidable(): + """The dangerous direction. An unmodelled field might be the one keeping + two rules apart; skipping it would WIDEN each space and invent an overlap + that is not there, refusing a policy that is fine.""" + loaded = load_policy(policy( + rule(1, "prohibition", [{"dstIps": "10.20.0.5", "icmpCodes": "0"}]), + rule(2, "requirement", [{"dstIps": "10.20.0.5"}]), + )) + assert loaded.undecidable + assert "icmpCodes" in loaded.undecidable[0] + + +def test_an_undecidable_pair_does_not_block_the_scan(): + """It is a warning, not a refusal. We have not shown a contradiction, and + refusing on a maybe would make the subset this module parses into a + restriction on what a user may write.""" + loaded = load_policy(policy( + rule(1, "prohibition", [{"dstIps": "0.0.0.0/0 \\ (10.20.0.5)"}]), + rule(2, "requirement", [{"dstIps": "10.20.0.5"}]), + )) + assert loaded.entries_for("policy_compliance"), "the policy still loaded" + assert loaded.undecidable, "and the pair was recorded rather than cleared" + + +def test_a_proven_contradiction_wins_over_an_undecidable_one(): + """A rule with two queries, one undecidable and one provably colliding. + The provable one must still raise rather than be softened to a note.""" + with pytest.raises(PolicyError): + load_policy(policy( + rule(1, "prohibition", [{"dstIps": "0.0.0.0/0 \\ (1.1.1.1)"}, + {"dstIps": "10.20.0.5"}]), + rule(2, "requirement", [{"dstIps": "10.20.0.5"}]), + )) + + +# --- 4. the shipped example ---------------------------------------------- + +def test_the_example_policy_still_loads_and_reports_what_it_could_not_compare(): + """docs/examples/policy.example.json is what a user copies. It uses the + specifier syntax this module does not parse, so it must load AND say so.""" + loaded = load_policy_file("docs/examples/policy.example.json") + assert loaded.entries_for("policy_compliance") + assert loaded.undecidable, ( + "the example uses set-difference and a negated protocol; reporting " + "no undecidable pairs would mean the third answer had been lost" + ) + + +# --- 5. the overlap primitive itself ------------------------------------- + +@pytest.mark.parametrize("a,b,expected", [ + ({"srcIps": "10.10.10.0/24"}, {"srcIps": "10.10.10.7"}, True), + ({"srcIps": "10.10.10.0/24"}, {"srcIps": "192.168.1.0/24"}, False), + ({"dstPorts": "440-500"}, {"dstPorts": "443"}, True), + ({"dstPorts": "0-442,444-65535"}, {"dstPorts": "443"}, False), + ({"ipProtocols": ["tcp", "udp"]}, {"ipProtocols": ["udp"]}, True), + ({}, {"dstIps": "10.20.0.5"}, True), # absent field constrains nothing +]) +def test_overlap_decides_the_cases_it_claims_to(a, b, expected): + verdict, _ = policy_conflicts.overlap(a, b) + assert verdict is expected + + +@pytest.mark.parametrize("spec", [ + {"dstIps": "0.0.0.0/0 \\ (10.20.0.5)"}, + {"ipProtocols": ["!tcp"]}, + {"dstIps": "ref.Interface(name)"}, + {"dstPorts": "not-a-port"}, + {"unknownField": "x"}, +]) +def test_overlap_returns_undecidable_rather_than_guessing(spec): + verdict, reason = policy_conflicts.overlap(spec, {"dstIps": "10.20.0.5"}) + assert verdict is None + assert reason From 8a190a3dbec011a2737ec965737cc8b94b52afea Mon Sep 17 00:00:00 2001 From: Shubham Date: Sun, 18 Oct 2026 15:08:20 +1300 Subject: [PATCH 2/2] Address #387 review: protocols, filter scope, field-by-field, CLI warnings Every item from @ARSH871-bot's review of ab9a784, each reproduced first. 1. Protocol tokens were guessed. Anything not negated was taken as a name, so "6", "tcp,udp" and ["tcp, udp"] never met tcp/udp and a real contradiction loaded as consistent. Batfish reads "6" as TCP. Protocols now come from a closed list -- tcp, udp, icmp by name or number -- with comma lists split the way pybatfish splits them. Anything else is undecidable, never a new protocol. 2. Filter scope compared raw strings, so ACL_IN vs acl_in was "different filter: cannot collide". Batfish resolves ACL_IN, "acl_in ", /acl_in/ and @in(...) to the same ACL. Filters that match only after strip/casefold, or that are specifiers (/ @ ( ,), now make the pair undecidable rather than skipped. Node is still compared exactly: a mis-cased node never runs. 3. "Never dropped" held on the CLI only. docs/policy-rules.md now says the pipeline and change_impact CLIs print each pair and the dashboard does not yet, pointing at #404. change_impact's CLI now prints them too, and two tests run both CLIs on the shipped example and count the warnings -- deleting the pipeline's print used to fail nothing. 4. One unparseable field hid a disjoint parsed one. Comparison is now field by field: a query is a conjunction, so one provably disjoint field makes the pair disjoint whatever the others say. That cannot invent an overlap. The shipped example drops from two warnings to one; rule 2 vs rule 4 was false (10.20.0.5 vs 218.8.104.58). Nits: overlapping port ranges are described once; a bool is not a port; the "last, because" ordering in load_policy is pinned by a test with no numbers; the example path is anchored on __file__; the _COMPLEX comment no longer mentions "/". Mutation-tested: unknown protocol accepted, commas not split, raw filter compare, specifiers skipped, giving up at the first unparsed field, the pipeline print removed, the check moved before numbering, and a bool port each fail the tests. The "first unparsed field" mutant initially SURVIVED -- both disjoint-beats-unparsed tests put the disjoint field first in field order -- so a test with the unparsed field first now pins it. 1932 passed, 13 skipped with Batfish running. ruff clean. --- analysis/change_impact.py | 5 + analysis/policy_conflicts.py | 143 +++++++++++++++++++++------- docs/policy-rules.md | 41 ++++++-- tests/test_policy_contradictions.py | 118 ++++++++++++++++++++++- 4 files changed, 263 insertions(+), 44 deletions(-) diff --git a/analysis/change_impact.py b/analysis/change_impact.py index 7e97d9b..0f38339 100644 --- a/analysis/change_impact.py +++ b/analysis/change_impact.py @@ -660,6 +660,11 @@ def main() -> None: policy = None if len(sys.argv) == 4: policy = policy_module.load_policy_file(sys.argv[3]) + # Same contract as `python -m analysis.pipeline --policy` (#387 + # review): a pair the contradiction check could not compare is said, + # not dropped. + for note in policy.undecidable: + print(f"warning: {note}") results = analyse_change(sys.argv[1], sys.argv[2], policy=policy) print(json.dumps(results, indent=2)) print() diff --git a/analysis/policy_conflicts.py b/analysis/policy_conflicts.py index 4a586fa..ac3113c 100644 --- a/analysis/policy_conflicts.py +++ b/analysis/policy_conflicts.py @@ -64,8 +64,8 @@ KNOWN_FIELDS = ("srcIps", "dstIps", "srcPorts", "dstPorts", "ipProtocols") #: Characters that put a specifier outside the plain-list subset parsed here. -#: `\` is set difference, `!` negation, `(`/`)` grouping, `@` and `/` with -#: letters are named specifiers like `@enter(...)` or `ref.Interface(...)`. +#: `\` is set difference, `!` negation, `(`/`)` grouping, and `@` or `ref.` +#: start a named specifier like `@enter(...)` or `ref.Interface(...)`. _COMPLEX = re.compile(r"[\\!()@]|\bref\.", re.IGNORECASE) ANY = "ANY" #: the field was absent, so it constrains nothing @@ -102,6 +102,8 @@ def _networks(field: str, spec: Any) -> List[ipaddress.IPv4Network]: def _ports(field: str, spec: Any) -> List[Tuple[int, int]]: """`"443"`, `"0-442,444-65535"` -> inclusive integer ranges.""" + if isinstance(spec, bool): + raise _Undecidable(field, spec, "a boolean is not a port") if isinstance(spec, int): return [(spec, spec)] if not isinstance(spec, str): @@ -130,6 +132,15 @@ def _ports(field: str, spec: Any) -> List[Tuple[int, int]]: return out +#: Protocol tokens this module understands, by name and by IANA number. +#: A CLOSED list (#387 review, @ARSH871-bot): anything else is undecidable. +#: Taking every unrecognised token as a protocol name was a guess, and it +#: guessed wrong in the reassuring direction -- `"6"` never met `"tcp"`, so a +#: real contradiction loaded as consistent. Batfish reads `"6"` as TCP. +_PROTOCOLS = {"tcp": "tcp", "udp": "udp", "icmp": "icmp", + "6": "tcp", "17": "udp", "1": "icmp"} + + def _protocols(field: str, spec: Any) -> Set[str]: if isinstance(spec, str): spec = [spec] @@ -137,33 +148,55 @@ def _protocols(field: str, spec: Any) -> Set[str]: raise _Undecidable(field, spec, "not a list") out = set() for item in spec: - text = str(item).strip().lower() - if not text or _COMPLEX.search(text): - raise _Undecidable(field, spec, f"{item!r} is negated or not a plain protocol name") - out.add(text) + # pybatfish splits "tcp,udp" into two protocols, so this does too. + for token in str(item).split(","): + text = token.strip().lower() + if text not in _PROTOCOLS: + raise _Undecidable( + field, spec, + f"{token.strip()!r} is not a protocol this check recognises " + f"({', '.join(sorted(k for k in _PROTOCOLS if not k.isdigit()))}, " + "or their numbers)", + ) + out.add(_PROTOCOLS[text]) if not out: raise _Undecidable(field, spec, "empty") return out -def _parse(query: Any) -> Dict[str, Any]: - """One query to {field: parsed value or ANY}. Raises _Undecidable.""" +class _Unknown: + """One field we could not parse. Kept per field, not per query (#387).""" + + def __init__(self, reason: str) -> None: + self.reason = reason + + +def _parse_field(field: str, value: Any) -> Any: + try: + if field in ("srcIps", "dstIps"): + return _networks(field, value) + if field in ("srcPorts", "dstPorts"): + return _ports(field, value) + return _protocols(field, value) + except _Undecidable as exc: + return _Unknown(str(exc)) + + +def _parse(query: Any) -> Tuple[Dict[str, Any], List[str]]: + """One query to ({field: value, ANY, or _Unknown}, [reasons]). + + Unmodelled keys are returned as reasons rather than raised: they make a + pair undecidable, but they cannot make it OVERLAP, and a field we did + parse may still prove the two queries disjoint. + """ if not isinstance(query, dict): raise _Undecidable("query", query, "not an object") - unknown = sorted(set(query) - set(KNOWN_FIELDS)) - if unknown: - raise _Undecidable("query", query, f"has field(s) this check does not model: {', '.join(unknown)}") - parsed: Dict[str, Any] = {} - for field in KNOWN_FIELDS: - if field not in query: - parsed[field] = ANY - elif field in ("srcIps", "dstIps"): - parsed[field] = _networks(field, query[field]) - elif field in ("srcPorts", "dstPorts"): - parsed[field] = _ports(field, query[field]) - else: - parsed[field] = _protocols(field, query[field]) - return parsed + reasons = [ + f"query has field(s) this check does not model: {', '.join(sorted(set(query) - set(KNOWN_FIELDS)))}" + ] if set(query) - set(KNOWN_FIELDS) else [] + parsed = {f: (ANY if f not in query else _parse_field(f, query[f])) for f in KNOWN_FIELDS} + reasons += [v.reason for v in parsed.values() if isinstance(v, _Unknown)] + return parsed, reasons def _nets_meet(a: Sequence[ipaddress.IPv4Network], @@ -174,7 +207,9 @@ def _nets_meet(a: Sequence[ipaddress.IPv4Network], for x in a: for y in b: if x.overlaps(y): - shared.append(x if x.prefixlen >= y.prefixlen else y) + pick = x if x.prefixlen >= y.prefixlen else y + if pick not in shared: + shared.append(pick) return shared @@ -184,7 +219,7 @@ def _ranges_meet(a: Sequence[Tuple[int, int]], for lo1, hi1 in a: for lo2, hi2 in b: lo, hi = max(lo1, lo2), min(hi1, hi2) - if lo <= hi: + if lo <= hi and (lo, hi) not in shared: shared.append((lo, hi)) return shared @@ -222,25 +257,61 @@ def overlap(query_a: Any, query_b: Any) -> Tuple[Optional[bool], str]: """Do two queries describe any traffic in common? Returns `(True, "src ..., dst ...")` when they provably do, `(False, "")` - when they provably do not, and `(None, reason)` when a specifier is - outside the subset this module parses. Never guesses. + when they provably do not, and `(None, reason)` when it cannot be decided. + Never guesses. + + FIELD BY FIELD (#387 review). A query is a conjunction, so ONE field that + is provably disjoint makes the pair disjoint -- whatever the other fields + say, including ones we could not parse. That can never invent an overlap, + because a conjunct only ever shrinks a space. Only when nothing proves + them disjoint does an unparsed field make the answer undecidable. """ try: - a, b = _parse(query_a), _parse(query_b) + a, reasons_a = _parse(query_a) + b, reasons_b = _parse(query_b) except _Undecidable as exc: return None, str(exc) parts = [] for field in KNOWN_FIELDS: - meets, detail = _field_meet(field, a[field], b[field]) + x, y = a[field], b[field] + if isinstance(x, _Unknown) or isinstance(y, _Unknown): + continue + meets, detail = _field_meet(field, x, y) if not meets: return False, "" if detail: parts.append(detail) + reasons = reasons_a + reasons_b + if reasons: + return None, reasons[0] return True, ", ".join(parts) if parts else "all traffic" -def _scope(entry: Any) -> Tuple[str, str]: - return (str(entry.get("node", "")), str(entry.get("filter", ""))) +#: Characters that make a `filter` a Batfish SPECIFIER rather than a name -- +#: `/acl_in/` is a regex, `@in(Gi0/0)` an interface's filter, `a,b` a list. +#: Such a filter can name the same ACL as another rule's plain name. +_FILTER_SPECIFIER = re.compile(r"[/@(,]") + + +def _same_scope(a: Any, b: Any) -> Optional[bool]: + """Are two rules about the same device AND the same filter? + + True or False when it is clear, None when it is not. Batfish resolves + `ACL_IN`, `acl_in ` and `/acl_in/` all to `acl_in` (#387 review), so a raw + string comparison called those "different filter: cannot collide" and + skipped a real contradiction. Node is compared exactly: a mis-cased node + never runs at all, so it cannot produce a contradictory result. + """ + if str(a.get("node", "")) != str(b.get("node", "")): + return False + fa, fb = str(a.get("filter", "")), str(b.get("filter", "")) + if fa == fb: + return True + if _FILTER_SPECIFIER.search(fa) or _FILTER_SPECIFIER.search(fb): + return None + if fa.strip().casefold() == fb.strip().casefold(): + return None + return False def _ref(entry: Any) -> str: @@ -265,10 +336,18 @@ def contradictions(entries: Sequence[Any]) -> Tuple[List[str], List[str]]: undecided: List[str] = [] for forbid in prohibitions: for require in requirements: - if _scope(forbid) != _scope(require): + scope = _same_scope(forbid, require) + if scope is False: continue # different device or filter: cannot collide - node, filter_name = _scope(forbid) + node, filter_name = forbid.get("node", ""), forbid.get("filter", "") where = f"on {node} filter {filter_name}" if node and filter_name else "" + if scope is None: + undecided.append( + f"{_ref(forbid)} and {_ref(require)} name filters " + f"{forbid.get('filter')!r} and {require.get('filter')!r} on {node}, " + "which Batfish may resolve to the same ACL; could not be compared" + ) + continue unknown_here: List[str] = [] found = False for qa in forbid.get("queries", []) or []: diff --git a/docs/policy-rules.md b/docs/policy-rules.md index 5f378a5..82eb4fc 100644 --- a/docs/policy-rules.md +++ b/docs/policy-rules.md @@ -371,7 +371,16 @@ file alone. Two prohibitions over the same traffic are consistent — denying twice is denying — and so are two requirements. Only permit-against-deny is -unsatisfiable. Rules on different devices or different filters never collide. +unsatisfiable. Rules on different devices never collide, and nor do rules on +filters that are plainly different names. + +Filter names are the exception that needs care. Batfish resolves `ACL_IN`, +`acl_in ` and the regex `/acl_in/` all to `acl_in`, so two rules can name the +same ACL in different spellings. When two filters match only after trimming +and ignoring case, or either is a specifier rather than a plain name (`/`, +`@`, `(`, `,`), the pair is reported as **undecidable**, not skipped. Found in +review on #387: the first version compared the raw strings and called +`ACL_IN` versus `acl_in` "cannot collide". A rule's `queries` are a union, because `policy_compliance` fails the rule if any one of them fails, so a contradiction is any query of the prohibition @@ -381,8 +390,12 @@ meeting any query of the requirement. Batfish header specifiers are a language, not a list. `dstIps` can read `0.0.0.0/0 \ (218.8.104.58, 10.20.0.5)`; `ipProtocols` can read `!tcp`. This -module parses a deliberate subset — plain addresses, CIDRs, port ranges and -protocol names — and **refuses to guess at the rest**. +module parses a deliberate subset — plain addresses, CIDRs, port ranges, and +the protocols `tcp`, `udp` and `icmp` by name or number (`6`, `17`, `1`), with +comma-separated lists split the way pybatfish splits them — and **refuses to +guess at the rest**. An unrecognised protocol token is undecidable, not a new +protocol: the first version took `"6"` as a name, so it never met `"tcp"`, and +a real contradiction loaded as consistent. So every comparison has three answers, not two: @@ -393,19 +406,33 @@ So every comparison has three answers, not two: | **undecidable** | a specifier outside the parsed subset | loads, and is **reported** | The third is the one that matters. An undecidable pair is carried out on -`Policy.undecidable` and printed by the pipeline as a `warning:`, never -dropped. Treating "we could not compare these" as "these are fine" is F-4 +`Policy.undecidable`. Both command lines print each pair as a `warning:` — +`python -m analysis.pipeline … --policy` and `python -m analysis.change_impact +… policy.json`. **The dashboard does not show them yet**: `/api/policy` returns +renamed keys only, so an upload through the browser drops the pairs silently. +That gap is #404, and until it closes "never dropped" holds on the command +line only. Treating "we could not compare these" as "these are fine" is F-4 arriving through the policy file: *we checked and found nothing* standing in for *we could not check*. +Comparison is **field by field**. A query is a conjunction, so one field that +is provably disjoint makes the pair disjoint whatever the other fields say — +including fields this module could not parse. That can never invent an +overlap, because a conjunct only shrinks a space. Only when nothing proves the +pair disjoint does an unparsed field make it undecidable. + An unmodelled query field makes the pair undecidable rather than being skipped, and that direction is deliberate. Skipping a field would widen both header spaces and could invent an overlap that is not there — refusing a policy that is perfectly consistent. The shipped `docs/examples/policy.example.json` demonstrates this: it uses -both set difference and a negated protocol, so it loads and reports two pairs -it could not compare. +set difference, so it loads and reports **one** pair it could not compare: +rule 1 against rule 4. It used to report two. The second, rule 2 against +rule 4, was a false warning — rule 2 negates a protocol, but it is about +10.20.0.5 and rule 4 about 218.8.104.58, which is provably disjoint on a field +that *is* parsed. A false warning in our own example would teach people to +skip the real ones. `tests/test_policy_contradictions.py` pins all of it, and the five mutations that matter — undecidable collapsing to "no overlap", unmodelled fields being diff --git a/tests/test_policy_contradictions.py b/tests/test_policy_contradictions.py index df71094..2bc71c0 100644 --- a/tests/test_policy_contradictions.py +++ b/tests/test_policy_contradictions.py @@ -16,11 +16,16 @@ policy file. """ +from pathlib import Path + import pytest from analysis import policy_conflicts from analysis.policy import PolicyError, load_policy, load_policy_file +REPO = Path(__file__).resolve().parents[1] +EXAMPLE = REPO / "docs" / "examples" / "policy.example.json" + def rule(number, kind, queries, node="rtr-us5", filter_name="acl_in", description=None): return { @@ -211,12 +216,115 @@ def test_a_proven_contradiction_wins_over_an_undecidable_one(): def test_the_example_policy_still_loads_and_reports_what_it_could_not_compare(): """docs/examples/policy.example.json is what a user copies. It uses the specifier syntax this module does not parse, so it must load AND say so.""" - loaded = load_policy_file("docs/examples/policy.example.json") + loaded = load_policy_file(EXAMPLE) assert loaded.entries_for("policy_compliance") - assert loaded.undecidable, ( - "the example uses set-difference and a negated protocol; reporting " - "no undecidable pairs would mean the third answer had been lost" - ) + assert len(loaded.undecidable) == 1, loaded.undecidable + assert "rule 1" in loaded.undecidable[0] and "rule 4" in loaded.undecidable[0] + + +def test_the_example_does_not_warn_about_a_pair_a_parsed_field_proves_disjoint(): + """#387 review. Rules 2 and 4 used to warn because rule 2 says `!tcp` -- + but rule 2 is about 10.20.0.5 and rule 4 about 218.8.104.58, which is + provably disjoint on a field we DO parse. A false warning in our own + example teaches people to skip the warnings.""" + loaded = load_policy_file(EXAMPLE) + assert not any("rule 2" in u for u in loaded.undecidable) + + +# --- 4b. what the review found --------------------------------------------- + +P = {"srcIps": "10.10.10.0/24", "dstIps": "10.20.0.5"} +R = {"srcIps": "10.10.10.64/26", "dstIps": "10.20.0.0/16", "ipProtocols": ["tcp"], "dstPorts": "443"} + + +@pytest.mark.parametrize("tokens", [["6"], "tcp,udp", ["tcp, udp"], ["TCP"]]) +def test_protocol_numbers_and_comma_lists_are_understood_not_guessed(tokens): + """Batfish reads `"6"` as TCP and pybatfish splits `"tcp,udp"`. Taking + them as unknown names made a real contradiction load as consistent.""" + with pytest.raises(PolicyError): + load_policy(policy(rule(1, "prohibition", [{**P, "ipProtocols": tokens}]), + rule(2, "requirement", [R]))) + + +def test_an_unrecognised_protocol_is_undecidable_not_disjoint(): + loaded = load_policy(policy(rule(1, "prohibition", [{**P, "ipProtocols": ["banana"]}]), + rule(2, "requirement", [R]))) + assert len(loaded.undecidable) == 1 + + +@pytest.mark.parametrize("other", ["ACL_IN", "acl_in ", "/acl_in/", "@in(GigabitEthernet0/0)"]) +def test_a_filter_batfish_may_resolve_to_the_same_acl_is_not_skipped(other): + """`ACL_IN` and `acl_in` were "different filter: cannot collide". Batfish + resolves them to the same ACL, so the pair is reported, not skipped.""" + loaded = load_policy(policy(rule(1, "prohibition", [P]), + rule(2, "requirement", [R], filter_name=other))) + assert len(loaded.undecidable) == 1 + assert "could not be compared" in loaded.undecidable[0] + + +def test_one_disjoint_parsed_field_beats_an_unparsed_one(): + verdict, _ = policy_conflicts.overlap( + {"dstIps": "10.20.0.5", "ipProtocols": ["!tcp"]}, {"dstIps": "218.8.104.58"}) + assert verdict is False + + +def test_an_unparsed_field_earlier_in_field_order_cannot_hide_a_disjoint_later_one(): + """The order matters for the mutant this pins. srcIps is compared before + dstIps, so an implementation that gave up at the first unparsed field + would answer "undecidable" here instead of "disjoint".""" + verdict, _ = policy_conflicts.overlap( + {"srcIps": "0.0.0.0/0 \\ (1.1.1.1)", "dstIps": "10.20.0.5"}, + {"dstIps": "218.8.104.58"}) + assert verdict is False + + +def test_an_unmodelled_field_cannot_hide_a_disjoint_parsed_one(): + verdict, _ = policy_conflicts.overlap( + {"dstIps": "10.20.0.5", "icmpCodes": "0"}, {"dstIps": "218.8.104.58"}) + assert verdict is False + + +def test_rules_without_numbers_are_named_by_the_numbers_assigned_to_them(): + """Pins the ordering in load_policy(): the contradiction check runs AFTER + numbers are assigned. Before it, both rules read "an unnumbered rule".""" + a, b = rule(1, "prohibition", [P]), rule(2, "requirement", [R]) + del a["number"], b["number"] + with pytest.raises(PolicyError) as caught: + load_policy(policy(a, b)) + assert "unnumbered" not in str(caught.value) + assert "rule 1" in str(caught.value) and "rule 2" in str(caught.value) + + +def test_overlapping_port_ranges_are_described_once(): + _, detail = policy_conflicts.overlap({"dstPorts": "1-100,50-200"}, {"dstPorts": "60-70"}) + assert detail == "dst ports 60-70" + + +def test_a_boolean_is_not_a_port(): + verdict, _ = policy_conflicts.overlap({"dstPorts": True}, {"dstPorts": "1"}) + assert verdict is None + + +# --- 4c. the CLIs say it ----------------------------------------------------- + +def test_the_pipeline_cli_prints_each_undecidable_pair(monkeypatch, capsys): + """Deleting the print in analysis/pipeline.py used to fail no test.""" + from analysis import pipeline + monkeypatch.setattr(pipeline, "analyse", lambda *a, **k: []) + monkeypatch.setattr(pipeline, "_print_summary", lambda results: None) + monkeypatch.setattr("sys.argv", ["pipeline", "unused-dir", "--policy", str(EXAMPLE)]) + pipeline.main() + warnings = [line for line in capsys.readouterr().out.splitlines() if line.startswith("warning:")] + assert len(warnings) == 1 and "rule 1" in warnings[0] and "rule 4" in warnings[0] + + +def test_the_change_impact_cli_prints_each_undecidable_pair(monkeypatch, capsys): + from analysis import change_impact + monkeypatch.setattr(change_impact, "analyse_change", lambda *a, **k: []) + monkeypatch.setattr("sys.argv", ["change_impact", "before", "after", str(EXAMPLE)]) + change_impact.main() + warnings = [line for line in capsys.readouterr().out.splitlines() if line.startswith("warning:")] + assert len(warnings) == 1 # --- 5. the overlap primitive itself -------------------------------------