From c805520dc29108a78358f5955025d7800c7fdb98 Mon Sep 17 00:00:00 2001 From: Shubham Date: Thu, 29 Oct 2026 14:22:01 +1300 Subject: [PATCH 1/2] Check two policies against each other, not just each against itself (closes #419) #419 (multi-policy conflict check). Stacked on #387, whose comparison code it reuses. load_policy() refuses a policy that contradicts itself, but it reads one file at a time. Two teams can each write a consistent policy for the same firewall that contradict each other once applied together. policy_conflicts.between() compares two policies in both directions -- A's prohibitions against B's requirements, and B's against A's -- and names every rule with its file, because both may have a "rule 1". The pairwise loop is now one function, _compare(), shared by contradictions() and between(), so the single-policy and two-policy checks cannot disagree about what a contradiction is. contradictions() behaves exactly as before: all 45 of its tests pass unchanged. `python -m analysis.policy_conflicts a.json b.json` loads each file the normal way (a file that contradicts itself is refused and named), prints CONFLICT and warning lines, and exits 1 on a conflict so it can gate a script. Mutation-tested: one direction only, rules not named with their policy, the CLI exiting 0 on a conflict, and undecidable pairs dropped each fail the new tests. --- analysis/policy_conflicts.py | 104 ++++++++++++++++++++++++++---- docs/policy-rules.md | 23 +++++++ tests/test_policy_between.py | 120 +++++++++++++++++++++++++++++++++++ 3 files changed, 234 insertions(+), 13 deletions(-) create mode 100644 tests/test_policy_between.py diff --git a/analysis/policy_conflicts.py b/analysis/policy_conflicts.py index ac3113c..6d424f5 100644 --- a/analysis/policy_conflicts.py +++ b/analysis/policy_conflicts.py @@ -321,17 +321,15 @@ def _ref(entry: Any) -> str: return f"{label} ({description!r})" if description else label -def contradictions(entries: Sequence[Any]) -> Tuple[List[str], List[str]]: - """Find prohibition/requirement pairs that cannot both hold. - - Returns `(contradictions, undecidable)`, both lists of human-readable - sentences naming each pair. A caller that refuses the policy should use - the first; it must not treat the second as nothing -- see the module - docstring. +def _compare(prohibitions: Sequence[Any], requirements: Sequence[Any], + name_p: Any = _ref, name_r: Any = _ref) -> Tuple[List[str], List[str]]: + """Every prohibition against every requirement. The one comparison loop. + + Shared by contradictions() (one policy) and between() (two policies), so + the two can never disagree about what a contradiction is. `name_p` and + `name_r` say how to name a rule from each side -- the only difference + between the two callers. """ - prohibitions = [e for e in entries if isinstance(e, dict) and e.get("kind") == "prohibition"] - requirements = [e for e in entries if isinstance(e, dict) and e.get("kind") == "requirement"] - conflicts: List[str] = [] undecided: List[str] = [] for forbid in prohibitions: @@ -343,7 +341,7 @@ def contradictions(entries: Sequence[Any]) -> Tuple[List[str], List[str]]: where = f"on {node} filter {filter_name}" if node and filter_name else "" if scope is None: undecided.append( - f"{_ref(forbid)} and {_ref(require)} name filters " + f"{name_p(forbid)} and {name_r(require)} name filters " f"{forbid.get('filter')!r} and {require.get('filter')!r} on {node}, " "which Batfish may resolve to the same ACL; could not be compared" ) @@ -355,7 +353,7 @@ def contradictions(entries: Sequence[Any]) -> Tuple[List[str], List[str]]: verdict, detail = overlap(qa, qb) if verdict is True: conflicts.append( - f"{_ref(forbid)} forbids what {_ref(require)} requires {where}" + f"{name_p(forbid)} forbids what {name_r(require)} requires {where}" f": both describe {detail}".replace(" ", " ") ) found = True @@ -366,7 +364,87 @@ def contradictions(entries: Sequence[Any]) -> Tuple[List[str], List[str]]: break if not found and unknown_here: undecided.append( - f"{_ref(forbid)} and {_ref(require)} {where} could not be compared" + f"{name_p(forbid)} and {name_r(require)} {where} could not be compared" f": {unknown_here[0]}".replace(" ", " ") ) return conflicts, undecided + + +def _kind(entries: Sequence[Any], kind: str) -> List[Any]: + return [e for e in entries if isinstance(e, dict) and e.get("kind") == kind] + + +def contradictions(entries: Sequence[Any]) -> Tuple[List[str], List[str]]: + """Find prohibition/requirement pairs that cannot both hold. + + Returns `(contradictions, undecidable)`, both lists of human-readable + sentences naming each pair. A caller that refuses the policy should use + the first; it must not treat the second as nothing -- see the module + docstring. + """ + return _compare(_kind(entries, "prohibition"), _kind(entries, "requirement")) + + +def between(entries_a: Sequence[Any], entries_b: Sequence[Any], + name_a: str = "policy A", name_b: str = "policy B") -> Tuple[List[str], List[str]]: + """Find rules in one policy that forbid what the OTHER policy requires. + + WHY THIS EXISTS (Studio 6, multi-policy conflict check) + Two teams can each write a policy for the same firewall -- security's + and the application owners' -- and each can be consistent on its own. + Applied together, a prohibition in one and a requirement in the other + can describe the same traffic, and then no configuration satisfies + both. load_policy() cannot see that, because it only ever reads one + file at a time. + + Both directions are checked: A's prohibitions against B's requirements, + and B's prohibitions against A's. Same three answers as within one policy + -- a pair that cannot be decided is returned in the second list, never + dropped. Rules are named with their policy, so the user knows which file + to change. + """ + ref_a = lambda e: f"{name_a} {_ref(e)}" # noqa: E731 + ref_b = lambda e: f"{name_b} {_ref(e)}" # noqa: E731 + c1, u1 = _compare(_kind(entries_a, "prohibition"), _kind(entries_b, "requirement"), ref_a, ref_b) + c2, u2 = _compare(_kind(entries_b, "prohibition"), _kind(entries_a, "requirement"), ref_b, ref_a) + return c1 + c2, u1 + u2 + + +def main(argv: Optional[Sequence[str]] = None) -> int: + """`python -m analysis.policy_conflicts first.json second.json` + + Loads each file the normal way -- so a file that contradicts ITSELF is + refused with the loader's own message, naming that file -- then compares + the two. Exit status 1 when they contradict each other, 0 otherwise. + """ + import sys + from pathlib import Path + + from analysis.policy import PolicyError, load_policy_file + + args = list(sys.argv[1:] if argv is None else argv) + if len(args) != 2: + print("usage: python -m analysis.policy_conflicts ") + return 2 + loaded = [] + for path in args: + try: + loaded.append(load_policy_file(path)) + except PolicyError as error: + print(f"{path}: not loaded -- {error}") + return 2 + a, b = (Path(x).name for x in args) + conflicts, undecided = between( + loaded[0].entries_for("policy_compliance"), loaded[1].entries_for("policy_compliance"), a, b) + for line in conflicts: + print(f"CONFLICT: {line}") + for line in undecided: + print(f"warning: {line}") + if not conflicts: + print(f"No rule in {a} forbids what {b} requires, or the other way round" + + (f" ({len(undecided)} pair(s) could not be compared, above)." if undecided else ".")) + return 1 if conflicts else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/docs/policy-rules.md b/docs/policy-rules.md index 82eb4fc..08b5201 100644 --- a/docs/policy-rules.md +++ b/docs/policy-rules.md @@ -367,6 +367,29 @@ leaves the user diffing two header specifiers by eye. This needs no Batfish, no Docker and no config. It is a property of the policy file alone. +### Two policies that contradict each other (Studio 6) + +`load_policy()` refuses a policy that contradicts **itself**, but it reads one +file at a time. Two teams can each write a consistent policy for the same +firewall — security's and the application owners' — that contradict each +other once applied together. `policy_conflicts.between()` compares two +policies in both directions, using the same comparison loop as the +single-policy check, so the two can never disagree about what a contradiction +is: + +``` +$ python -m analysis.policy_conflicts security.json app-owners.json +CONFLICT: security.json rule 1 ('Guests must not reach the finance server') +forbids what app-owners.json rule 1 ('The kiosk subnet must reach the finance +API over HTTPS') requires on rtr-us5 filter acl_in: both describe +src 10.10.10.64/26, dst 10.20.0.5/32, dst ports 443, protocol tcp +``` + +Every rule is named with its file, because both files may have a "rule 1". +A pair that cannot be compared is printed as a `warning:`, the same three +answers as within one policy. The exit status is 1 for a conflict, so it can +gate a script. + #### What is not a contradiction Two prohibitions over the same traffic are consistent — denying twice is diff --git a/tests/test_policy_between.py b/tests/test_policy_between.py new file mode 100644 index 0000000..3709ebc --- /dev/null +++ b/tests/test_policy_between.py @@ -0,0 +1,120 @@ +"""Two policies that are each fine, and contradict each other (Studio 6). + +Security writes one policy for a firewall, the application owners another. +Each loads cleanly on its own, because load_policy() reads one file at a +time. Applied together, a prohibition in one and a requirement in the other +can describe the same traffic, and then no configuration satisfies both. + +These pin `policy_conflicts.between()` and its command line. +""" + +import json + + +from analysis import policy_conflicts +from analysis.policy import load_policy + + +def rule(n, kind, queries, node="rtr-us5", flt="acl_in"): + return {"number": n, "description": f"rule {n}", "node": node, "kind": kind, "filter": flt, + "violation_severity": "high", "violation_summary": "x", "queries": queries} + + +SECURITY = [rule(1, "prohibition", [{"srcIps": "10.10.10.0/24", "dstIps": "10.20.0.5"}])] +APP_OWNERS = [rule(1, "requirement", [{"srcIps": "10.10.10.64/26", "dstIps": "10.20.0.0/16", + "ipProtocols": ["tcp"], "dstPorts": "443"}])] + + +def test_each_policy_loads_on_its_own(): + """The premise: nothing catches this today.""" + load_policy({"device": "rtr-us5", "policy_compliance": SECURITY}) + assert load_policy({"device": "rtr-us5", "policy_compliance": APP_OWNERS}) + + +def test_a_prohibition_in_one_against_a_requirement_in_the_other_is_a_conflict(): + conflicts, undecided = policy_conflicts.between(SECURITY, APP_OWNERS, "security.json", "apps.json") + assert len(conflicts) == 1 and undecided == [] + # The claim, not the sentence: which file forbids, which requires. + forbidder, _, requirer = conflicts[0].partition(" forbids what ") + assert forbidder.startswith("security.json rule 1") + assert requirer.startswith("apps.json rule 1") and " requires " in requirer + assert "10.10.10.64/26" in conflicts[0] and "443" in conflicts[0], "names the overlap" + + +def test_both_directions_are_checked(): + """The prohibition is in the SECOND file here.""" + conflicts, _ = policy_conflicts.between(APP_OWNERS, SECURITY, "apps.json", "security.json") + assert len(conflicts) == 1 + forbidder, _, requirer = conflicts[0].partition(" forbids what ") + assert forbidder.startswith("security.json rule 1"), "the file with the prohibition forbids" + assert requirer.startswith("apps.json rule 1") + + +def test_rules_with_the_same_number_are_told_apart_by_their_file(): + """Both files have a "rule 1" -- the message must say which is which.""" + conflicts, _ = policy_conflicts.between(SECURITY, APP_OWNERS, "a.json", "b.json") + assert "a.json rule 1" in conflicts[0] and "b.json rule 1" in conflicts[0] + + +def test_two_policies_that_agree_do_not_conflict(): + same = [rule(9, "prohibition", [{"srcIps": "10.10.10.0/24", "dstIps": "10.20.0.5"}])] + assert policy_conflicts.between(SECURITY, same) == ([], []) + + +def test_disjoint_traffic_across_policies_does_not_conflict(): + other = [rule(1, "requirement", [{"srcIps": "192.168.1.0/24", "dstIps": "10.20.0.5"}])] + assert policy_conflicts.between(SECURITY, other) == ([], []) + + +def test_different_devices_across_policies_do_not_conflict(): + elsewhere = [dict(APP_OWNERS[0], node="rtr-hq")] + assert policy_conflicts.between(SECURITY, elsewhere) == ([], []) + + +def test_a_pair_that_cannot_be_compared_is_reported_not_dropped(): + vague = [rule(1, "requirement", [{"dstIps": "0.0.0.0/0 \\ (1.1.1.1)"}])] + conflicts, undecided = policy_conflicts.between(SECURITY, vague, "a.json", "b.json") + assert conflicts == [] and len(undecided) == 1 + assert "a.json rule 1" in undecided[0] and "b.json rule 1" in undecided[0] + + +def test_within_one_policy_is_unchanged_by_the_refactor(): + """contradictions() and between() share one loop; this pins that the + single-policy wording did not pick up a policy name.""" + conflicts, _ = policy_conflicts.contradictions(SECURITY + [dict(APP_OWNERS[0], number=2)]) + assert conflicts[0].startswith("rule 1 ('rule 1') forbids what rule 2") + + +# --- the command line ---------------------------------------------------------- + +def _write(path, entries): + path.write_text(json.dumps({"device": "rtr-us5", "policy_compliance": entries})) + return str(path) + + +def test_cli_reports_a_conflict_and_exits_1(tmp_path, capsys): + a, b = _write(tmp_path / "security.json", SECURITY), _write(tmp_path / "apps.json", APP_OWNERS) + assert policy_conflicts.main([a, b]) == 1 + out = capsys.readouterr().out + assert out.startswith("CONFLICT: security.json rule 1") + assert "forbids what apps.json rule 1" in out + + +def test_cli_exits_0_when_the_policies_agree(tmp_path, capsys): + other = [rule(1, "requirement", [{"srcIps": "192.168.1.0/24", "dstIps": "10.20.0.5"}])] + a, b = _write(tmp_path / "a.json", SECURITY), _write(tmp_path / "b.json", other) + assert policy_conflicts.main([a, b]) == 0 + assert "No rule in a.json forbids what b.json requires" in capsys.readouterr().out + + +def test_cli_refuses_a_file_that_contradicts_itself_and_names_it(tmp_path, capsys): + bad = _write(tmp_path / "bad.json", SECURITY + [dict(APP_OWNERS[0], number=2)]) + ok = _write(tmp_path / "ok.json", SECURITY) + assert policy_conflicts.main([bad, ok]) == 2 + out = capsys.readouterr().out + assert "bad.json: not loaded" in out and "contradicts itself" in out + + +def test_cli_prints_usage_with_the_wrong_number_of_files(capsys): + assert policy_conflicts.main(["only-one.json"]) == 2 + assert "usage:" in capsys.readouterr().out From 7518ab16a5a01fb29926d046e1ea105b4dd39792 Mon Sep 17 00:00:00 2001 From: Shubham Date: Fri, 30 Oct 2026 11:04:02 +1300 Subject: [PATCH 2/2] Pin address de-duplication in overlap() and say why filters need their own regex Answers two of Samika's questions on #387. Mutation-testing _nets_meet and _ranges_meet killed 13 of 14 mutants; the survivor was removing the address de-duplication, which the port side already had a test for. Now 14 of 14. Kept off #387's branch so its approval is not voided for a test-only change. --- analysis/policy_conflicts.py | 2 ++ tests/test_policy_contradictions.py | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/analysis/policy_conflicts.py b/analysis/policy_conflicts.py index 6d424f5..bfbe595 100644 --- a/analysis/policy_conflicts.py +++ b/analysis/policy_conflicts.py @@ -290,6 +290,8 @@ def overlap(query_a: Any, query_b: Any) -> Tuple[Optional[bool], str]: #: Characters that make a `filter` a Batfish SPECIFIER rather than a name -- #: `/acl_in/` is a regex, `@in(Gi0/0)` an interface's filter, `a,b` a list. #: Such a filter can name the same ACL as another rule's plain name. +#: Not `_COMPLEX`: in an address list `/` is a CIDR and `,` separates entries, +#: both parsed above, while in a filter name they mean regex and list. _FILTER_SPECIFIER = re.compile(r"[/@(,]") diff --git a/tests/test_policy_contradictions.py b/tests/test_policy_contradictions.py index 2bc71c0..9a8c491 100644 --- a/tests/test_policy_contradictions.py +++ b/tests/test_policy_contradictions.py @@ -300,6 +300,14 @@ def test_overlapping_port_ranges_are_described_once(): assert detail == "dst ports 60-70" +def test_overlapping_address_blocks_are_described_once(): + # 10.20.0.5 sits inside BOTH blocks on the other side, so without the + # de-duplication it would be listed twice. + _, detail = policy_conflicts.overlap({"dstIps": "10.20.0.5"}, + {"dstIps": "10.20.0.0/24,10.0.0.0/8"}) + assert detail == "dst 10.20.0.5/32" + + def test_a_boolean_is_not_a_port(): verdict, _ = policy_conflicts.overlap({"dstPorts": True}, {"dstPorts": "1"}) assert verdict is None