Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
104 changes: 92 additions & 12 deletions analysis/policy_conflicts.py
Original file line number Diff line number Diff line change
Expand Up @@ -290,6 +290,8 @@ def overlap(query_a: Any, query_b: Any) -> Tuple[Optional[bool], str]:
#: Characters that make a `filter` a Batfish SPECIFIER rather than a name --
#: `/acl_in/` is a regex, `@in(Gi0/0)` an interface's filter, `a,b` a list.
#: Such a filter can name the same ACL as another rule's plain name.
#: Not `_COMPLEX`: in an address list `/` is a CIDR and `,` separates entries,
#: both parsed above, while in a filter name they mean regex and list.
_FILTER_SPECIFIER = re.compile(r"[/@(,]")


Expand Down Expand Up @@ -321,17 +323,15 @@ def _ref(entry: Any) -> str:
return f"{label} ({description!r})" if description else label


def contradictions(entries: Sequence[Any]) -> Tuple[List[str], List[str]]:
"""Find prohibition/requirement pairs that cannot both hold.
def _compare(prohibitions: Sequence[Any], requirements: Sequence[Any],
name_p: Any = _ref, name_r: Any = _ref) -> Tuple[List[str], List[str]]:
"""Every prohibition against every requirement. The one comparison loop.

Returns `(contradictions, undecidable)`, both lists of human-readable
sentences naming each pair. A caller that refuses the policy should use
the first; it must not treat the second as nothing -- see the module
docstring.
Shared by contradictions() (one policy) and between() (two policies), so
the two can never disagree about what a contradiction is. `name_p` and
`name_r` say how to name a rule from each side -- the only difference
between the two callers.
"""
prohibitions = [e for e in entries if isinstance(e, dict) and e.get("kind") == "prohibition"]
requirements = [e for e in entries if isinstance(e, dict) and e.get("kind") == "requirement"]

conflicts: List[str] = []
undecided: List[str] = []
for forbid in prohibitions:
Expand All @@ -343,7 +343,7 @@ def contradictions(entries: Sequence[Any]) -> Tuple[List[str], List[str]]:
where = f"on {node} filter {filter_name}" if node and filter_name else ""
if scope is None:
undecided.append(
f"{_ref(forbid)} and {_ref(require)} name filters "
f"{name_p(forbid)} and {name_r(require)} name filters "
f"{forbid.get('filter')!r} and {require.get('filter')!r} on {node}, "
"which Batfish may resolve to the same ACL; could not be compared"
)
Expand All @@ -355,7 +355,7 @@ def contradictions(entries: Sequence[Any]) -> Tuple[List[str], List[str]]:
verdict, detail = overlap(qa, qb)
if verdict is True:
conflicts.append(
f"{_ref(forbid)} forbids what {_ref(require)} requires {where}"
f"{name_p(forbid)} forbids what {name_r(require)} requires {where}"
f": both describe {detail}".replace(" ", " ")
)
found = True
Expand All @@ -366,7 +366,87 @@ def contradictions(entries: Sequence[Any]) -> Tuple[List[str], List[str]]:
break
if not found and unknown_here:
undecided.append(
f"{_ref(forbid)} and {_ref(require)} {where} could not be compared"
f"{name_p(forbid)} and {name_r(require)} {where} could not be compared"
f": {unknown_here[0]}".replace(" ", " ")
)
return conflicts, undecided


def _kind(entries: Sequence[Any], kind: str) -> List[Any]:
return [e for e in entries if isinstance(e, dict) and e.get("kind") == kind]


def contradictions(entries: Sequence[Any]) -> Tuple[List[str], List[str]]:
"""Find prohibition/requirement pairs that cannot both hold.

Returns `(contradictions, undecidable)`, both lists of human-readable
sentences naming each pair. A caller that refuses the policy should use
the first; it must not treat the second as nothing -- see the module
docstring.
"""
return _compare(_kind(entries, "prohibition"), _kind(entries, "requirement"))


def between(entries_a: Sequence[Any], entries_b: Sequence[Any],
name_a: str = "policy A", name_b: str = "policy B") -> Tuple[List[str], List[str]]:
"""Find rules in one policy that forbid what the OTHER policy requires.

WHY THIS EXISTS (Studio 6, multi-policy conflict check)
Two teams can each write a policy for the same firewall -- security's
and the application owners' -- and each can be consistent on its own.
Applied together, a prohibition in one and a requirement in the other
can describe the same traffic, and then no configuration satisfies
both. load_policy() cannot see that, because it only ever reads one
file at a time.

Both directions are checked: A's prohibitions against B's requirements,
and B's prohibitions against A's. Same three answers as within one policy
-- a pair that cannot be decided is returned in the second list, never
dropped. Rules are named with their policy, so the user knows which file
to change.
"""
ref_a = lambda e: f"{name_a} {_ref(e)}" # noqa: E731
ref_b = lambda e: f"{name_b} {_ref(e)}" # noqa: E731
c1, u1 = _compare(_kind(entries_a, "prohibition"), _kind(entries_b, "requirement"), ref_a, ref_b)
c2, u2 = _compare(_kind(entries_b, "prohibition"), _kind(entries_a, "requirement"), ref_b, ref_a)
return c1 + c2, u1 + u2


def main(argv: Optional[Sequence[str]] = None) -> int:
"""`python -m analysis.policy_conflicts first.json second.json`

Loads each file the normal way -- so a file that contradicts ITSELF is
refused with the loader's own message, naming that file -- then compares
the two. Exit status 1 when they contradict each other, 0 otherwise.
"""
import sys
from pathlib import Path

from analysis.policy import PolicyError, load_policy_file

args = list(sys.argv[1:] if argv is None else argv)
if len(args) != 2:
print("usage: python -m analysis.policy_conflicts <first-policy.json> <second-policy.json>")
return 2
loaded = []
for path in args:
try:
loaded.append(load_policy_file(path))
except PolicyError as error:
print(f"{path}: not loaded -- {error}")
return 2
a, b = (Path(x).name for x in args)
conflicts, undecided = between(
loaded[0].entries_for("policy_compliance"), loaded[1].entries_for("policy_compliance"), a, b)
for line in conflicts:
print(f"CONFLICT: {line}")
for line in undecided:
print(f"warning: {line}")
if not conflicts:
print(f"No rule in {a} forbids what {b} requires, or the other way round"
+ (f" ({len(undecided)} pair(s) could not be compared, above)." if undecided else "."))
return 1 if conflicts else 0


if __name__ == "__main__":
raise SystemExit(main())
23 changes: 23 additions & 0 deletions docs/policy-rules.md
Original file line number Diff line number Diff line change
Expand Up @@ -367,6 +367,29 @@ leaves the user diffing two header specifiers by eye.
This needs no Batfish, no Docker and no config. It is a property of the policy
file alone.

### Two policies that contradict each other (Studio 6)

`load_policy()` refuses a policy that contradicts **itself**, but it reads one
file at a time. Two teams can each write a consistent policy for the same
firewall — security's and the application owners' — that contradict each
other once applied together. `policy_conflicts.between()` compares two
policies in both directions, using the same comparison loop as the
single-policy check, so the two can never disagree about what a contradiction
is:

```
$ python -m analysis.policy_conflicts security.json app-owners.json
CONFLICT: security.json rule 1 ('Guests must not reach the finance server')
forbids what app-owners.json rule 1 ('The kiosk subnet must reach the finance
API over HTTPS') requires on rtr-us5 filter acl_in: both describe
src 10.10.10.64/26, dst 10.20.0.5/32, dst ports 443, protocol tcp
```

Every rule is named with its file, because both files may have a "rule 1".
A pair that cannot be compared is printed as a `warning:`, the same three
answers as within one policy. The exit status is 1 for a conflict, so it can
gate a script.

#### What is not a contradiction

Two prohibitions over the same traffic are consistent — denying twice is
Expand Down
120 changes: 120 additions & 0 deletions tests/test_policy_between.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
"""Two policies that are each fine, and contradict each other (Studio 6).

Security writes one policy for a firewall, the application owners another.
Each loads cleanly on its own, because load_policy() reads one file at a
time. Applied together, a prohibition in one and a requirement in the other
can describe the same traffic, and then no configuration satisfies both.

These pin `policy_conflicts.between()` and its command line.
"""

import json


from analysis import policy_conflicts
from analysis.policy import load_policy


def rule(n, kind, queries, node="rtr-us5", flt="acl_in"):
return {"number": n, "description": f"rule {n}", "node": node, "kind": kind, "filter": flt,
"violation_severity": "high", "violation_summary": "x", "queries": queries}


SECURITY = [rule(1, "prohibition", [{"srcIps": "10.10.10.0/24", "dstIps": "10.20.0.5"}])]
APP_OWNERS = [rule(1, "requirement", [{"srcIps": "10.10.10.64/26", "dstIps": "10.20.0.0/16",
"ipProtocols": ["tcp"], "dstPorts": "443"}])]


def test_each_policy_loads_on_its_own():
"""The premise: nothing catches this today."""
load_policy({"device": "rtr-us5", "policy_compliance": SECURITY})
assert load_policy({"device": "rtr-us5", "policy_compliance": APP_OWNERS})


def test_a_prohibition_in_one_against_a_requirement_in_the_other_is_a_conflict():
conflicts, undecided = policy_conflicts.between(SECURITY, APP_OWNERS, "security.json", "apps.json")
assert len(conflicts) == 1 and undecided == []
# The claim, not the sentence: which file forbids, which requires.
forbidder, _, requirer = conflicts[0].partition(" forbids what ")
assert forbidder.startswith("security.json rule 1")
assert requirer.startswith("apps.json rule 1") and " requires " in requirer
assert "10.10.10.64/26" in conflicts[0] and "443" in conflicts[0], "names the overlap"


def test_both_directions_are_checked():
"""The prohibition is in the SECOND file here."""
conflicts, _ = policy_conflicts.between(APP_OWNERS, SECURITY, "apps.json", "security.json")
assert len(conflicts) == 1
forbidder, _, requirer = conflicts[0].partition(" forbids what ")
assert forbidder.startswith("security.json rule 1"), "the file with the prohibition forbids"
assert requirer.startswith("apps.json rule 1")


def test_rules_with_the_same_number_are_told_apart_by_their_file():
"""Both files have a "rule 1" -- the message must say which is which."""
conflicts, _ = policy_conflicts.between(SECURITY, APP_OWNERS, "a.json", "b.json")
assert "a.json rule 1" in conflicts[0] and "b.json rule 1" in conflicts[0]


def test_two_policies_that_agree_do_not_conflict():
same = [rule(9, "prohibition", [{"srcIps": "10.10.10.0/24", "dstIps": "10.20.0.5"}])]
assert policy_conflicts.between(SECURITY, same) == ([], [])


def test_disjoint_traffic_across_policies_does_not_conflict():
other = [rule(1, "requirement", [{"srcIps": "192.168.1.0/24", "dstIps": "10.20.0.5"}])]
assert policy_conflicts.between(SECURITY, other) == ([], [])


def test_different_devices_across_policies_do_not_conflict():
elsewhere = [dict(APP_OWNERS[0], node="rtr-hq")]
assert policy_conflicts.between(SECURITY, elsewhere) == ([], [])


def test_a_pair_that_cannot_be_compared_is_reported_not_dropped():
vague = [rule(1, "requirement", [{"dstIps": "0.0.0.0/0 \\ (1.1.1.1)"}])]
conflicts, undecided = policy_conflicts.between(SECURITY, vague, "a.json", "b.json")
assert conflicts == [] and len(undecided) == 1
assert "a.json rule 1" in undecided[0] and "b.json rule 1" in undecided[0]


def test_within_one_policy_is_unchanged_by_the_refactor():
"""contradictions() and between() share one loop; this pins that the
single-policy wording did not pick up a policy name."""
conflicts, _ = policy_conflicts.contradictions(SECURITY + [dict(APP_OWNERS[0], number=2)])
assert conflicts[0].startswith("rule 1 ('rule 1') forbids what rule 2")


# --- the command line ----------------------------------------------------------

def _write(path, entries):
path.write_text(json.dumps({"device": "rtr-us5", "policy_compliance": entries}))
return str(path)


def test_cli_reports_a_conflict_and_exits_1(tmp_path, capsys):
a, b = _write(tmp_path / "security.json", SECURITY), _write(tmp_path / "apps.json", APP_OWNERS)
assert policy_conflicts.main([a, b]) == 1
out = capsys.readouterr().out
assert out.startswith("CONFLICT: security.json rule 1")
assert "forbids what apps.json rule 1" in out


def test_cli_exits_0_when_the_policies_agree(tmp_path, capsys):
other = [rule(1, "requirement", [{"srcIps": "192.168.1.0/24", "dstIps": "10.20.0.5"}])]
a, b = _write(tmp_path / "a.json", SECURITY), _write(tmp_path / "b.json", other)
assert policy_conflicts.main([a, b]) == 0
assert "No rule in a.json forbids what b.json requires" in capsys.readouterr().out


def test_cli_refuses_a_file_that_contradicts_itself_and_names_it(tmp_path, capsys):
bad = _write(tmp_path / "bad.json", SECURITY + [dict(APP_OWNERS[0], number=2)])
ok = _write(tmp_path / "ok.json", SECURITY)
assert policy_conflicts.main([bad, ok]) == 2
out = capsys.readouterr().out
assert "bad.json: not loaded" in out and "contradicts itself" in out


def test_cli_prints_usage_with_the_wrong_number_of_files(capsys):
assert policy_conflicts.main(["only-one.json"]) == 2
assert "usage:" in capsys.readouterr().out
8 changes: 8 additions & 0 deletions tests/test_policy_contradictions.py
Original file line number Diff line number Diff line change
Expand Up @@ -300,6 +300,14 @@ def test_overlapping_port_ranges_are_described_once():
assert detail == "dst ports 60-70"


def test_overlapping_address_blocks_are_described_once():
# 10.20.0.5 sits inside BOTH blocks on the other side, so without the
# de-duplication it would be listed twice.
_, detail = policy_conflicts.overlap({"dstIps": "10.20.0.5"},
{"dstIps": "10.20.0.0/24,10.0.0.0/8"})
assert detail == "dst 10.20.0.5/32"


def test_a_boolean_is_not_a_port():
verdict, _ = policy_conflicts.overlap({"dstPorts": True}, {"dstPorts": "1"})
assert verdict is None
Expand Down
Loading