diff --git a/analysis/checks/policy_compliance.py b/analysis/checks/policy_compliance.py index 4f2475a..a6734ce 100644 --- a/analysis/checks/policy_compliance.py +++ b/analysis/checks/policy_compliance.py @@ -116,7 +116,8 @@ is reported as proved, and what could not be reached is reported separately. """ -from typing import Any, Dict, List, Tuple +import re +from typing import Any, Dict, List, Set, Tuple from pybatfish.client.session import Session from pybatfish.datamodel.flow import HeaderConstraints @@ -302,6 +303,47 @@ def _search( return [frame.iloc[i] for i in range(len(frame))] +# A `filter` containing any of these is a Batfish specifier (a regex, an +# interface's filter, a list), not one name, so there is no single name to look +# for. Such rules are checked exactly as before #430. +_FILTER_SPECIFIER = re.compile(r"[/@(,]") + + +def _applied_filters(bf: Session) -> Dict[str, Set[str]]: + """{node: filter names applied to any of its interfaces, either direction}. + + WHY (#430). searchFilters proves what a filter WOULD do to traffic passing + through it, not that any traffic passes through it. Measured on + rtr-us5-secure with `ip access-group acl_in in` removed: every rule held, + PC-000 gave a green tick, and traceroute delivered telnet from the LAN to + the server. So a rule may only pass if its filter is in the path. + + Names are casefolded and stripped, because Batfish resolves `ACL_IN` and + `acl_in ` to `acl_in` (#387 review). Exceptions are NOT caught: run() + turns them into "could not check", because not knowing whether the filter + is applied is not knowing whether the rule holds. + """ + frame = ( + bf.q.interfaceProperties(properties="Incoming_Filter_Name,Outgoing_Filter_Name") + .answer() + .frame() + ) + applied: Dict[str, Set[str]] = {} + for _, row in frame.iterrows(): + names = applied.setdefault(row["Interface"].hostname, set()) + for column in ("Incoming_Filter_Name", "Outgoing_Filter_Name"): + if row[column]: + names.add(str(row[column]).strip().casefold()) + return applied + + +def _not_in_path(node: str, filter_name: str, applied: Dict[str, Set[str]]) -> bool: + """True when Batfish's model applies `filter_name` to no interface of `node`.""" + if _FILTER_SPECIFIER.search(filter_name): + return False + return filter_name.strip().casefold() not in applied.get(node, set()) + + def _describe(rule: Dict[str, Any], hits: List[Any]) -> str: """Turn matched rows into the evidence string, in Batfish's own words. @@ -601,11 +643,51 @@ def run(bf: Session) -> List[Dict[str, Any]]: ) ) + # Is each rule's filter in the path at all? (#430). Asked once, not per rule. + applied: Dict[str, Set[str]] = {} + lookup_error = None + if applicable: + try: + applied = _applied_filters(bf) + except Exception as error: + lookup_error = error + for rule in applicable: node = rule["node"] filter_name = rule["filter"] action = ACTION_BY_KIND[rule["kind"]] + # A filter in no interface's path decides nothing, so whatever + # searchFilters says about it, the rule is unchecked rather than + # held or broken. The wording says "Batfish's model", not "your + # config": on vendor-asa the config does apply its ACL, in a line + # Batfish did not parse, and claiming otherwise would be false. + # A failed lookup leaves `applied` empty, so every plainly named filter + # lands here too; a specifier filter is checked as before either way. + if _not_in_path(node, filter_name, applied): + if lookup_error is not None: + why = ("We could not find out which filters are applied to " + f"{node}'s interfaces, so we cannot tell whether " + f"{filter_name!r} is in the path of any traffic. " + + findings.describe_error(lookup_error)) + else: + why = (f"In Batfish's model of {node}, filter {filter_name!r} " + "is not applied to any interface, so no traffic passes " + "through it and what it permits or denies is not what " + "the device does. If your config does apply it, Batfish " + "did not understand that line; check the parse warnings.") + results.append( + findings.error_finding( + check=CHECK_NAME, + device=node, + summary=f"Could not check policy rule: {rule['description'].lower()}", + detail=_with_provenance(why, policy_label), + source=f"{node}:{filter_name}", + number=rule["number"] + ERROR_NUMBER_OFFSET, + ) + ) + continue + # Each arm is isolated. The arms describe ONE space between them, so a # hit in any of them violates the rule -- but a failure in one must not # discard what another one proved. See issue #22 and the PARTIAL CHECKS diff --git a/docs/policy-rules.md b/docs/policy-rules.md index a05e08f..20d7490 100644 --- a/docs/policy-rules.md +++ b/docs/policy-rules.md @@ -104,8 +104,17 @@ can surface as `success`. **Neither bucket means what a policy check needs it to mean.** `searchFilters` reasons at the filter level and needs no routing, so it is -correct on these fixtures and on any config where policy intent lives in ACLs -— and it cannot be fooled in either direction. +correct on these fixtures and on any config where policy intent lives in ACLs. + +**It can be fooled in one way: a filter that is in no traffic's path (#430).** +With `ip access-group acl_in in` removed from `rtr-us5-secure`, every rule held +and `PC-000` gave a green tick, while `traceroute` delivered telnet from the LAN +to the server. The proof was right about the filter and wrong about the device. +So before a rule can pass, the check confirms its filter is some interface's +incoming or outgoing filter on that node (`interfaceProperties`). If it is not, +or that cannot be found out, the rule reports **could not check** (`PC-05n`). +A `filter` written as a specifier (`/regex/`, `@in(...)`, a list) names no +single filter to look for, and is checked as before. > Found by @ARSH871-bot re-running this section rather than reading it, after > it was quoted in a team message. The measurement here was right and the @@ -122,7 +131,8 @@ name **raises `BatfishException`** rather than returning an empty frame. So the check must let that exception surface as a `status="error"` finding and must never wrap these calls in a bare `except: pass`. Empty then means exactly -one thing: the policy holds. +one thing: the policy holds **for that filter**, and the applied-filter check +above is what makes the filter's answer the device's answer. ### Do not use `invertSearch` to express "everything except X" diff --git a/tests/test_device_scoping.py b/tests/test_device_scoping.py index da1e260..31d4b0f 100644 --- a/tests/test_device_scoping.py +++ b/tests/test_device_scoping.py @@ -201,6 +201,9 @@ def test_a_missing_device_never_becomes_a_clean_result(monkeypatch): def _run_policy_with_devices(monkeypatch, present): """Run policy_compliance with a known device set and Batfish stubbed out.""" monkeypatch.setattr(policy_compliance.snapshot, "device_names", lambda bf: present) + # The filter is in the path (#430); that guard has its own tests. + monkeypatch.setattr(policy_compliance, "_not_in_path", lambda *args: False) + monkeypatch.setattr(policy_compliance, "_applied_filters", lambda bf: {}) # No rule may actually reach Batfish: every one that runs returns "holds". monkeypatch.setattr(policy_compliance, "_search", lambda *a, **k: []) return policy_compliance.run(_FakeSession()) diff --git a/tests/test_found_status_is_reported.py b/tests/test_found_status_is_reported.py index 1765dff..6374dc6 100644 --- a/tests/test_found_status_is_reported.py +++ b/tests/test_found_status_is_reported.py @@ -267,6 +267,9 @@ def test_policy_compliance_reports_a_violated_rule_as_found(monkeypatch): monkeypatch.setattr(policy_compliance, "POLICY_RULES", [rule]) monkeypatch.setattr(policy_compliance.snapshot, "device_names", lambda bf: {"rtr-us5"}) + # The filter is in the path (#430); that guard has its own tests. + monkeypatch.setattr(policy_compliance, "_not_in_path", lambda *args: False) + monkeypatch.setattr(policy_compliance, "_applied_filters", lambda bf: {}) # A non-empty hit list means the rule is violated. The shape is what # `_describe()` actually reads, so this exercises the real evidence text # rather than a stand-in that would break the moment it was used. diff --git a/tests/test_policy_filter_applied.py b/tests/test_policy_filter_applied.py new file mode 100644 index 0000000..660f5c6 --- /dev/null +++ b/tests/test_policy_filter_applied.py @@ -0,0 +1,210 @@ +""" +Netwise -- a policy rule only passes if its filter is in the path (#430). + +WHY THIS FILE EXISTS + searchFilters proves what a filter WOULD do to traffic passing through it. + It does not prove any traffic passes through it. Measured while researching + #423, on rtr-us5-secure with `ip access-group acl_in in` removed: + + PC-000 none No issues found by policy compliance + traceroute LAN -> 10.20.0.5 TCP/23 DELIVERED_TO_SUBNET + + A green tick while forbidden traffic is delivered. A rule whose filter is + applied to no interface now reports "could not check" instead. + + Most tests here fake the session. The last one runs the measurement above + against real Batfish, because the claim is about what Batfish's model + says, and a fake can only repeat what the test told it. + +RUN + pytest tests/test_policy_filter_applied.py -v +""" + +from pathlib import Path + +import pytest + +from analysis import pipeline +from analysis.checks import policy_compliance +from conftest import needs_batfish + +RULE = { + "number": 2, + "description": "The internal server is reachable only over HTTPS", + "kind": "prohibition", + "node": "rtr-us5", + "filter": "acl_in", + "violation_severity": "high", + "violation_summary": "The internal server accepts traffic other than HTTPS", + "queries": [{"srcIps": "10.10.10.0/24", "dstIps": "10.20.0.5"}], +} + + +# --- which filters count as applied ----------------------------------------- + + +class _Interface: + def __init__(self, hostname): + self.hostname = hostname + + +class _Frame: + """Just enough of a pandas frame for _applied_filters to iterate.""" + + def __init__(self, rows): + self._rows = rows + + def iterrows(self): + return enumerate(self._rows) + + +class _Session: + def __init__(self, rows): + frame = _Frame(rows) + + class _Q: + def interfaceProperties(self, **kwargs): + class _A: + def answer(self): + return self + + def frame(self): + return frame + return _A() + self.q = _Q() + + +def _row(node, incoming=None, outgoing=None): + return {"Interface": _Interface(node), "Incoming_Filter_Name": incoming, + "Outgoing_Filter_Name": outgoing} + + +def test_both_directions_count_and_names_are_casefolded(): + applied = policy_compliance._applied_filters(_Session([ + _row("rtr-us5", incoming="ACL_IN"), + _row("rtr-us5", outgoing="acl_out"), + _row("rtr-us5"), # an interface with no filter + _row("rtr-hq", incoming="hq_in"), + ])) + assert applied == {"rtr-us5": {"acl_in", "acl_out"}, "rtr-hq": {"hq_in"}} + + +@pytest.mark.parametrize("node, name, expected", [ + ("rtr-us5", "acl_in", False), # applied + ("rtr-us5", "ACL_IN ", False), # Batfish resolves this to acl_in + ("rtr-us5", "acl_out", True), # defined or not, applied nowhere + ("rtr-hq", "acl_in", True), # applied, but on another device + ("rtr-us5", "/acl/", False), # a specifier: not one name to look for + ("rtr-us5", "@in(Gi0/0)", False), +]) +def test_not_in_path(node, name, expected): + applied = {"rtr-us5": {"acl_in"}} + assert policy_compliance._not_in_path(node, name, applied) is expected + + +# --- what run() reports ----------------------------------------------------- + + +@pytest.fixture +def one_rule(monkeypatch): + monkeypatch.setattr(policy_compliance, "POLICY_RULES", [RULE]) + monkeypatch.setattr(policy_compliance.snapshot, "device_names", + lambda bf: {"rtr-us5"}) + searched = [] + + def _search(bf, node, filt, action, headers): + searched.append(filt) + return [] + monkeypatch.setattr(policy_compliance, "_search", _search) + return searched + + +def test_an_unapplied_filter_is_could_not_check_not_a_pass(one_rule, monkeypatch): + monkeypatch.setattr(policy_compliance, "_applied_filters", + lambda bf: {"rtr-us5": {"some_other_acl"}}) + + results = policy_compliance.run(object()) + + assert [f["status"] for f in results] == ["error"] + assert results[0]["id"] == "PC-052" + assert results[0]["summary"].startswith("Could not check policy rule") + assert "is not applied to any interface" in results[0]["evidence"]["detail"] + # The filter's answer is not asked for at all, so it cannot leak through. + assert one_rule == [] + + +def test_the_message_blames_the_model_not_the_config(one_rule, monkeypatch): + # vendor-asa applies its ACL in a line Batfish does not parse, so "your + # config does not apply it" would be false there. + monkeypatch.setattr(policy_compliance, "_applied_filters", lambda bf: {}) + + detail = policy_compliance.run(object())[0]["evidence"]["detail"] + + assert "In Batfish's model of rtr-us5" in detail + assert "check the parse warnings" in detail + + +def test_not_knowing_what_is_applied_is_could_not_check(one_rule, monkeypatch): + def _broken(bf): + raise RuntimeError("interfaceProperties failed") + monkeypatch.setattr(policy_compliance, "_applied_filters", _broken) + + results = policy_compliance.run(object()) + + assert [f["status"] for f in results] == ["error"] + assert "could not find out which filters are applied" in results[0]["evidence"]["detail"] + assert one_rule == [] + + +def test_a_specifier_filter_is_still_searched_when_the_lookup_fails(one_rule, monkeypatch): + # The guard never applies to a specifier, so its failing must not either. + monkeypatch.setattr(policy_compliance, "POLICY_RULES", [dict(RULE, filter="/acl_in/")]) + + def _broken(bf): + raise RuntimeError("interfaceProperties failed") + monkeypatch.setattr(policy_compliance, "_applied_filters", _broken) + + results = policy_compliance.run(object()) + + assert [f["status"] for f in results] == ["none"] + assert one_rule == ["/acl_in/"] + + +def test_an_applied_filter_is_checked_as_before(one_rule, monkeypatch): + monkeypatch.setattr(policy_compliance, "_applied_filters", + lambda bf: {"rtr-us5": {"acl_in"}}) + + results = policy_compliance.run(object()) + + assert [f["status"] for f in results] == ["none"] + assert one_rule == ["acl_in"] + + +# --- the measurement, against real Batfish ---------------------------------- + + +SECURE = Path(__file__).parent / "fixtures" / "rtr-us5-secure" / "configs" / "rtr-us5.cfg" + + +@needs_batfish +def test_real_batfish_no_green_tick_for_an_acl_applied_nowhere(tmp_path): + configs = tmp_path / "configs" + configs.mkdir() + text = SECURE.read_text() + assert "ip access-group acl_in in" in text + (configs / "rtr-us5.cfg").write_text(text.replace(" ip access-group acl_in in\n", "")) + + results = [f for f in pipeline.analyse(tmp_path, check_names=["policy_compliance"]) + if f["check"] == "policy_compliance"] + + assert "none" not in {f["status"] for f in results} + assert {f["status"] for f in results} == {"error"} + + +@needs_batfish +def test_real_batfish_the_secure_fixture_still_passes(): + results = [f for f in pipeline.analyse(SECURE.parent.parent, + check_names=["policy_compliance"]) + if f["check"] == "policy_compliance"] + + assert "error" not in {f["status"] for f in results} diff --git a/tests/test_policy_partial_checks.py b/tests/test_policy_partial_checks.py index ad086d1..102218e 100644 --- a/tests/test_policy_partial_checks.py +++ b/tests/test_policy_partial_checks.py @@ -73,6 +73,9 @@ def two_arm_rule(monkeypatch): """ monkeypatch.setattr(policy_compliance.snapshot, "device_names", lambda bf: {"rtr-us5"}) + # The filter is in the path (#430); that guard has its own tests. + monkeypatch.setattr(policy_compliance, "_not_in_path", lambda *args: False) + monkeypatch.setattr(policy_compliance, "_applied_filters", lambda bf: {}) rule = { "number": 2, "description": "The internal server is reachable only over HTTPS", diff --git a/tests/test_policy_wiring.py b/tests/test_policy_wiring.py index dfa8c93..b7185ba 100644 --- a/tests/test_policy_wiring.py +++ b/tests/test_policy_wiring.py @@ -255,6 +255,9 @@ def test_a_complete_entry_loads_and_runs(monkeypatch): everything.""" monkeypatch.setattr(policy_compliance.snapshot, "device_names", lambda bf: {"rtr-us5"}) + # The filter is in the path (#430); that guard has its own tests. + monkeypatch.setattr(policy_compliance, "_not_in_path", lambda *args: False) + monkeypatch.setattr(policy_compliance, "_applied_filters", lambda bf: {}) monkeypatch.setattr( policy_compliance, "_search", lambda *a, **k: [{"Flow": "f", "Line_Content": "deny ip any any"}]) @@ -512,6 +515,9 @@ def test_the_users_rules_are_the_ones_queried(monkeypatch): """ monkeypatch.setattr(policy_compliance.snapshot, "device_names", lambda bf: {"rtr-acme"}) + # The filter is in the path (#430); that guard has its own tests. + monkeypatch.setattr(policy_compliance, "_not_in_path", lambda *args: False) + monkeypatch.setattr(policy_compliance, "_applied_filters", lambda bf: {}) asked: List[str] = [] @@ -534,6 +540,9 @@ def test_without_a_policy_our_own_device_is_queried(monkeypatch): """The other direction, so the test above cannot pass by accident.""" monkeypatch.setattr(policy_compliance.snapshot, "device_names", lambda bf: {"rtr-us5"}) + # The filter is in the path (#430); that guard has its own tests. + monkeypatch.setattr(policy_compliance, "_not_in_path", lambda *args: False) + monkeypatch.setattr(policy_compliance, "_applied_filters", lambda bf: {}) asked: List[str] = [] monkeypatch.setattr( policy_compliance, "_search", @@ -555,6 +564,9 @@ def test_every_finding_says_which_policy_it_came_from(monkeypatch, supplied): node = "rtr-acme" if supplied else "rtr-us5" monkeypatch.setattr(policy_compliance.snapshot, "device_names", lambda bf: {node}) + # The filter is in the path (#430); that guard has its own tests. + monkeypatch.setattr(policy_compliance, "_not_in_path", lambda *args: False) + monkeypatch.setattr(policy_compliance, "_applied_filters", lambda bf: {}) monkeypatch.setattr( policy_compliance, "_search", lambda bf, n, f, a, h: [{"Flow": "a flow", "Line_Content": "permit ip any any"}]) @@ -587,6 +599,9 @@ def test_the_provenance_does_not_break_the_145_inversion_guard(monkeypatch): monkeypatch.setattr(policy_compliance.snapshot, "device_names", lambda bf: {"rtr-us5"}) + # The filter is in the path (#430); that guard has its own tests. + monkeypatch.setattr(policy_compliance, "_not_in_path", lambda *args: False) + monkeypatch.setattr(policy_compliance, "_applied_filters", lambda bf: {}) monkeypatch.setattr( policy_compliance, "_search", lambda bf, n, f, a, h: [{"Flow": "10.10.10.1 -> 8.8.8.8", @@ -796,6 +811,9 @@ def _run_against(monkeypatch, devices, pol): """Run the real check with a known device set and no Batfish.""" monkeypatch.setattr(policy_compliance.snapshot, "device_names", lambda bf: set(devices)) + # The filter is in the path (#430); that guard has its own tests. + monkeypatch.setattr(policy_compliance, "_not_in_path", lambda *args: False) + monkeypatch.setattr(policy_compliance, "_applied_filters", lambda bf: {}) monkeypatch.setattr(policy_compliance, "_search", lambda *a, **k: []) policy.set_active_policy(pol) try: