From 89b4256fc316839dbd461b76c09f73654992372d Mon Sep 17 00:00:00 2001 From: Shubham Date: Fri, 30 Oct 2026 11:19:30 +1300 Subject: [PATCH] Do not pass a policy rule whose filter is applied to no interface (closes #430) searchFilters proves what a filter would do to traffic passing through it, not that any traffic does. With the access-group removed from rtr-us5-secure, every rule held and PC-000 gave a green tick while traceroute delivered telnet from the LAN to the server. Each rule's filter must now be an incoming or outgoing filter on one of its node's interfaces (interfaceProperties), or the rule reports could not check. Names compare casefolded, as Batfish resolves them. A specifier filter names no single filter, so it is checked as before. The message says Batfish's model does not apply the filter, not that the config does not: vendor-asa applies its ACL in a line Batfish does not parse. stranger_config output is identical before and after on every fixture. --- analysis/checks/policy_compliance.py | 84 +++++++++- docs/policy-rules.md | 16 +- tests/test_device_scoping.py | 3 + tests/test_found_status_is_reported.py | 3 + tests/test_policy_filter_applied.py | 210 +++++++++++++++++++++++++ tests/test_policy_partial_checks.py | 3 + tests/test_policy_wiring.py | 18 +++ 7 files changed, 333 insertions(+), 4 deletions(-) create mode 100644 tests/test_policy_filter_applied.py diff --git a/analysis/checks/policy_compliance.py b/analysis/checks/policy_compliance.py index 4f2475a..a6734ce 100644 --- a/analysis/checks/policy_compliance.py +++ b/analysis/checks/policy_compliance.py @@ -116,7 +116,8 @@ is reported as proved, and what could not be reached is reported separately. """ -from typing import Any, Dict, List, Tuple +import re +from typing import Any, Dict, List, Set, Tuple from pybatfish.client.session import Session from pybatfish.datamodel.flow import HeaderConstraints @@ -302,6 +303,47 @@ def _search( return [frame.iloc[i] for i in range(len(frame))] +# A `filter` containing any of these is a Batfish specifier (a regex, an +# interface's filter, a list), not one name, so there is no single name to look +# for. Such rules are checked exactly as before #430. +_FILTER_SPECIFIER = re.compile(r"[/@(,]") + + +def _applied_filters(bf: Session) -> Dict[str, Set[str]]: + """{node: filter names applied to any of its interfaces, either direction}. + + WHY (#430). searchFilters proves what a filter WOULD do to traffic passing + through it, not that any traffic passes through it. Measured on + rtr-us5-secure with `ip access-group acl_in in` removed: every rule held, + PC-000 gave a green tick, and traceroute delivered telnet from the LAN to + the server. So a rule may only pass if its filter is in the path. + + Names are casefolded and stripped, because Batfish resolves `ACL_IN` and + `acl_in ` to `acl_in` (#387 review). Exceptions are NOT caught: run() + turns them into "could not check", because not knowing whether the filter + is applied is not knowing whether the rule holds. + """ + frame = ( + bf.q.interfaceProperties(properties="Incoming_Filter_Name,Outgoing_Filter_Name") + .answer() + .frame() + ) + applied: Dict[str, Set[str]] = {} + for _, row in frame.iterrows(): + names = applied.setdefault(row["Interface"].hostname, set()) + for column in ("Incoming_Filter_Name", "Outgoing_Filter_Name"): + if row[column]: + names.add(str(row[column]).strip().casefold()) + return applied + + +def _not_in_path(node: str, filter_name: str, applied: Dict[str, Set[str]]) -> bool: + """True when Batfish's model applies `filter_name` to no interface of `node`.""" + if _FILTER_SPECIFIER.search(filter_name): + return False + return filter_name.strip().casefold() not in applied.get(node, set()) + + def _describe(rule: Dict[str, Any], hits: List[Any]) -> str: """Turn matched rows into the evidence string, in Batfish's own words. @@ -601,11 +643,51 @@ def run(bf: Session) -> List[Dict[str, Any]]: ) ) + # Is each rule's filter in the path at all? (#430). Asked once, not per rule. + applied: Dict[str, Set[str]] = {} + lookup_error = None + if applicable: + try: + applied = _applied_filters(bf) + except Exception as error: + lookup_error = error + for rule in applicable: node = rule["node"] filter_name = rule["filter"] action = ACTION_BY_KIND[rule["kind"]] + # A filter in no interface's path decides nothing, so whatever + # searchFilters says about it, the rule is unchecked rather than + # held or broken. The wording says "Batfish's model", not "your + # config": on vendor-asa the config does apply its ACL, in a line + # Batfish did not parse, and claiming otherwise would be false. + # A failed lookup leaves `applied` empty, so every plainly named filter + # lands here too; a specifier filter is checked as before either way. + if _not_in_path(node, filter_name, applied): + if lookup_error is not None: + why = ("We could not find out which filters are applied to " + f"{node}'s interfaces, so we cannot tell whether " + f"{filter_name!r} is in the path of any traffic. " + + findings.describe_error(lookup_error)) + else: + why = (f"In Batfish's model of {node}, filter {filter_name!r} " + "is not applied to any interface, so no traffic passes " + "through it and what it permits or denies is not what " + "the device does. If your config does apply it, Batfish " + "did not understand that line; check the parse warnings.") + results.append( + findings.error_finding( + check=CHECK_NAME, + device=node, + summary=f"Could not check policy rule: {rule['description'].lower()}", + detail=_with_provenance(why, policy_label), + source=f"{node}:{filter_name}", + number=rule["number"] + ERROR_NUMBER_OFFSET, + ) + ) + continue + # Each arm is isolated. The arms describe ONE space between them, so a # hit in any of them violates the rule -- but a failure in one must not # discard what another one proved. See issue #22 and the PARTIAL CHECKS diff --git a/docs/policy-rules.md b/docs/policy-rules.md index a05e08f..20d7490 100644 --- a/docs/policy-rules.md +++ b/docs/policy-rules.md @@ -104,8 +104,17 @@ can surface as `success`. **Neither bucket means what a policy check needs it to mean.** `searchFilters` reasons at the filter level and needs no routing, so it is -correct on these fixtures and on any config where policy intent lives in ACLs -— and it cannot be fooled in either direction. +correct on these fixtures and on any config where policy intent lives in ACLs. + +**It can be fooled in one way: a filter that is in no traffic's path (#430).** +With `ip access-group acl_in in` removed from `rtr-us5-secure`, every rule held +and `PC-000` gave a green tick, while `traceroute` delivered telnet from the LAN +to the server. The proof was right about the filter and wrong about the device. +So before a rule can pass, the check confirms its filter is some interface's +incoming or outgoing filter on that node (`interfaceProperties`). If it is not, +or that cannot be found out, the rule reports **could not check** (`PC-05n`). +A `filter` written as a specifier (`/regex/`, `@in(...)`, a list) names no +single filter to look for, and is checked as before. > Found by @ARSH871-bot re-running this section rather than reading it, after > it was quoted in a team message. The measurement here was right and the @@ -122,7 +131,8 @@ name **raises `BatfishException`** rather than returning an empty frame. So the check must let that exception surface as a `status="error"` finding and must never wrap these calls in a bare `except: pass`. Empty then means exactly -one thing: the policy holds. +one thing: the policy holds **for that filter**, and the applied-filter check +above is what makes the filter's answer the device's answer. ### Do not use `invertSearch` to express "everything except X" diff --git a/tests/test_device_scoping.py b/tests/test_device_scoping.py index da1e260..31d4b0f 100644 --- a/tests/test_device_scoping.py +++ b/tests/test_device_scoping.py @@ -201,6 +201,9 @@ def test_a_missing_device_never_becomes_a_clean_result(monkeypatch): def _run_policy_with_devices(monkeypatch, present): """Run policy_compliance with a known device set and Batfish stubbed out.""" monkeypatch.setattr(policy_compliance.snapshot, "device_names", lambda bf: present) + # The filter is in the path (#430); that guard has its own tests. + monkeypatch.setattr(policy_compliance, "_not_in_path", lambda *args: False) + monkeypatch.setattr(policy_compliance, "_applied_filters", lambda bf: {}) # No rule may actually reach Batfish: every one that runs returns "holds". monkeypatch.setattr(policy_compliance, "_search", lambda *a, **k: []) return policy_compliance.run(_FakeSession()) diff --git a/tests/test_found_status_is_reported.py b/tests/test_found_status_is_reported.py index 1765dff..6374dc6 100644 --- a/tests/test_found_status_is_reported.py +++ b/tests/test_found_status_is_reported.py @@ -267,6 +267,9 @@ def test_policy_compliance_reports_a_violated_rule_as_found(monkeypatch): monkeypatch.setattr(policy_compliance, "POLICY_RULES", [rule]) monkeypatch.setattr(policy_compliance.snapshot, "device_names", lambda bf: {"rtr-us5"}) + # The filter is in the path (#430); that guard has its own tests. + monkeypatch.setattr(policy_compliance, "_not_in_path", lambda *args: False) + monkeypatch.setattr(policy_compliance, "_applied_filters", lambda bf: {}) # A non-empty hit list means the rule is violated. The shape is what # `_describe()` actually reads, so this exercises the real evidence text # rather than a stand-in that would break the moment it was used. diff --git a/tests/test_policy_filter_applied.py b/tests/test_policy_filter_applied.py new file mode 100644 index 0000000..660f5c6 --- /dev/null +++ b/tests/test_policy_filter_applied.py @@ -0,0 +1,210 @@ +""" +Netwise -- a policy rule only passes if its filter is in the path (#430). + +WHY THIS FILE EXISTS + searchFilters proves what a filter WOULD do to traffic passing through it. + It does not prove any traffic passes through it. Measured while researching + #423, on rtr-us5-secure with `ip access-group acl_in in` removed: + + PC-000 none No issues found by policy compliance + traceroute LAN -> 10.20.0.5 TCP/23 DELIVERED_TO_SUBNET + + A green tick while forbidden traffic is delivered. A rule whose filter is + applied to no interface now reports "could not check" instead. + + Most tests here fake the session. The last one runs the measurement above + against real Batfish, because the claim is about what Batfish's model + says, and a fake can only repeat what the test told it. + +RUN + pytest tests/test_policy_filter_applied.py -v +""" + +from pathlib import Path + +import pytest + +from analysis import pipeline +from analysis.checks import policy_compliance +from conftest import needs_batfish + +RULE = { + "number": 2, + "description": "The internal server is reachable only over HTTPS", + "kind": "prohibition", + "node": "rtr-us5", + "filter": "acl_in", + "violation_severity": "high", + "violation_summary": "The internal server accepts traffic other than HTTPS", + "queries": [{"srcIps": "10.10.10.0/24", "dstIps": "10.20.0.5"}], +} + + +# --- which filters count as applied ----------------------------------------- + + +class _Interface: + def __init__(self, hostname): + self.hostname = hostname + + +class _Frame: + """Just enough of a pandas frame for _applied_filters to iterate.""" + + def __init__(self, rows): + self._rows = rows + + def iterrows(self): + return enumerate(self._rows) + + +class _Session: + def __init__(self, rows): + frame = _Frame(rows) + + class _Q: + def interfaceProperties(self, **kwargs): + class _A: + def answer(self): + return self + + def frame(self): + return frame + return _A() + self.q = _Q() + + +def _row(node, incoming=None, outgoing=None): + return {"Interface": _Interface(node), "Incoming_Filter_Name": incoming, + "Outgoing_Filter_Name": outgoing} + + +def test_both_directions_count_and_names_are_casefolded(): + applied = policy_compliance._applied_filters(_Session([ + _row("rtr-us5", incoming="ACL_IN"), + _row("rtr-us5", outgoing="acl_out"), + _row("rtr-us5"), # an interface with no filter + _row("rtr-hq", incoming="hq_in"), + ])) + assert applied == {"rtr-us5": {"acl_in", "acl_out"}, "rtr-hq": {"hq_in"}} + + +@pytest.mark.parametrize("node, name, expected", [ + ("rtr-us5", "acl_in", False), # applied + ("rtr-us5", "ACL_IN ", False), # Batfish resolves this to acl_in + ("rtr-us5", "acl_out", True), # defined or not, applied nowhere + ("rtr-hq", "acl_in", True), # applied, but on another device + ("rtr-us5", "/acl/", False), # a specifier: not one name to look for + ("rtr-us5", "@in(Gi0/0)", False), +]) +def test_not_in_path(node, name, expected): + applied = {"rtr-us5": {"acl_in"}} + assert policy_compliance._not_in_path(node, name, applied) is expected + + +# --- what run() reports ----------------------------------------------------- + + +@pytest.fixture +def one_rule(monkeypatch): + monkeypatch.setattr(policy_compliance, "POLICY_RULES", [RULE]) + monkeypatch.setattr(policy_compliance.snapshot, "device_names", + lambda bf: {"rtr-us5"}) + searched = [] + + def _search(bf, node, filt, action, headers): + searched.append(filt) + return [] + monkeypatch.setattr(policy_compliance, "_search", _search) + return searched + + +def test_an_unapplied_filter_is_could_not_check_not_a_pass(one_rule, monkeypatch): + monkeypatch.setattr(policy_compliance, "_applied_filters", + lambda bf: {"rtr-us5": {"some_other_acl"}}) + + results = policy_compliance.run(object()) + + assert [f["status"] for f in results] == ["error"] + assert results[0]["id"] == "PC-052" + assert results[0]["summary"].startswith("Could not check policy rule") + assert "is not applied to any interface" in results[0]["evidence"]["detail"] + # The filter's answer is not asked for at all, so it cannot leak through. + assert one_rule == [] + + +def test_the_message_blames_the_model_not_the_config(one_rule, monkeypatch): + # vendor-asa applies its ACL in a line Batfish does not parse, so "your + # config does not apply it" would be false there. + monkeypatch.setattr(policy_compliance, "_applied_filters", lambda bf: {}) + + detail = policy_compliance.run(object())[0]["evidence"]["detail"] + + assert "In Batfish's model of rtr-us5" in detail + assert "check the parse warnings" in detail + + +def test_not_knowing_what_is_applied_is_could_not_check(one_rule, monkeypatch): + def _broken(bf): + raise RuntimeError("interfaceProperties failed") + monkeypatch.setattr(policy_compliance, "_applied_filters", _broken) + + results = policy_compliance.run(object()) + + assert [f["status"] for f in results] == ["error"] + assert "could not find out which filters are applied" in results[0]["evidence"]["detail"] + assert one_rule == [] + + +def test_a_specifier_filter_is_still_searched_when_the_lookup_fails(one_rule, monkeypatch): + # The guard never applies to a specifier, so its failing must not either. + monkeypatch.setattr(policy_compliance, "POLICY_RULES", [dict(RULE, filter="/acl_in/")]) + + def _broken(bf): + raise RuntimeError("interfaceProperties failed") + monkeypatch.setattr(policy_compliance, "_applied_filters", _broken) + + results = policy_compliance.run(object()) + + assert [f["status"] for f in results] == ["none"] + assert one_rule == ["/acl_in/"] + + +def test_an_applied_filter_is_checked_as_before(one_rule, monkeypatch): + monkeypatch.setattr(policy_compliance, "_applied_filters", + lambda bf: {"rtr-us5": {"acl_in"}}) + + results = policy_compliance.run(object()) + + assert [f["status"] for f in results] == ["none"] + assert one_rule == ["acl_in"] + + +# --- the measurement, against real Batfish ---------------------------------- + + +SECURE = Path(__file__).parent / "fixtures" / "rtr-us5-secure" / "configs" / "rtr-us5.cfg" + + +@needs_batfish +def test_real_batfish_no_green_tick_for_an_acl_applied_nowhere(tmp_path): + configs = tmp_path / "configs" + configs.mkdir() + text = SECURE.read_text() + assert "ip access-group acl_in in" in text + (configs / "rtr-us5.cfg").write_text(text.replace(" ip access-group acl_in in\n", "")) + + results = [f for f in pipeline.analyse(tmp_path, check_names=["policy_compliance"]) + if f["check"] == "policy_compliance"] + + assert "none" not in {f["status"] for f in results} + assert {f["status"] for f in results} == {"error"} + + +@needs_batfish +def test_real_batfish_the_secure_fixture_still_passes(): + results = [f for f in pipeline.analyse(SECURE.parent.parent, + check_names=["policy_compliance"]) + if f["check"] == "policy_compliance"] + + assert "error" not in {f["status"] for f in results} diff --git a/tests/test_policy_partial_checks.py b/tests/test_policy_partial_checks.py index ad086d1..102218e 100644 --- a/tests/test_policy_partial_checks.py +++ b/tests/test_policy_partial_checks.py @@ -73,6 +73,9 @@ def two_arm_rule(monkeypatch): """ monkeypatch.setattr(policy_compliance.snapshot, "device_names", lambda bf: {"rtr-us5"}) + # The filter is in the path (#430); that guard has its own tests. + monkeypatch.setattr(policy_compliance, "_not_in_path", lambda *args: False) + monkeypatch.setattr(policy_compliance, "_applied_filters", lambda bf: {}) rule = { "number": 2, "description": "The internal server is reachable only over HTTPS", diff --git a/tests/test_policy_wiring.py b/tests/test_policy_wiring.py index dfa8c93..b7185ba 100644 --- a/tests/test_policy_wiring.py +++ b/tests/test_policy_wiring.py @@ -255,6 +255,9 @@ def test_a_complete_entry_loads_and_runs(monkeypatch): everything.""" monkeypatch.setattr(policy_compliance.snapshot, "device_names", lambda bf: {"rtr-us5"}) + # The filter is in the path (#430); that guard has its own tests. + monkeypatch.setattr(policy_compliance, "_not_in_path", lambda *args: False) + monkeypatch.setattr(policy_compliance, "_applied_filters", lambda bf: {}) monkeypatch.setattr( policy_compliance, "_search", lambda *a, **k: [{"Flow": "f", "Line_Content": "deny ip any any"}]) @@ -512,6 +515,9 @@ def test_the_users_rules_are_the_ones_queried(monkeypatch): """ monkeypatch.setattr(policy_compliance.snapshot, "device_names", lambda bf: {"rtr-acme"}) + # The filter is in the path (#430); that guard has its own tests. + monkeypatch.setattr(policy_compliance, "_not_in_path", lambda *args: False) + monkeypatch.setattr(policy_compliance, "_applied_filters", lambda bf: {}) asked: List[str] = [] @@ -534,6 +540,9 @@ def test_without_a_policy_our_own_device_is_queried(monkeypatch): """The other direction, so the test above cannot pass by accident.""" monkeypatch.setattr(policy_compliance.snapshot, "device_names", lambda bf: {"rtr-us5"}) + # The filter is in the path (#430); that guard has its own tests. + monkeypatch.setattr(policy_compliance, "_not_in_path", lambda *args: False) + monkeypatch.setattr(policy_compliance, "_applied_filters", lambda bf: {}) asked: List[str] = [] monkeypatch.setattr( policy_compliance, "_search", @@ -555,6 +564,9 @@ def test_every_finding_says_which_policy_it_came_from(monkeypatch, supplied): node = "rtr-acme" if supplied else "rtr-us5" monkeypatch.setattr(policy_compliance.snapshot, "device_names", lambda bf: {node}) + # The filter is in the path (#430); that guard has its own tests. + monkeypatch.setattr(policy_compliance, "_not_in_path", lambda *args: False) + monkeypatch.setattr(policy_compliance, "_applied_filters", lambda bf: {}) monkeypatch.setattr( policy_compliance, "_search", lambda bf, n, f, a, h: [{"Flow": "a flow", "Line_Content": "permit ip any any"}]) @@ -587,6 +599,9 @@ def test_the_provenance_does_not_break_the_145_inversion_guard(monkeypatch): monkeypatch.setattr(policy_compliance.snapshot, "device_names", lambda bf: {"rtr-us5"}) + # The filter is in the path (#430); that guard has its own tests. + monkeypatch.setattr(policy_compliance, "_not_in_path", lambda *args: False) + monkeypatch.setattr(policy_compliance, "_applied_filters", lambda bf: {}) monkeypatch.setattr( policy_compliance, "_search", lambda bf, n, f, a, h: [{"Flow": "10.10.10.1 -> 8.8.8.8", @@ -796,6 +811,9 @@ def _run_against(monkeypatch, devices, pol): """Run the real check with a known device set and no Batfish.""" monkeypatch.setattr(policy_compliance.snapshot, "device_names", lambda bf: set(devices)) + # The filter is in the path (#430); that guard has its own tests. + monkeypatch.setattr(policy_compliance, "_not_in_path", lambda *args: False) + monkeypatch.setattr(policy_compliance, "_applied_filters", lambda bf: {}) monkeypatch.setattr(policy_compliance, "_search", lambda *a, **k: []) policy.set_active_policy(pol) try: