From 55c8f85b5bd4b67bc6a6ca47056c831f6b0cd4f0 Mon Sep 17 00:00:00 2001 From: Shubham Date: Wed, 11 Nov 2026 11:19:47 +1300 Subject: [PATCH 1/2] Refuse at load a rule whose traffic Batfish cannot be asked about Follow-up I offered on #442, with the two cases Arsh listed on #467. policy_compliance queries and access_control headers go straight to HeaderConstraints, which accepts almost anything locally, so these loaded and failed only at scan time with an error the user could not act on: queries: [] -> 'this rule has no queries' queries: '10.10.10.0/24' -> 13 queries, one per character a query that is a string -> TypeError from pybatfish {'srcIp': ...} -> TypeError, unexpected keyword {'srcIps': '10.10.10.300'} -> 'Work terminated abnormally' Each is now refused at load, naming the rule. Field names are read from HeaderConstraints itself. Only an address element that is plainly one address or prefix is checked; ranges, wildcards, set difference and named groups are left to Batfish, and each accepted form is run against real Batfish in the new tests. An empty query still loads: it means all traffic. Two existing tests used placeholder traffic ('x', []) and now use a real query, as the loader test already did for queries. --- analysis/policy.py | 67 +++++++++++++++ docs/policy-rules.md | 22 +++++ tests/test_policy_loader.py | 2 + tests/test_policy_traffic_shape.py | 132 +++++++++++++++++++++++++++++ tests/test_session_isolation.py | 2 +- 5 files changed, 224 insertions(+), 1 deletion(-) create mode 100644 tests/test_policy_traffic_shape.py diff --git a/analysis/policy.py b/analysis/policy.py index 6209d4c..d7123db 100644 --- a/analysis/policy.py +++ b/analysis/policy.py @@ -61,10 +61,13 @@ from __future__ import annotations import json +import re import threading from pathlib import Path from typing import Any, Dict, List, Mapping, Optional, Tuple +import attr + from analysis import policy_conflicts from analysis.findings import PREFIX_BY_CHECK @@ -465,9 +468,73 @@ def _validate_entry( if bad: raise PolicyError(f"{where}: " + "; ".join(bad)) + # THE TRAFFIC A RULE DESCRIBES (#442 review, #467). Both sections hand + # these straight to Batfish's HeaderConstraints, which accepts almost + # anything locally, so a rule the check cannot use used to load and fail + # only at scan time, as "could not check" with a Python or Batfish error + # the user could not act on: `"queries": "10.10.10.0/24"` ran thirteen + # queries, one per character. Refused here, naming the rule, instead. + if section == "policy_compliance" and "queries" in normalised: + queries = normalised["queries"] + if not isinstance(queries, list) or not queries: + raise PolicyError( + f"{where}: 'queries' must be a non-empty list of traffic descriptions, " + f"got {queries!r}. A rule with no queries asks Batfish nothing" + ) + for number, query in enumerate(queries, start=1): + _check_traffic(f"{where}, query {number}", query) + if section == "access_control" and "headers" in normalised: + _check_traffic(f"{where}, 'headers'", normalised["headers"]) + return normalised, renamed +#: An address element that is plainly meant as one address or prefix: only +#: digits, dots and at most a slash. Batfish's address language is much +#: larger -- ranges (`a-b`), wildcards (`a:b`), set difference, `@` and +#: `ref.` names -- and none of that is judged here; Batfish does. +_PLAIN_ADDRESS = re.compile(r"^[0-9.]+(/[0-9]+)?$") + + +def _check_traffic(where: str, spec: Any) -> None: + """Refuse a traffic description Batfish cannot be asked. Raises PolicyError. + + Checked: it is a mapping, every key is a HeaderConstraints field, and every + plain address in srcIps/dstIps is a real address or prefix. An EMPTY + mapping is allowed: it means all traffic, and "this filter must permit + nothing" is a rule someone can mean. + """ + import ipaddress + + from pybatfish.datamodel.flow import HeaderConstraints + + if not isinstance(spec, Mapping): + raise PolicyError( + f"{where}: expected a mapping such as {{\"srcIps\": \"10.10.10.0/24\"}}, " + f"got {type(spec).__name__} {spec!r}" + ) + fields = set(attr.fields_dict(HeaderConstraints)) + unknown = sorted(set(spec) - fields) + if unknown: + raise PolicyError( + f"{where}: unknown traffic field(s) {', '.join(map(repr, unknown))}. " + f"Batfish accepts: {', '.join(sorted(fields))}" + ) + for key in ("srcIps", "dstIps"): + value = spec.get(key) + if not isinstance(value, str): + continue + for part in (p.strip() for p in value.split(",")): + if not _PLAIN_ADDRESS.match(part): + continue + try: + ipaddress.ip_network(part, strict=False) + except ValueError: + raise PolicyError( + f"{where}: {key} {part!r} is not a valid address or prefix" + ) from None + + def _highest_numbers() -> Dict[str, int]: """The highest `number` each numbered section can use (#376). diff --git a/docs/policy-rules.md b/docs/policy-rules.md index 9ae8930..d64f8c9 100644 --- a/docs/policy-rules.md +++ b/docs/policy-rules.md @@ -346,6 +346,28 @@ consistently with POL-4. --- +### A rule whose traffic Batfish cannot be asked about is refused + +A rule's `queries` (and an `access_control` statement's `headers`) go straight +to Batfish's `HeaderConstraints`, which accepts almost anything locally. Such +a rule used to load and then fail at scan time as "could not check", with an +error the user could not act on: `"queries": "10.10.10.0/24"` ran thirteen +queries, one per character. `load_policy()` now refuses, naming the rule: + +- `queries` that is not a non-empty list, or a query that is not a mapping; +- a field name `HeaderConstraints` does not have (`srcIp` for `srcIps`); +- an address element that is plainly one address or prefix (digits, dots, a + slash) but is not valid, such as `10.10.10.300`. + +Batfish's address language is larger than that: ranges (`a-b`), wildcards +(`a:b`), set difference and named groups are passed through for Batfish to +judge, and each accepted form is checked against real Batfish in +`tests/test_policy_traffic_shape.py`. An empty query, `{}`, still loads: it +means all traffic, and "this filter must permit nothing" is a rule someone +can mean. + +--- + ### A policy that forbids what it requires is refused (#322, US-30) A `prohibition` says traffic must be DENIED. A `requirement` says traffic must diff --git a/tests/test_policy_loader.py b/tests/test_policy_loader.py index 985189e..6c12601 100644 --- a/tests/test_policy_loader.py +++ b/tests/test_policy_loader.py @@ -371,6 +371,8 @@ def test_a_legacy_key_loads_and_is_reported_rather_than_refused(section, legacy) for key in _SECTION_REQUIRED.get(section, ()): if key == "queries": filler = [{"srcIps": "10.0.0.0/8"}] + elif key == "headers": + filler = {"srcIps": "10.0.0.0/8"} else: filler = allowed[key][0] if key in allowed else "x" entry.setdefault(key, filler) diff --git a/tests/test_policy_traffic_shape.py b/tests/test_policy_traffic_shape.py new file mode 100644 index 0000000..789244f --- /dev/null +++ b/tests/test_policy_traffic_shape.py @@ -0,0 +1,132 @@ +""" +Netwise -- the policy loader refuses a traffic description Batfish cannot be +asked (#442 review, #467). + +WHY THIS FILE EXISTS + `policy_compliance` queries and `access_control` headers go straight to + Batfish's HeaderConstraints, which accepts almost anything locally. So a + rule the check could not use loaded fine and failed only at scan time, + as "could not check" with an error the user could not act on. Measured on + main 2833a1e, through load_policy() and a real scan: + + queries: [] -> PC-051 "this rule has no queries" + queries: "10.10.10.0/24" -> PC-051 "13 of 13 queries could not run" + (one query per CHARACTER) + queries: ["10.10.10.0/24"] -> PC-051 TypeError from pybatfish + {"srcIp": ...} (typo) -> PC-051 TypeError, unexpected keyword + {"srcIps": "10.10.10.300"} -> PC-051 "Work terminated abnormally" + + Each is now refused at load, naming the rule and what is wrong. + + The other direction matters as much. Batfish's address language is large + -- ranges, wildcards, set difference -- and refusing a valid rule would be + worse than the problem. Only an element that is plainly meant as one + address or prefix is checked; the live test at the bottom proves each + accepted form really runs in Batfish. + +RUN + pytest tests/test_policy_traffic_shape.py -v +""" + +import json +from pathlib import Path + +import pytest + +from analysis import pipeline +from analysis.policy import PolicyError, load_policy +from conftest import needs_batfish + +FIXTURES = Path(__file__).parent / "fixtures" + + +def compliance(queries): + return {"device": "rtr-us5", "policy_compliance": [{ + "number": 1, "description": "x", "kind": "prohibition", "filter": "acl_in", + "violation_severity": "high", "violation_summary": "y", "queries": queries}]} + + +def access(headers): + return {"device": "rtr-us5", "access_control": [{ + "description": "x", "filter": "acl_in", "expected": "DENY", + "violation_severity": "high", "violation_summary": "y", "headers": headers}]} + + +@pytest.mark.parametrize("queries,says", [ + ([], "non-empty list"), + ("10.10.10.0/24", "non-empty list"), + (["10.10.10.0/24"], "expected a mapping"), + ([{"srcIp": "10.10.10.0/24"}], "unknown traffic field(s) 'srcIp'"), + ([{"srcIps": "10.10.10.300"}], "srcIps '10.10.10.300' is not a valid address"), + ([{"dstIps": "10.20.0.5, 999.1.1.1"}], "dstIps '999.1.1.1' is not a valid address"), + ([{"srcIps": "10.10.10.0/24"}, "oops"], "query 2"), +]) +def test_a_query_batfish_cannot_be_asked_is_refused_by_name(queries, says): + with pytest.raises(PolicyError, match="query|queries") as raised: + load_policy(compliance(queries)) + assert says in str(raised.value) + assert "entry 1 ('x')" in str(raised.value) + + +@pytest.mark.parametrize("headers,says", [ + ("10.10.10.0/24", "expected a mapping"), + ({"source": "10.10.10.0/24"}, "unknown traffic field(s) 'source'"), + ({"dstIps": "10.20.0.500"}, "not a valid address"), +]) +def test_access_control_headers_get_the_same_check(headers, says): + with pytest.raises(PolicyError) as raised: + load_policy(access(headers)) + assert says in str(raised.value) and "'headers'" in str(raised.value) + + +@pytest.mark.parametrize("spec", [ + {}, # all traffic: a rule can mean it + {"srcIps": "10.10.10.0/24", "dstIps": "10.20.0.5"}, + {"srcIps": "10.0.0.1, 10.0.0.2"}, # a plain list + {"srcIps": "10.10.10.1-10.10.10.9"}, # a range + {"srcIps": "10.10.10.0:0.0.0.255"}, # a wildcard + {"dstIps": "0.0.0.0/0 \\ (218.8.104.58, 10.20.0.5)"}, # set difference, as POL-1 + {"ipProtocols": ["!tcp"], "dstPorts": "0-442,444-65535"}, + {"applications": ["dns"]}, +]) +def test_every_form_batfish_accepts_still_loads(spec): + assert load_policy(compliance([spec])).entries_for("policy_compliance")[0]["queries"] == [spec] + assert load_policy(access(spec)).entries_for("access_control")[0]["headers"] == spec + + +def test_the_shipped_example_still_loads(): + example = Path(__file__).resolve().parents[1] / "docs" / "examples" / "policy.example.json" + assert load_policy(json.loads(example.read_text())).entries_for("policy_compliance") + + +@needs_batfish +@pytest.mark.parametrize("spec", [ + {"srcIps": "10.10.10.1-10.10.10.9", "dstIps": "10.20.0.5"}, + {"srcIps": "10.10.10.0:0.0.0.255", "dstIps": "10.20.0.5"}, + {"srcIps": "10.10.10.0/24", "dstIps": "0.0.0.0/0 \\ (218.8.104.58, 10.20.0.5)"}, +]) +def test_live_each_accepted_form_really_runs_in_batfish(spec): + """No false refusal: what the loader lets through, Batfish can answer.""" + results = [f for f in pipeline.analyse(FIXTURES / "rtr-us5-secure", + check_names=["policy_compliance"], + policy=load_policy(compliance([spec]))) + if f["check"] == "policy_compliance"] + assert "error" not in {f["status"] for f in results}, [f["evidence"]["detail"] for f in results] + + +@needs_batfish +def test_live_the_refused_address_really_fails_in_batfish(): + """And the one refused: Batfish itself cannot use it. Built past the + loader, the way main used to let it through.""" + from analysis.checks import policy_compliance + rule = dict(compliance([{"srcIps": "10.10.10.300", "dstIps": "10.20.0.5"}])["policy_compliance"][0], + node="rtr-us5") + original = policy_compliance.POLICY_RULES + policy_compliance.POLICY_RULES = [rule] + try: + results = [f for f in pipeline.analyse(FIXTURES / "rtr-us5-secure", + check_names=["policy_compliance"]) + if f["check"] == "policy_compliance"] + finally: + policy_compliance.POLICY_RULES = original + assert [f["status"] for f in results] == ["error"] diff --git a/tests/test_session_isolation.py b/tests/test_session_isolation.py index c044584..7d88ee3 100644 --- a/tests/test_session_isolation.py +++ b/tests/test_session_isolation.py @@ -283,7 +283,7 @@ def test_a_policy_uploaded_by_one_session_is_not_staged_for_another(analysed): "description": "d", "filter": "acl_in", "kind": "prohibition", - "queries": [], + "queries": [{"srcIps": "10.0.0.0/8"}], "violation_severity": "high", "violation_summary": "s", } From e99412be8cff8a7f83a12bf3dc0981cb1acc05bf Mon Sep 17 00:00:00 2001 From: Shubham Date: Mon, 16 Nov 2026 13:41:24 +1300 Subject: [PATCH 2/2] Say what is wrong with 'queries' in its own words (#468 review) For null or a string, the message ended 'A rule with no queries asks Batfish nothing', which reads oddly when the rule does have something under 'queries'. A non-list and an empty list now get their own messages, each pinned by a test, with null added to the cases. --- analysis/policy.py | 10 +++++++--- tests/test_policy_traffic_shape.py | 5 +++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/analysis/policy.py b/analysis/policy.py index d7123db..3476872 100644 --- a/analysis/policy.py +++ b/analysis/policy.py @@ -476,10 +476,14 @@ def _validate_entry( # queries, one per character. Refused here, naming the rule, instead. if section == "policy_compliance" and "queries" in normalised: queries = normalised["queries"] - if not isinstance(queries, list) or not queries: + if not isinstance(queries, list): raise PolicyError( - f"{where}: 'queries' must be a non-empty list of traffic descriptions, " - f"got {queries!r}. A rule with no queries asks Batfish nothing" + f"{where}: 'queries' must be a list of traffic descriptions, " + f"got {type(queries).__name__} {queries!r}" + ) + if not queries: + raise PolicyError( + f"{where}: 'queries' is empty. A rule with no queries asks Batfish nothing" ) for number, query in enumerate(queries, start=1): _check_traffic(f"{where}, query {number}", query) diff --git a/tests/test_policy_traffic_shape.py b/tests/test_policy_traffic_shape.py index 789244f..2f5f3f5 100644 --- a/tests/test_policy_traffic_shape.py +++ b/tests/test_policy_traffic_shape.py @@ -53,8 +53,9 @@ def access(headers): @pytest.mark.parametrize("queries,says", [ - ([], "non-empty list"), - ("10.10.10.0/24", "non-empty list"), + ([], "'queries' is empty"), + ("10.10.10.0/24", "must be a list of traffic descriptions, got str"), + (None, "must be a list of traffic descriptions, got NoneType"), (["10.10.10.0/24"], "expected a mapping"), ([{"srcIp": "10.10.10.0/24"}], "unknown traffic field(s) 'srcIp'"), ([{"srcIps": "10.10.10.300"}], "srcIps '10.10.10.300' is not a valid address"),