From 0aff2c27fe8d907d4fb5130a027ac6e3850b77cc Mon Sep 17 00:00:00 2001 From: ARSH871-bot Date: Tue, 3 Nov 2026 23:34:38 +1300 Subject: [PATCH] Refuse input files Netwise used to crash on or read wrongly Found in a stability audit (21-30 Oct), re-measured on main db59b00. Policy and business-context files: - a key written twice is refused; json.loads used to keep the last one, so a policy rule (or a "critical" tier) was silently dropped and the upload still returned 200 - non-UTF-8 text, nesting past the recursion limit, and integers too long to parse are 400s naming the file, not 500s; no second encoding is guessed PF Sense converter: - an interface tag that is not a plain name is refused; a namespaced tag carried free text into the ACL name and injected "permit ip any any" ahead of the real rules - a rule is left out, as pfSense itself does (filter.inc:5543); it used to be converted as live - rules with (filter.inc:4249), IPv6-only rules (4333) and ICMP rules limited to certain types (4522) are skipped and named, not converted as something they are not The list of what the converter refuses is updated where it is stated: CLAUDE.md section 7, CLAUDE.md's status table, and the README. --- CLAUDE.md | 8 ++- README.md | 6 +- analysis/business_context.py | 40 +++++++++++- analysis/pfsense_convert.py | 63 +++++++++++++++++-- analysis/policy.py | 43 +++++++++++-- tests/test_business_context.py | 23 +++++++ tests/test_business_context_wiring.py | 24 +++++++ tests/test_pfsense_convert.py | 91 +++++++++++++++++++++++++++ tests/test_policy_loader.py | 32 ++++++++++ tests/test_web_policy_upload.py | 29 +++++++++ 10 files changed, 343 insertions(+), 16 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 6e89dad..89eb1b1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -117,8 +117,10 @@ Batfish runs in Docker container `batfish` (image `batfish/allinone`), exposing - The client's real firewall is **PF Sense**, which exports XML that Batfish cannot read at all. `analysis/pfsense_convert.py` translates it into Cisco IOS text, so a converted config re-enters the same `analyse()` as everything - else. Interfaces and filter rules are covered; NAT, aliases, DHCP, VPN, IPv6 - and combined `tcp/udp` rules are not, and each raises rather than guessing. + else. Interfaces and filter rules are covered; NAT, aliases, DHCP, VPN, IPv6, + combined `tcp/udp`, negated (``) addresses and ICMP-type rules are + not, and each raises rather than guessing. A disabled rule is left out, + exactly as pfSense itself leaves it out. **Rule order: PF Sense and Cisco disagree, and the converter now MODELS the difference where it can and refuses where it cannot.** PF Sense evaluates @@ -722,7 +724,7 @@ project's recurring failure family arriving through process rather than code. | **AI explanation on screen** | Ankeet + Samika | Done (#56, closing #31). `/api/findings` attaches an `explanation` to every `status="found"` finding, **not** an F-1 field, added downstream of validation so the contract is untouched. Since #109/#143, also attaches `explanation_source` (`"model"` or `"fallback"`), same reasoning, so the dashboard can eventually stop labelling deterministic fallback text as AI-authored, which it still does today (#109 stays open until the byline itself changes) | | **`risk` scoring** | Samika | Done (#60) — `POST_PROCESSORS`, ruleset in `docs/severity-rules.md`. Re-rates severity and sorts worst-first; the two limits are enforced in `pipeline.run_post_processors()`, not trusted | | Dashboard + secure upload | Samika | Done — real findings on screen since #39 | -| PF Sense conversion | Ankeet | Done — `analysis/pfsense_convert.py`, and **hardened**: refuses config injection and path traversal via free-text fields (#53), an unbound ACL / empty rule set / unvalidated addressing (#54), and ambiguous rule order (#58, closing #47). See §7 | +| PF Sense conversion | Ankeet | Done — `analysis/pfsense_convert.py`, and **hardened**: refuses config injection and path traversal via free-text fields (#53), an unbound ACL / empty rule set / unvalidated addressing (#54), and ambiguous rule order (#58, closing #47). Also refuses an interface tag that is not a plain name (a namespaced tag injected config lines), and, per rule, ``, IPv6-only and ICMP-type rules; disabled rules are left out, as pfSense does. See §7 | | **`change_impact`** | Shubham | Done (#140, merged 17 August) — `analysis/change_impact.py`, `analyse_change(before, after)`. Shape C: **not** in `CHECKS`, and both the module docstring and the registry comment say so. Pairs `compareFilters` (which ACL *lines* moved) with `differentialReachability` (which *traffic* changed fate), because a changed line that moves no traffic is noise and moved traffic with no changed line is the case a filter diff alone misses. Direction-aware per `docs/policy-rules.md`: opening `high`, tightening `medium`, both reported | | **A user policy reaching a check** | Arsh | Done (#181, merged 28 August) — `analysis/policy.py` loads and validates it; `policy_compliance` reads it via `rules_in_use()`, keyed on `active_policy() is None` rather than on emptiness. Measured below: policy-driven detections on a stranger's network go 3 → 8 | | **Draft a policy from your own config** | Arsh | Done (#326) — `analysis/draft_policy.py`, `/api/policy/draft`, a button under the policy picker. One `policy_compliance` rule per line of each applied filter, citing the file and line Batfish records for it. Each rule is checked with the same `searchFilters` question the check will ask, so a line an earlier line partly decides is listed as not drafted, with Batfish's example packet, rather than copied. `kind` and `violation_severity` are left as ``, and `analysis/policy.py` refuses the file until they are decided and its `draft` block is deleted: accepted as it stands, a draft passes on `rtr-us5-insecure` | diff --git a/README.md b/README.md index 1878e9b..20d2a76 100644 --- a/README.md +++ b/README.md @@ -124,8 +124,10 @@ permitted. Those remain ours. its own ACL, checked independently — and it **models** PF Sense's last-match-wins order rather than refusing it wherever it can do so exactly (#104). It **refuses rather than guesses** on NAT, aliases, VPN, IPv6, - interfaces with no static address, interfaces named by a rule but never - declared, and rule orders it cannot model exactly. A real client export is + negated (``) addresses, ICMP-type rules, interface tags that are not + plain names, interfaces with no static address, interfaces named by a rule + but never declared, and rule orders it cannot model exactly. A disabled + rule is left out, as pfSense itself does. A real client export is still refused, on the last two of those. `analysis/pfsense_convert.REFUSALS` is the single source of truth for exactly what is refused and why (#155). diff --git a/analysis/business_context.py b/analysis/business_context.py index eca110e..30fb48a 100644 --- a/analysis/business_context.py +++ b/analysis/business_context.py @@ -334,17 +334,51 @@ def load_business_context_file(path: Any) -> BusinessContext: if not file_path.exists(): raise BusinessContextError(f"no business context file at {file_path}") - text = file_path.read_text(encoding="utf-8").strip() + try: + text = file_path.read_text(encoding="utf-8").strip() + except UnicodeDecodeError as error: + raise BusinessContextError( + f"{file_path} is not UTF-8 text: byte {error.start} cannot be " + f"decoded. Save the file as UTF-8 and try again" + ) from error if not text: # An empty FILE is an empty CONTEXT, which is valid. return BusinessContext() + # Same three escapes, and the same fix, as policy.load_policy_file(). try: - parsed = json.loads(text) + parsed = json.loads(text, object_pairs_hook=_refuse_duplicate_keys) + return load_business_context(parsed) except json.JSONDecodeError as error: raise BusinessContextError( f"{file_path} is not valid JSON: line {error.lineno}, " f"column {error.colno}: {error.msg}" ) from error + except BusinessContextError: + raise + except RecursionError as error: + raise BusinessContextError( + f"{file_path} is nested too deeply to be a business context" + ) from error + except ValueError as error: + raise BusinessContextError( + f"{file_path} holds a value that cannot be read: {error}" + ) from error + + +def _refuse_duplicate_keys(pairs: List[Tuple[str, Any]]) -> Dict[str, Any]: + """Build one JSON object, refusing a key that appears twice in it. - return load_business_context(parsed) + Mirrors policy._refuse_duplicate_keys(). json.loads keeps the LAST of two + equal keys: `{"device": "rtr-us5", "tier": "critical", "tier": + "standard"}` loaded as standard, the user's "critical" dropped silently. + """ + built: Dict[str, Any] = {} + for key, value in pairs: + if key in built: + raise BusinessContextError( + f"the key {key!r} appears twice in one entry. JSON keeps only " + f"the last, so the first would be silently ignored" + ) + built[key] = value + return built diff --git a/analysis/pfsense_convert.py b/analysis/pfsense_convert.py index b80dd2a..d4bc636 100644 --- a/analysis/pfsense_convert.py +++ b/analysis/pfsense_convert.py @@ -178,6 +178,10 @@ "injection or path traversal" ), "no_interfaces_section": "the export has no section at all", + "invalid_interface_role": ( + "an interface's tag under is not a plain name like " + "'lan' or 'opt1', and it is written into the generated config" + ), "invalid_interface_address": ( "an interface's / do not form a valid IPv4 network" ), @@ -205,6 +209,18 @@ "like 'tcp/udp' need two Cisco ACL lines and are not handled" ), "missing_source_or_destination": "a rule has no or ", + "negated_endpoint": ( + "a or carries , meaning everything " + "EXCEPT that address -- one Cisco ACL line cannot express 'except'" + ), + "ipv6_rule": ( + "an IPv6-only rule (inet6) -- this module models IPv4 " + "only, so the rule's IPv6 traffic is not analysed" + ), + "icmp_type": ( + "an ICMP rule limited to particular ICMP types -- this module does " + "not translate pfSense's ICMP type names" + ), "invalid_port": ( "a rule's is not a single numeric port -- named aliases and " "ranges are out of scope" @@ -357,6 +373,10 @@ def _sanitised_hostname(text: str) -> str: return text +# pfSense's own interface keys: wan, lan, opt1, ... -- see _parse_interfaces(). +_ROLE_PATTERN = re.compile(r"[A-Za-z0-9_-]+") + + def _parse_interfaces(root: ET.Element) -> Dict[str, Dict[str, str]]: """Return {role: {"if": ..., "ipaddr": ..., "subnet": ..., "descr": ...}} for every interface PF Sense defines, in document order. @@ -373,6 +393,16 @@ def _parse_interfaces(root: ET.Element) -> Dict[str, Dict[str, str]]: interfaces: Dict[str, Dict[str, str]] = {} for iface_el in interfaces_el: role = iface_el.tag + # The role becomes part of an ACL name in the emitted config (see + # _acl_name()), so it gets the same allowlist treatment as + # . A tag name looks safe, but ElementTree reports a + # namespaced tag as "{namespace}name", and the namespace is free + # text: `xmlns:x=" ip access-list extended acl_in permit ip + # any any"` on put that permit line at the top of acl_in. + if not _ROLE_PATTERN.fullmatch(role): + raise PfSenseConversionError( + f"{REFUSALS['invalid_interface_role']}: {role!r}" + ) ipaddr = _text(iface_el, "ipaddr") subnet = _text(iface_el, "subnet") if not ipaddr or not subnet: @@ -516,10 +546,9 @@ def _acl_name(role: str, *, rule_bearing_roles: "set[str]") -> str: the same convention _assign_cisco_interface_names() already uses for interface names, rather than a second naming scheme. - `role` is an XML tag name (see _parse_interfaces()), not free text pulled - from an element's content the way or are, so it is - already constrained by XML's own tag-name grammar and does not need the - separate validation those fields do. + `role` is an XML tag name, validated against _ROLE_PATTERN in + _parse_interfaces(). This docstring used to say a tag name was already + safe by XML's own grammar; a namespaced tag is not -- see there. """ if role in rule_bearing_roles and len(rule_bearing_roles) == 1: return ACL_NAME @@ -561,6 +590,14 @@ def _resolve_endpoint(el: ET.Element, interfaces: Dict[str, Dict[str, str]]) -> if el.find("any") is not None: return "any" + # inverts the match: "everything EXCEPT this" (filter.inc:4249). + # It used to be ignored, so the emitted line matched exactly the traffic + # the rule exempts and none of the traffic it covers. One Cisco line + # cannot say "except", so the rule is refused. After the check on + # purpose: pfSense ignores beside too (filter.inc:4103). + if el.find("not") is not None: + raise PfSenseConversionError(REFUSALS["negated_endpoint"]) + network = _text(el, "network") if network is not None: if network not in interfaces: @@ -755,6 +792,18 @@ def _rule_to_acl_line(rule_el: ET.Element, interfaces: Dict[str, Dict[str, str]] f"{pf_protocol!r}" ) + # Two fields that make pfSense match LESS than the fields above say, and + # that used to be ignored (filter.inc at 480a1c0626). Ignored, a block + # rule blocked more in the model than on the firewall: a stricter ACL, + # the direction that turns a real violation into a clean result (#302). + if _text(rule_el, "ipprotocol") == "inet6": # filter.inc:4333 + raise PfSenseConversionError(REFUSALS["ipv6_rule"]) + icmp_type = _text(rule_el, "icmptype") + if pf_protocol == "icmp" and icmp_type not in (None, "any"): # filter.inc:4522 + raise PfSenseConversionError( + f"{REFUSALS['icmp_type']}: is {icmp_type!r}" + ) + source_el = rule_el.find("source") destination_el = rule_el.find("destination") if source_el is None or destination_el is None: @@ -848,6 +897,12 @@ def convert(xml_path: Union[str, Path]) -> ConversionResult: filter_el = root.find("filter") rule_els = filter_el.findall("rule") if filter_el is not None else [] + # A disabled rule is not loaded by pfSense at all (filter.inc:5543), so + # leaving it out is the exact model, not a gap -- nothing is skipped. + # It used to be converted as live: a disabled block denied traffic in + # the model that the real firewall lets through. + rule_els = [r for r in rule_els if r.find("disabled") is None] + # WHY AN EMPTY RULE SET IS REFUSED, NOT CONVERTED AS "NO ACL" # Found in a senior-level adversarial QA pass. PF Sense fails CLOSED # with no rules configured on an interface -- it blocks everything. diff --git a/analysis/policy.py b/analysis/policy.py index 50d15bd..6209d4c 100644 --- a/analysis/policy.py +++ b/analysis/policy.py @@ -699,20 +699,55 @@ def load_policy_file(path: Any) -> Policy: if not file_path.exists(): raise PolicyError(f"no policy file at {file_path}") - text = file_path.read_text(encoding="utf-8").strip() + try: + text = file_path.read_text(encoding="utf-8").strip() + except UnicodeDecodeError as error: + raise PolicyError( + f"{file_path} is not UTF-8 text: byte {error.start} cannot be " + f"decoded. Save the policy as UTF-8 and try again" + ) from error if not text: # An empty FILE is an empty POLICY, which is valid and loud (D4). return Policy({section: [] for section in POLICY_SECTIONS}) + # Three more ways a file used to escape as a raw exception -- an HTTP 500 + # on upload rather than a refusal naming the file: an integer of more + # than 4,300 digits (ValueError), and nesting deeper than Python's + # recursion limit, hit by json.loads or by load_policy() walking it. try: - parsed = json.loads(text) + parsed = json.loads(text, object_pairs_hook=_refuse_duplicate_keys) + return load_policy(parsed) except json.JSONDecodeError as error: raise PolicyError( f"{file_path} is not valid JSON: line {error.lineno}, " f"column {error.colno}: {error.msg}" ) from error - - return load_policy(parsed) + except PolicyError: + raise + except RecursionError as error: + raise PolicyError(f"{file_path} is nested too deeply to be a policy") from error + except ValueError as error: + raise PolicyError(f"{file_path} holds a value that cannot be read: {error}") from error + + +def _refuse_duplicate_keys(pairs: List[Tuple[str, Any]]) -> Dict[str, Any]: + """Build one JSON object, refusing a key that appears twice in it. + + json.loads keeps the LAST of two equal keys and drops the first without a + word. Measured: `{"policy_compliance": [], "policy_compliance": + []}` loaded as an EMPTY policy, and the upload said so with HTTP 200 -- + the user's rule gone. An unknown key is refused (D5) because a key nobody + reads is a rule nobody enforces; a key written twice is the same failure. + """ + built: Dict[str, Any] = {} + for key, value in pairs: + if key in built: + raise PolicyError( + f"the key {key!r} appears twice in one object. JSON keeps only " + f"the last, so the first would be silently ignored" + ) + built[key] = value + return built # ------------------------------------------------------------------------------ diff --git a/tests/test_business_context.py b/tests/test_business_context.py index 6d9429e..3635fd8 100644 --- a/tests/test_business_context.py +++ b/tests/test_business_context.py @@ -383,6 +383,29 @@ def test_malformed_json_names_the_line_and_column(tmp_path): assert "line" in message and "column" in message +@pytest.mark.parametrize( + "contents, said", + [ + # Loaded as "standard" before the fix: the user's "critical" dropped. + pytest.param(b'[{"device": "rtr-us5", "tier": "critical", "tier": "standard"}]', + "appears twice", id="duplicate-key"), + pytest.param(b'[{"device": "r\xe9", "tier": "critical"}]', "not UTF-8", id="not-utf8"), + pytest.param(b'[{"device": "r", "tier": 1' + b"0" * 5000 + b"}]", "cannot be read", + id="5000-digit-int"), + pytest.param(b"[" * 100000 + b"]" * 100000, "nested too deeply", id="deep-json"), + ], +) +def test_a_file_json_cannot_hold_faithfully_is_refused_not_crashed_or_trimmed( + tmp_path, contents, said +): + """Mirrors the policy loader's test: a raw exception (HTTP 500 on + upload), or a silently dropped value, before the fix.""" + path = tmp_path / "business-context.json" + path.write_bytes(contents) + with pytest.raises(BusinessContextError, match=said): + load_business_context_file(path) + + def test_a_real_file_round_trips(tmp_path): path = tmp_path / "business-context.json" path.write_text( diff --git a/tests/test_business_context_wiring.py b/tests/test_business_context_wiring.py index 35ddbb9..1e550ff 100644 --- a/tests/test_business_context_wiring.py +++ b/tests/test_business_context_wiring.py @@ -227,6 +227,30 @@ def test_an_invalid_context_is_rejected_with_the_loaders_own_message(body, expec assert expected in response.json()["detail"] +@pytest.mark.parametrize( + "raw, expected", + [ + # PowerShell 5.1's `>` and Notepad's "Unicode" both save UTF-16. + (json.dumps([{"device": "rtr-us5", "tier": "critical"}]).encode("utf-16"), + "not UTF-8"), + (json.dumps([{"device": "rtr-café", "tier": "critical"}], + ensure_ascii=False).encode("latin-1"), + "not UTF-8"), + (b"[" * 200_000 + b"]" * 200_000, "nested too deeply"), + ], + ids=["utf-16", "latin-1", "deep-nesting"], +) +def test_a_file_the_loader_cannot_decode_is_a_400_not_a_500(raw, expected): + """Measured on origin/main: all three were HTTP 500 Internal Server + Error, the same escape as the policy loader's (see + test_web_policy_upload.py). Refused, never guessed at.""" + response = _post_context(raw) + + assert response.status_code == 400 + assert expected in response.json()["detail"] + assert not main.BUSINESS_CONTEXT_PATH.exists() + + def test_a_rejected_context_is_never_staged(): """The safety property the validate-then-stage order exists for.""" _post_context([{"device": "rtr-us5", "tier": "critical"}]) diff --git a/tests/test_pfsense_convert.py b/tests/test_pfsense_convert.py index cfa978a..099d295 100644 --- a/tests/test_pfsense_convert.py +++ b/tests/test_pfsense_convert.py @@ -987,6 +987,74 @@ def test_a_whitespace_only_interface_gets_the_right_refusal_message(): ) +# --- Fields that change what a rule matches (pfSense filter.inc at 480a1c0626) --- +# Each used to be ignored, so the rule converted as if it matched more traffic +# than it does. Every rule below is , so the order model is not in play. + + +def _acl_lines(text: str) -> list: + return [ln.strip() for ln in text.splitlines() if ln.strip().startswith(("permit", "deny"))] + + +def test_a_disabled_rule_is_left_out_as_pfsense_does(): + """Measured before the fix: a disabled catch-all block came out as the + first line of acl_in, so Batfish denied LAN->8.8.8.8:80, which the real + firewall permits (it does not load a disabled rule, filter.inc:5543).""" + result = _convert_string_full(_minimal_xml(rules_xml=""" + blocklan + + passlan + lan + """)) + assert _acl_lines(result.text) == ["permit ip 10.0.0.0 0.0.0.255 any"] + assert result.skipped == [], "an exact model of a disabled rule is not a gap" + + +def test_a_negated_destination_is_refused_not_inverted(): + """ means "every destination EXCEPT 10.20.0.5". Before the fix it + emitted `permit tcp ... host 10.20.0.5`: Batfish permitted exactly the + host the rule exempts and denied the traffic it allows.""" + result = _convert_string_full(_minimal_xml(rules_xml=""" + passlantcp + lan +
10.20.0.5
+ blocklan + + """)) + assert _acl_lines(result.text) == ["deny ip any any"] + assert len(result.skipped) == 1 + assert REFUSALS["negated_endpoint"] in result.skipped[0] + + +def test_an_ipv6_only_rule_is_not_applied_to_ipv4_traffic(): + """An inet6 block on the LAN subnet came out as an IPv4 `deny ip + 10.0.0.0 0.0.0.255 any` ahead of the IPv4 pass, so every IPv4 flow from + LAN was denied in the model and permitted by the firewall.""" + result = _convert_string_full(_minimal_xml(rules_xml=""" + blocklaninet6 + lan + passlaninet + lan + """)) + assert _acl_lines(result.text) == ["permit ip 10.0.0.0 0.0.0.255 any"] + assert len(result.skipped) == 1 + assert REFUSALS["ipv6_rule"] in result.skipped[0] + + +def test_an_icmp_type_limit_is_refused_not_widened_to_all_icmp(): + result = _convert_string_full(_minimal_xml(rules_xml=""" + blocklanicmp + echoreq + + passlanicmp + any + + """)) + assert _acl_lines(result.text) == ["permit icmp any any"] + assert len(result.skipped) == 1 + assert REFUSALS["icmp_type"] in result.skipped[0] + + # --- Rule order is preserved ----------------------------------------------------- @@ -1164,6 +1232,29 @@ def test_if_with_an_embedded_newline_also_raises(): _convert_string(xml_text) +def test_a_namespaced_interface_tag_cannot_inject_config_lines(): + """The interface ROLE is a tag name, and ElementTree reports a namespaced + tag as "{namespace}name" -- free text, newlines included. Before the fix + this emitted `permit ip any any` as the first line of acl_in, and + Batfish's testFilters on the LAN ACL answered PERMIT for every flow, + against a rule set whose only rule is a deny.""" + xml_text = _minimal_xml( + interfaces_xml=""" + em1LAN10.0.0.124 + + em2OPT110.30.0.124 + + """, + rules_xml=""" + blocklan + + """, + ) + with pytest.raises(PfSenseConversionError) as excinfo: + _convert_string(xml_text) + assert REFUSALS["invalid_interface_role"] in str(excinfo.value) + + def test_a_hostname_with_ordinary_punctuation_still_converts(): """The fix must not become a blanket ban -- real PF Sense hostnames can contain dots and hyphens (e.g. "fw-01.branch.example"), and that must diff --git a/tests/test_policy_loader.py b/tests/test_policy_loader.py index 8ffa184..985189e 100644 --- a/tests/test_policy_loader.py +++ b/tests/test_policy_loader.py @@ -210,6 +210,38 @@ def test_an_empty_file_is_an_empty_policy_not_a_parse_error(tmp_path): assert load_policy_file(path).is_empty +_A_RULE = (b'{"description": "DNS allowed", "node": "rtr-us5", "filter": "acl_in", ' + b'"kind": "requirement", "queries": [{"dstIps": "218.8.104.58"}], ' + b'"violation_severity": "high", "violation_summary": "DNS is blocked"}') + + +@pytest.mark.parametrize( + "contents, said", + [ + # The rule is silently dropped by json.loads: the policy loaded EMPTY. + pytest.param(b'{"policy_compliance": [' + _A_RULE + b'], "policy_compliance": []}', + "appears twice", id="duplicate-key"), + pytest.param(b'{"device": "r\xe9", "policy_compliance": []}', "not UTF-8", id="not-utf8"), + pytest.param(b'{"policy_compliance": [{"number": 1' + b"0" * 5000 + b"}]}", + "cannot be read", id="5000-digit-int"), + pytest.param(b"[" * 100000 + b"]" * 100000, "nested too deeply", id="deep-json"), + # json.loads manages this one; the validation walking it did not. + pytest.param(b'{"policy_compliance": [' + b"[" * 2000 + b"]" * 2000 + b"]}", + "nested too deeply", id="deep-validation"), + ], +) +def test_a_file_json_cannot_hold_faithfully_is_refused_not_crashed_or_trimmed( + tmp_path, contents, said +): + """Each of these escaped as a raw UnicodeDecodeError, ValueError or + RecursionError (HTTP 500 on upload), or -- the duplicate key -- loaded + with the user's rule gone and HTTP 200.""" + path = tmp_path / "policy.json" + path.write_bytes(contents) + with pytest.raises(PolicyError, match=said): + load_policy_file(path) + + # --------------------------------------------------------------------------- # D2 -- per entry, with an optional top-level default # --------------------------------------------------------------------------- diff --git a/tests/test_web_policy_upload.py b/tests/test_web_policy_upload.py index c4d8a94..77e36c7 100644 --- a/tests/test_web_policy_upload.py +++ b/tests/test_web_policy_upload.py @@ -330,6 +330,12 @@ def test_an_empty_upload_is_rejected_rather_than_read_as_asserting_nothing(): {"policy_compliance": [{"filter": "acl_in"}]}, "missing the required description/node", ), + # The next three were an HTTP 500 (raw exception) or, the last, a 200 + # with the rule silently dropped, until the loader refused them. + pytest.param(b'{"device": "r\xe9"}', "not UTF-8", id="not-utf8"), + pytest.param(b"[" * 100000 + b"]" * 100000, "nested too deeply", id="deep"), + pytest.param(b'{"routing": [{"description": "x"}], "routing": []}', + "duplicate key", id="duplicate-key"), ], ) def test_a_rejected_policy_is_never_staged(bad_body, why): @@ -351,6 +357,29 @@ def test_a_rejected_policy_is_never_staged(bad_body, why): ) +@pytest.mark.parametrize( + "raw, expected", + [ + # PowerShell 5.1's `>` and Notepad's "Unicode" both save UTF-16. + (json.dumps(VALID_POLICY).encode("utf-16"), "not UTF-8"), + (json.dumps(VALID_POLICY).replace("rtr-us5", "rtr-café").encode("latin-1"), + "not UTF-8"), + (b"[" * 200_000 + b"]" * 200_000, "nested too deeply"), + ], + ids=["utf-16", "latin-1", "deep-nesting"], +) +def test_a_file_the_loader_cannot_decode_is_a_400_not_a_500(raw, expected): + """Measured on origin/main: all three were HTTP 500 Internal Server + Error, because `UnicodeDecodeError` and `RecursionError` escaped the + loader instead of becoming a `PolicyError` the endpoint already turns + into a message. Refused, never guessed at: no other encoding is tried.""" + response = _post_policy(raw) + + assert response.status_code == 400 + assert expected in response.json()["detail"] + assert not main.POLICY_PATH.exists() + + def test_a_rejection_leaves_an_earlier_good_policy_alone(): """A bad upload must not destroy the policy already staged.