diff --git a/analysis/checks/risk.py b/analysis/checks/risk.py index f86fe56..334d4c9 100644 --- a/analysis/checks/risk.py +++ b/analysis/checks/risk.py @@ -55,18 +55,57 @@ def _evidence(finding: Dict[str, Any]) -> str: return str(evidence.get("detail", "")).lower() -def _rule_dead_rule_is_low(finding: Dict[str, Any]) -> Optional[str]: - """R-1: a line that can never match cannot expose anything. +# The marker access_control._check_dead_rules() writes after the dead line: +# +# "Unreachable line: (action PERMIT|DENY). Blocked by: . ..." +# +# Our own fixed wording, not config text, and the same shape ai/explain.py +# reads. The first "(action ...)." before "Blocked by:" is ours; the line +# text before it comes from Batfish and never carries one. +# +# R-1 matches the START of the evidence, not the word anywhere (#444). Every +# other producer opens with its own fixed words ("Expected", "Flow", +# "Example permitted flow", "Referenced as"), while "unreachable" turns up in +# routing assertions, Batfish dispositions and Cisco ICMP keywords. +_DEAD_LINE = "unreachable line: " +_DEAD_DENY = re.compile(r"unreachable line: .+?\(action deny\)\. blocked by:") + - FIRST ON PURPOSE. A dead-rule finding normally quotes the line that shadows - it, and that line is very often a blanket permit -- so if R-2 ran first it - would match the quoted BLOCKER and rate the dead rule "high". The exposure - is not lost by rating this low: the blanket permit is reported separately on - its own finding, where R-2 rates it high. +def _rule_dead_rule_is_low(finding: Dict[str, Any]) -> Optional[str]: + """R-1: a dead PERMIT is low. A dead DENY keeps its check's severity. + + A permit that can never match blocks traffic someone wanted: a hygiene + problem, worth reporting, not an exposure. A deny that can never match is + the opposite -- traffic someone meant to block gets through -- and + access_control already grades it high for that reason. + + It used to lower both, on the grounds that the exposure "is reported + separately by R-2". That only holds when the line shadowing the deny is + a blanket permit. Measured in docs/research/cvss-comparison.md section 4 + (#434): a deny shadowed by a SPECIFIC permit went to low, R-2 did not + fire, and the only finding about that traffic sat at the bottom. + + Only a clear "(action DENY)" marker is left alone. A dead-line finding + without one keeps the old answer, low. + + ONLY A DEAD ACL LINE. The evidence must start with "Unreachable line: ", + as access_control writes it. It used to be the word "unreachable" + anywhere, which lowered routing isolation assertions ("Expected + UNREACHABLE ...") and any finding quoting an ICMP `unreachable` line + (#444). + + FIRST ON PURPOSE. When the dead line is itself a blanket permit, R-2 + would match it and rate a dead permit "high" if it ran first. The + blocker is not the risk: it follows "Blocked by:", so R-2 treats it as a + citation. For a dead deny this rule passes, and R-2 does not promote it + for the same reason. """ - if "unreachable" in _evidence(finding): - return "low" - return None + detail = _evidence(finding) + if not detail.startswith(_DEAD_LINE): + return None + if _DEAD_DENY.search(detail): + return None + return "low" # The phrases our checks use to ATTRIBUTE a violation to the ACL line that diff --git a/docs/severity-rules.md b/docs/severity-rules.md index 06a7bb1..af21071 100644 --- a/docs/severity-rules.md +++ b/docs/severity-rules.md @@ -46,21 +46,48 @@ move it down the page, which is the F-4 failure by another route. Applied in order. The first that matches wins; anything unmatched keeps the severity its check assigned. -### R-1 — A dead rule is low, whatever it says +### R-1 — A dead permit is low; a dead deny keeps its check's severity -**Matches:** `status="found"` and the evidence describes an unreachable / -shadowed ACL line. +**Matches:** `status="found"`, `evidence.detail` **starts with** +`Unreachable line: ` (the dead ACL line `access_control` reports), and that +line is **not** marked `(action DENY)`. **Sets:** `low` -A line that can never match cannot expose anything. It is a hygiene problem — +A dead **permit** blocks traffic someone wanted. It is a hygiene problem — someone believes a rule is in force when it is not — and it is worth reporting, but it is not an exposure and should not sit above one. -This rule is deliberately **first**, because a dead-rule finding usually quotes -the line that shadows it, and that line is very often a blanket permit. Without -the ordering, R-2 would match the quoted blocker and rate the dead rule `high`. -The exposure is not lost by rating it low: the blanket permit is reported -separately, on its own finding, by R-2. +A dead **deny** is the opposite: traffic someone meant to block gets through. +`access_control` grades it `high` for that reason, and R-1 leaves it alone. + +**This used to lower every dead line**, on the grounds that the exposure is +reported separately by R-2. That only holds when the line shadowing the deny is +a blanket permit. Measured in `docs/research/cvss-comparison.md` section 4 +(#434): a telnet deny shadowed by a *specific* permit went to `low`, R-2 did not +fire, and the only finding about that traffic sat at the bottom of the list. +§6 item 2 below also records that R-2 has no observed effect on any fixture we +have, so R-1 should not lean on it. + +**How the action is read.** The finding has no structured field for it, so R-1 +reads the fixed marker `access_control` writes into `evidence.detail` — +*"Unreachable line: … (action DENY). Blocked by: …"*. That is our wording, not +config text. A dead-rule finding without a clear `(action DENY)` marker keeps +the old answer, `low`. + +**Only the start of the evidence counts (#444).** R-1 used to fire on the word +"unreachable" anywhere in the evidence. That lowered findings that have nothing +to do with a dead line: a routing assertion that something must be +*"Expected UNREACHABLE"*, a path Batfish ended in `NEIGHBOR_UNREACHABLE`, and +any finding quoting an ACL line with Cisco's ICMP `unreachable`, +`host-unreachable` or `port-unreachable` keyword. Every other check opens its +evidence with different fixed words, so the opening is what tells a dead line +apart. + +This rule is deliberately **first**, for a dead line that is itself a blanket +permit: R-2 would match it and rate a dead permit `high`. The line shadowing a +dead line is not the problem: it follows *"Blocked by:"*, so R-2 treats it as +a citation. A dead deny passes through R-1 and is not promoted by R-2 for the +same reason. ### R-2 — A blanket permit is high @@ -124,7 +151,9 @@ registry order, which is alphabetical by accident rather than meaningful. - **It does not weight by device.** "This router is more important than that one" is real, but we have no source for it that is not guesswork, and inventing one would be exactly the unfounded claim the rest of this project - avoids. If the client tells us, it becomes data, not a rule. + avoids. If the client tells us, it becomes data, not a rule. That data is + business context (`critical` / `important` / `standard`), applied after these + rules: our equivalent of CVSS Environmental metrics. ## 6. Open, and deliberately not settled here diff --git a/tests/test_severity_rules.py b/tests/test_severity_rules.py index 815dc3d..6d0262a 100644 --- a/tests/test_severity_rules.py +++ b/tests/test_severity_rules.py @@ -39,20 +39,169 @@ def test_dead_rule_is_rated_low(): assert risk.severity_for(f) == "low" +# The evidence exactly as access_control._check_dead_rules() writes it, so these +# tests break if that wording drifts away from what R-1 reads. +def dead_line_detail(line, action, blocked_by): + return (f"Unreachable line: {line} (action {action}). " + f"Blocked by: {blocked_by}. Reason: BLOCKING_LINES") + + def test_dead_rule_beats_the_blanket_permit_it_quotes(): """R-1 before R-2, and this is the case that makes the order load-bearing. - A dead-rule finding normally names the line that shadows it, and that line - is very often the blanket permit. If R-2 ran first it would match the - quoted BLOCKER and rate the dead rule high -- overstating a line that, by - definition, can never match anything. + The dead line here IS a blanket permit, so it is not a citation and R-2 + would match it. If R-2 ran first it would rate the dead line high -- + overstating a line that, by definition, can never match anything. + + Written in the producer's own wording since #444. It used to read + "Unreachable: ... -- blocked by: ...", which no check emits, and R-1 now + matches only what access_control actually writes. """ - f = a_finding("access_control", "medium", - "Unreachable: permit tcp 10.10.10.0 host 10.20.0.5 eq 443 " - "-- blocked by: permit ip any any") + f = a_finding("access_control", "medium", dead_line_detail( + "permit ip any any", "PERMIT", "deny ip any any")) + assert risk.severity_for(f) == "low" + + +def test_dead_deny_behind_a_specific_permit_keeps_its_checks_severity(): + """The measured case from docs/research/cvss-comparison.md section 4 (#434). + + The telnet deny never fires because the permit above it already lets the + whole LAN through to the server. So telnet from .66 gets through, which + the author meant to block. R-2 does not fire (the blocker is not a + blanket permit), so if R-1 lowers this, nothing in the list says it is + an exposure. + """ + f = a_finding("access_control", "high", dead_line_detail( + "deny tcp host 10.10.10.66 host 10.20.0.5 eq telnet", "DENY", + "permit tcp 10.10.10.0 0.0.0.255 host 10.20.0.5")) + assert risk.severity_for(f) == "high" + + +def test_dead_deny_behind_a_blanket_permit_keeps_its_checks_severity(): + """Still an exposure in its own right, and R-2 does not promote the quoted + blocker, so the check's own `high` is what has to survive.""" + f = a_finding("access_control", "high", dead_line_detail( + "deny tcp host 10.10.10.66 host 10.20.0.5 eq telnet", "DENY", + "permit ip any any")) + assert risk.severity_for(f) == "high" + + +def test_dead_permit_is_still_rated_low(): + """A permit that never fires blocks wanted traffic. Hygiene, not exposure.""" + f = a_finding("access_control", "medium", dead_line_detail( + "permit tcp 10.10.10.0 0.0.0.255 host 10.20.0.5 eq 443", "PERMIT", + "deny ip 10.10.10.0 0.0.0.255 any")) assert risk.severity_for(f) == "low" +@pytest.mark.parametrize("detail", [ + dead_line_detail("deny tcp any host 10.20.0.5 eq 23", "UNKNOWN", "an earlier line"), + "Unreachable line: deny tcp any host 10.20.0.5 eq 23", +]) +def test_dead_line_not_clearly_a_deny_is_still_rated_low(detail): + """Only a clear `(action DENY)` marker keeps the check's severity. Anything + else keeps R-1's old behaviour, as docs/severity-rules.md says.""" + assert risk.severity_for(a_finding("access_control", "high", detail)) == "low" + + +def test_a_dead_deny_from_the_real_check_comes_out_high(): + """The producer and the rule together: the finding access_control builds + for a dead deny is still `high` after refine(), and still `found`.""" + import itertools + + import pandas as pd + + from analysis.checks import access_control + + class _Session: + class q: + @staticmethod + def filterLineReachability(): + frame = pd.DataFrame([{ + "Sources": ["rtr-us5: lan_in"], + "Unreachable_Line": "deny tcp host 10.10.10.66 host 10.20.0.5 eq telnet", + "Unreachable_Line_Action": "DENY", + "Blocking_Lines": ["permit tcp 10.10.10.0 0.0.0.255 host 10.20.0.5"], + "Reason": "BLOCKING_LINES", + }]) + return type("A", (), {"answer": lambda self: self, + "frame": lambda self: frame})() + + [dead] = access_control._check_dead_rules(_Session(), itertools.count(1)) + [refined] = risk.refine([dead]) + assert dead["severity"] == "high", "the check's own grading changed" + assert refined["severity"] == "high" + assert refined["status"] == "found" + + +# --- R-1 is for dead ACL lines, not for the word (#444) --------------------- +# +# R-1 used to match "unreachable" anywhere in the evidence. Routing writes +# "Expected UNREACHABLE ..." for an isolation assertion, Batfish can end a +# trace in NEIGHBOR_UNREACHABLE, and `unreachable`, `host-unreachable` and +# `port-unreachable` are ordinary Cisco ICMP keywords that reach the evidence +# as quoted ACL lines. Every one of those was rated low. + + +class _Trace: + """The two attributes routing._evaluate() reads from a Batfish trace.""" + + def __init__(self, disposition, nodes): + self.disposition = disposition + self.hops = [type("Hop", (), {"node": n})() for n in nodes] + + +# routing.py builds the detail as "Expected {expected} for a flow from {src} +# to {dst}, but {violation}", with the violation from _evaluate() itself. +def routing_detail(expected, trace): + from analysis.checks import routing + violation = routing._evaluate(expected, [trace]) + assert violation is not None, "the trace must break the assertion" + return (f"Expected {expected} for a flow from 10.10.10.5 to " + f"10.20.20.5, but {violation}") + + +def test_isolation_assertion_that_fails_is_routing_medium_not_low(): + """The case #444 was found on: the guest LAN must NOT reach finance, and it + does. The word UNREACHABLE is the assertion, not a dead ACL line.""" + f = a_finding("routing", "high", routing_detail( + "UNREACHABLE", _Trace("ACCEPTED", ["rtr-hq", "rtr-branch"]))) + assert risk.severity_for(f) == "medium", "R-3 applies; R-1 must not" + + +def test_path_ending_in_neighbor_unreachable_is_routing_medium_not_low(): + f = a_finding("routing", "high", routing_detail( + "REACHABLE", _Trace("NEIGHBOR_UNREACHABLE", ["rtr-hq"]))) + assert risk.severity_for(f) == "medium", "R-3 applies; R-1 must not" + + +@pytest.mark.parametrize("check, severity, detail", [ + # access_control testFilters, access_control.py:458-460 + ("access_control", "high", + "Expected DENY but got PERMIT, decided by: permit icmp any any unreachable"), + # access_control searchFilters, access_control.py:543-545 + ("access_control", "high", + "Example permitted flow: start=rtr-us5 [8.8.8.8->10.10.10.5 ICMP (type=3, " + "code=1)], allowed by: permit icmp any any host-unreachable"), + # policy_compliance, policy_compliance.py:389-391 + ("policy_compliance", "high", + "Flow start=rtr-us5 [8.8.8.8->10.20.0.5 ICMP (type=3, code=3)] is permitted " + "but policy requires it to be DENIED. Decided by: permit icmp any any " + "port-unreachable"), + # undefinedReferences, access_control.py:667-670 + ("access_control", "high", + "Referenced as: interface incoming filter. The structure 'DROP-UNREACHABLE' " + "is never defined in this snapshot."), + # unusedStructures, access_control.py:740-745 + ("access_control", "medium", + "ip access-list 'DROP-UNREACHABLE' is not applied to any interface or " + "referenced anywhere in this config, so none of its rules take effect."), +]) +def test_the_word_in_config_text_does_not_lower_a_finding(check, severity, detail): + """No rule but R-1 could match these, so each keeps its check's severity.""" + assert risk.severity_for(a_finding(check, severity, detail)) == severity + + # --- R-2: blanket permits ---------------------------------------------------