From 5e572b76c5618aaf6b8fc61aed13f4263e25b85e Mon Sep 17 00:00:00 2001 From: Samika Perera Date: Tue, 3 Nov 2026 18:47:51 +1300 Subject: [PATCH 1/8] Pin that a dead deny keeps its check's severity (red on purpose) R-1 rates every dead ACL line low. For a dead deny that hides an exposure: traffic someone meant to block gets through, and when the blocker is a specific permit R-2 does not fire either. Measured in docs/research/cvss-comparison.md section 4 (#434). Three of these tests fail on main for that reason, 'low' == 'high': a dead deny behind a specific permit, a dead deny behind a blanket permit, and the finding the real access_control check builds for a dead deny, run through refine(). The dead permit test and the case of a missing or odd action marker pass today and must keep passing. --- tests/test_severity_rules.py | 79 ++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/tests/test_severity_rules.py b/tests/test_severity_rules.py index 815dc3d..8c3638d 100644 --- a/tests/test_severity_rules.py +++ b/tests/test_severity_rules.py @@ -53,6 +53,85 @@ def test_dead_rule_beats_the_blanket_permit_it_quotes(): assert risk.severity_for(f) == "low" +# The evidence exactly as access_control._check_dead_rules() writes it, so these +# tests break if that wording drifts away from what R-1 reads. +def dead_line_detail(line, action, blocked_by): + return (f"Unreachable line: {line} (action {action}). " + f"Blocked by: {blocked_by}. Reason: BLOCKING_LINES") + + +def test_dead_deny_behind_a_specific_permit_keeps_its_checks_severity(): + """The measured case from docs/research/cvss-comparison.md section 4 (#434). + + The telnet deny never fires because the permit above it already lets the + whole LAN through to the server. So telnet from .66 gets through, which + the author meant to block. R-2 does not fire (the blocker is not a + blanket permit), so if R-1 lowers this, nothing in the list says it is + an exposure. + """ + f = a_finding("access_control", "high", dead_line_detail( + "deny tcp host 10.10.10.66 host 10.20.0.5 eq telnet", "DENY", + "permit tcp 10.10.10.0 0.0.0.255 host 10.20.0.5")) + assert risk.severity_for(f) == "high" + + +def test_dead_deny_behind_a_blanket_permit_keeps_its_checks_severity(): + """Still an exposure in its own right, and R-2 does not promote the quoted + blocker, so the check's own `high` is what has to survive.""" + f = a_finding("access_control", "high", dead_line_detail( + "deny tcp host 10.10.10.66 host 10.20.0.5 eq telnet", "DENY", + "permit ip any any")) + assert risk.severity_for(f) == "high" + + +def test_dead_permit_is_still_rated_low(): + """A permit that never fires blocks wanted traffic. Hygiene, not exposure.""" + f = a_finding("access_control", "medium", dead_line_detail( + "permit tcp 10.10.10.0 0.0.0.255 host 10.20.0.5 eq 443", "PERMIT", + "deny ip 10.10.10.0 0.0.0.255 any")) + assert risk.severity_for(f) == "low" + + +@pytest.mark.parametrize("detail", [ + dead_line_detail("deny tcp any host 10.20.0.5 eq 23", "UNKNOWN", "an earlier line"), + "Unreachable line: deny tcp any host 10.20.0.5 eq 23", +]) +def test_dead_line_not_clearly_a_deny_is_still_rated_low(detail): + """Only a clear `(action DENY)` marker keeps the check's severity. Anything + else keeps R-1's old behaviour, as docs/severity-rules.md says.""" + assert risk.severity_for(a_finding("access_control", "high", detail)) == "low" + + +def test_a_dead_deny_from_the_real_check_comes_out_high(): + """The producer and the rule together: the finding access_control builds + for a dead deny is still `high` after refine(), and still `found`.""" + import itertools + + import pandas as pd + + from analysis.checks import access_control + + class _Session: + class q: + @staticmethod + def filterLineReachability(): + frame = pd.DataFrame([{ + "Sources": ["rtr-us5: lan_in"], + "Unreachable_Line": "deny tcp host 10.10.10.66 host 10.20.0.5 eq telnet", + "Unreachable_Line_Action": "DENY", + "Blocking_Lines": ["permit tcp 10.10.10.0 0.0.0.255 host 10.20.0.5"], + "Reason": "BLOCKING_LINES", + }]) + return type("A", (), {"answer": lambda self: self, + "frame": lambda self: frame})() + + [dead] = access_control._check_dead_rules(_Session(), itertools.count(1)) + [refined] = risk.refine([dead]) + assert dead["severity"] == "high", "the check's own grading changed" + assert refined["severity"] == "high" + assert refined["status"] == "found" + + # --- R-2: blanket permits --------------------------------------------------- From ce5fd1407f8f0949bbce8e8064a08beffffb0948 Mon Sep 17 00:00:00 2001 From: Samika Perera Date: Tue, 3 Nov 2026 18:48:46 +1300 Subject: [PATCH 2/8] Let a dead deny keep its check's severity under R-1 R-1 now lowers a dead PERMIT only. A dead DENY means traffic someone meant to block is getting through, and access_control already grades it high for that reason, so R-1 leaves it alone. The finding has no structured field for the dead line's action, so the rule reads the fixed "(action DENY). Blocked by:" marker that access_control writes into evidence.detail, the same shape ai/explain.py reads. Anything that says "unreachable" without that marker keeps the old answer, low. R-2 still does not promote a dead deny behind a blanket permit: the blocker follows "Blocked by:", so it is a citation and R-2 strips it. --- analysis/checks/risk.py | 41 ++++++++++++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/analysis/checks/risk.py b/analysis/checks/risk.py index f86fe56..3049deb 100644 --- a/analysis/checks/risk.py +++ b/analysis/checks/risk.py @@ -55,18 +55,45 @@ def _evidence(finding: Dict[str, Any]) -> str: return str(evidence.get("detail", "")).lower() +# The marker access_control._check_dead_rules() writes after the dead line: +# +# "Unreachable line: (action PERMIT|DENY). Blocked by: . ..." +# +# Our own fixed wording, not config text, and the same shape ai/explain.py +# reads. The first "(action ...)." before "Blocked by:" is ours; the line +# text before it comes from Batfish and never carries one. +_DEAD_DENY = re.compile(r"unreachable line: .+?\(action deny\)\. blocked by:") + + def _rule_dead_rule_is_low(finding: Dict[str, Any]) -> Optional[str]: - """R-1: a line that can never match cannot expose anything. + """R-1: a dead PERMIT is low. A dead DENY keeps its check's severity. + + A permit that can never match blocks traffic someone wanted: a hygiene + problem, worth reporting, not an exposure. A deny that can never match is + the opposite -- traffic someone meant to block gets through -- and + access_control already grades it high for that reason. + + It used to lower both, on the grounds that the exposure "is reported + separately by R-2". That only holds when the line shadowing the deny is + a blanket permit. Measured in docs/research/cvss-comparison.md section 4 + (#434): a deny shadowed by a SPECIFIC permit went to low, R-2 did not + fire, and the only finding about that traffic sat at the bottom. + + Only a clear "(action DENY)" marker is left alone. Anything else that + says "unreachable" keeps the old answer, low. FIRST ON PURPOSE. A dead-rule finding normally quotes the line that shadows it, and that line is very often a blanket permit -- so if R-2 ran first it - would match the quoted BLOCKER and rate the dead rule "high". The exposure - is not lost by rating this low: the blanket permit is reported separately on - its own finding, where R-2 rates it high. + would match the quoted BLOCKER and rate a dead permit "high". For a dead + deny this rule passes, and R-2 still does not promote it, because the + blocker follows "Blocked by:" and is a citation. """ - if "unreachable" in _evidence(finding): - return "low" - return None + detail = _evidence(finding) + if "unreachable" not in detail: + return None + if _DEAD_DENY.search(detail): + return None + return "low" # The phrases our checks use to ATTRIBUTE a violation to the ACL line that From 2563f1eb1d89466836f663557f000d3d57451b4b Mon Sep 17 00:00:00 2001 From: Samika Perera Date: Tue, 3 Nov 2026 18:49:31 +1300 Subject: [PATCH 3/8] Rewrite R-1 in the severity rules: a dead deny keeps its severity The ruleset is the doc, so it now says what the code does: R-1 lowers a dead permit only, and a dead deny keeps the high access_control gives it. It records why, the measured case from #434 (section 4 of the CVSS comparison), that the old "reported separately by R-2" reasoning is kept as history, and how the action is read: from the fixed "(action DENY)" marker our own check writes, with anything unclear staying low. Also adds the one line agreed on #434: business context is our equivalent of CVSS Environmental metrics. --- docs/severity-rules.md | 33 ++++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/docs/severity-rules.md b/docs/severity-rules.md index 06a7bb1..1c7a79e 100644 --- a/docs/severity-rules.md +++ b/docs/severity-rules.md @@ -46,21 +46,38 @@ move it down the page, which is the F-4 failure by another route. Applied in order. The first that matches wins; anything unmatched keeps the severity its check assigned. -### R-1 — A dead rule is low, whatever it says +### R-1 — A dead permit is low; a dead deny keeps its check's severity **Matches:** `status="found"` and the evidence describes an unreachable / -shadowed ACL line. +shadowed ACL line that is **not** marked `(action DENY)`. **Sets:** `low` -A line that can never match cannot expose anything. It is a hygiene problem — +A dead **permit** blocks traffic someone wanted. It is a hygiene problem — someone believes a rule is in force when it is not — and it is worth reporting, but it is not an exposure and should not sit above one. +A dead **deny** is the opposite: traffic someone meant to block gets through. +`access_control` grades it `high` for that reason, and R-1 leaves it alone. + +**This used to lower every dead line**, on the grounds that the exposure is +reported separately by R-2. That only holds when the line shadowing the deny is +a blanket permit. Measured in `docs/research/cvss-comparison.md` section 4 +(#434): a telnet deny shadowed by a *specific* permit went to `low`, R-2 did not +fire, and the only finding about that traffic sat at the bottom of the list. +§6 item 2 below also records that R-2 has no observed effect on any fixture we +have, so R-1 should not lean on it. + +**How the action is read.** The finding has no structured field for it, so R-1 +reads the fixed marker `access_control` writes into `evidence.detail` — +*"Unreachable line: … (action DENY). Blocked by: …"*. That is our wording, not +config text. A dead-rule finding without a clear `(action DENY)` marker keeps +the old answer, `low`. + This rule is deliberately **first**, because a dead-rule finding usually quotes the line that shadows it, and that line is very often a blanket permit. Without -the ordering, R-2 would match the quoted blocker and rate the dead rule `high`. -The exposure is not lost by rating it low: the blanket permit is reported -separately, on its own finding, by R-2. +the ordering, R-2 would match the quoted blocker and rate a dead permit `high`. +A dead deny passes through R-1 and is not promoted by R-2 either: its blocker +follows *"Blocked by:"*, so R-2 treats it as a citation. ### R-2 — A blanket permit is high @@ -124,7 +141,9 @@ registry order, which is alphabetical by accident rather than meaningful. - **It does not weight by device.** "This router is more important than that one" is real, but we have no source for it that is not guesswork, and inventing one would be exactly the unfounded claim the rest of this project - avoids. If the client tells us, it becomes data, not a rule. + avoids. If the client tells us, it becomes data, not a rule. That data is + business context (`critical` / `important` / `standard`), applied after these + rules: our equivalent of CVSS Environmental metrics. ## 6. Open, and deliberately not settled here From 72af1095786aa788c8190173619eee45c22cf902 Mon Sep 17 00:00:00 2001 From: Samika Perera Date: Wed, 4 Nov 2026 10:22:16 +1300 Subject: [PATCH 4/8] Say that R-1 over-rates a dead deny whose blocker also denies (#447) A dead deny is only an exposure when the line shadowing it is a permit. Behind another deny the traffic is still blocked. R-1 reads only the dead line's action, so it keeps high for both. The docstring and the rules said every dead deny lets traffic through; they now name the case and link #447. No code change. --- analysis/checks/risk.py | 10 ++++++++-- docs/severity-rules.md | 10 ++++++++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/analysis/checks/risk.py b/analysis/checks/risk.py index 3049deb..8ef9735 100644 --- a/analysis/checks/risk.py +++ b/analysis/checks/risk.py @@ -70,8 +70,14 @@ def _rule_dead_rule_is_low(finding: Dict[str, Any]) -> Optional[str]: A permit that can never match blocks traffic someone wanted: a hygiene problem, worth reporting, not an exposure. A deny that can never match is - the opposite -- traffic someone meant to block gets through -- and - access_control already grades it high for that reason. + usually the opposite: when the line shadowing it is a permit, traffic + someone meant to block gets through, and access_control grades it high + for that reason. + + NOT WHEN THE BLOCKER ALSO DENIES. Then the traffic is still blocked, by + the earlier line, and nothing is exposed. R-1 cannot tell the two apart: + the evidence names the dead line's action, not the blocker's. So it keeps + the check's severity for both, and over-rates a deny behind a deny (#447). It used to lower both, on the grounds that the exposure "is reported separately by R-2". That only holds when the line shadowing the deny is diff --git a/docs/severity-rules.md b/docs/severity-rules.md index 1c7a79e..93986b2 100644 --- a/docs/severity-rules.md +++ b/docs/severity-rules.md @@ -56,8 +56,14 @@ A dead **permit** blocks traffic someone wanted. It is a hygiene problem — someone believes a rule is in force when it is not — and it is worth reporting, but it is not an exposure and should not sit above one. -A dead **deny** is the opposite: traffic someone meant to block gets through. -`access_control` grades it `high` for that reason, and R-1 leaves it alone. +A dead **deny** is usually the opposite: when the line shadowing it is a +permit, traffic someone meant to block gets through. `access_control` grades +it `high` for that reason, and R-1 leaves it alone. + +**Known over-rating: a dead deny behind another deny (#447).** Then the traffic +is still blocked, by the earlier line, and nothing is exposed. R-1 cannot tell +this case apart: the evidence names the dead line's action, not the blocker's. +So it keeps the check's `high` here too, which over-rates it. **This used to lower every dead line**, on the grounds that the exposure is reported separately by R-2. That only holds when the line shadowing the deny is From 01a41954b1a18aae668b9c31f027f7bf5c318aa1 Mon Sep 17 00:00:00 2001 From: Samika Perera Date: Wed, 4 Nov 2026 10:22:43 +1300 Subject: [PATCH 5/8] Cite PR #434 for the R-1 measurement, not a file main does not have risk.py, severity-rules.md and one test docstring pointed at docs/research/cvss-comparison.md, which only exists once #434 merges. --- analysis/checks/risk.py | 4 ++-- docs/severity-rules.md | 4 ++-- tests/test_severity_rules.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/analysis/checks/risk.py b/analysis/checks/risk.py index 8ef9735..431336d 100644 --- a/analysis/checks/risk.py +++ b/analysis/checks/risk.py @@ -81,8 +81,8 @@ def _rule_dead_rule_is_low(finding: Dict[str, Any]) -> Optional[str]: It used to lower both, on the grounds that the exposure "is reported separately by R-2". That only holds when the line shadowing the deny is - a blanket permit. Measured in docs/research/cvss-comparison.md section 4 - (#434): a deny shadowed by a SPECIFIC permit went to low, R-2 did not + a blanket permit. Measured in section 4 of the CVSS comparison in PR + #434: a deny shadowed by a SPECIFIC permit went to low, R-2 did not fire, and the only finding about that traffic sat at the bottom. Only a clear "(action DENY)" marker is left alone. Anything else that diff --git a/docs/severity-rules.md b/docs/severity-rules.md index 93986b2..96b2a19 100644 --- a/docs/severity-rules.md +++ b/docs/severity-rules.md @@ -67,8 +67,8 @@ So it keeps the check's `high` here too, which over-rates it. **This used to lower every dead line**, on the grounds that the exposure is reported separately by R-2. That only holds when the line shadowing the deny is -a blanket permit. Measured in `docs/research/cvss-comparison.md` section 4 -(#434): a telnet deny shadowed by a *specific* permit went to `low`, R-2 did not +a blanket permit. Measured in section 4 of the CVSS comparison in PR +#434: a telnet deny shadowed by a *specific* permit went to `low`, R-2 did not fire, and the only finding about that traffic sat at the bottom of the list. §6 item 2 below also records that R-2 has no observed effect on any fixture we have, so R-1 should not lean on it. diff --git a/tests/test_severity_rules.py b/tests/test_severity_rules.py index 8c3638d..436ff26 100644 --- a/tests/test_severity_rules.py +++ b/tests/test_severity_rules.py @@ -61,7 +61,7 @@ def dead_line_detail(line, action, blocked_by): def test_dead_deny_behind_a_specific_permit_keeps_its_checks_severity(): - """The measured case from docs/research/cvss-comparison.md section 4 (#434). + """The measured case from section 4 of the CVSS comparison in PR #434. The telnet deny never fires because the permit above it already lets the whole LAN through to the server. So telnet from .66 gets through, which From 9a5c954bd7ede8e82df089004e53a459a9a6b01a Mon Sep 17 00:00:00 2001 From: Samika Perera Date: Wed, 4 Nov 2026 10:22:57 +1300 Subject: [PATCH 6/8] Say which R-1 test actually catches wording drift The helper copies access_control's wording by hand, so tests built on it do not notice if the real wording changes. Only the test that runs the real producer does. The comment claimed all of them did. --- tests/test_severity_rules.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_severity_rules.py b/tests/test_severity_rules.py index 436ff26..75dbccd 100644 --- a/tests/test_severity_rules.py +++ b/tests/test_severity_rules.py @@ -53,8 +53,10 @@ def test_dead_rule_beats_the_blanket_permit_it_quotes(): assert risk.severity_for(f) == "low" -# The evidence exactly as access_control._check_dead_rules() writes it, so these -# tests break if that wording drifts away from what R-1 reads. +# The shape access_control._check_dead_rules() writes, copied by hand. A copy +# does not follow the real wording if it drifts: only +# test_a_dead_deny_from_the_real_check_comes_out_high, which runs the real +# producer, catches that. def dead_line_detail(line, action, blocked_by): return (f"Unreachable line: {line} (action {action}). " f"Blocked by: {blocked_by}. Reason: BLOCKING_LINES") From 139333f1219483ce2eb9d1d8d7bdde0905897cb7 Mon Sep 17 00:00:00 2001 From: Samika Perera Date: Wed, 4 Nov 2026 10:23:26 +1300 Subject: [PATCH 7/8] Pin that R-1 needs the whole dead-deny marker, not half of it R-1 keeps a dead deny's severity only when "(action DENY)" is followed by ". Blocked by:", the way access_control writes it. Deleting that second half of the pattern failed no test. This one fails if it is deleted. --- tests/test_severity_rules.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test_severity_rules.py b/tests/test_severity_rules.py index 75dbccd..74bd8f7 100644 --- a/tests/test_severity_rules.py +++ b/tests/test_severity_rules.py @@ -104,6 +104,14 @@ def test_dead_line_not_clearly_a_deny_is_still_rated_low(detail): assert risk.severity_for(a_finding("access_control", "high", detail)) == "low" +def test_action_deny_not_followed_by_blocked_by_is_still_rated_low(): + """The marker is "(action DENY). Blocked by:", both halves. Our check + always writes them together, so "(action DENY)" on its own is not our + marker and gets R-1's old answer, low.""" + detail = "Unreachable line: deny tcp any host 10.20.0.5 eq 23 (action DENY)" + assert risk.severity_for(a_finding("access_control", "high", detail)) == "low" + + def test_a_dead_deny_from_the_real_check_comes_out_high(): """The producer and the rule together: the finding access_control builds for a dead deny is still `high` after refine(), and still `found`.""" From 6b33709eade96d924cb74881150e59f3810f426c Mon Sep 17 00:00:00 2001 From: Samika Perera Date: Fri, 20 Nov 2026 00:35:46 +1300 Subject: [PATCH 8/8] Say what R-1 does in the group report: a dead permit is low, a dead deny keeps its severity Changes one line, build_group_report.py:408, which said a dead rule cannot expose anything. That no longer matches R-1 on this branch: a dead deny shadowed by a permit lets through the traffic it was meant to block, and keeps its check's severity. Ankeet and Arsh flagged the line in their reviews of this PR. CODEOWNERS gives the file to Arsh. --- docs/document-builds/build_group_report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/document-builds/build_group_report.py b/docs/document-builds/build_group_report.py index 0f33779..2f31135 100644 --- a/docs/document-builds/build_group_report.py +++ b/docs/document-builds/build_group_report.py @@ -405,7 +405,7 @@ def code(t): "possible traffic", "No rows is a proof that no such flow exists. One row is a counter-example"], ["filterLineReachability", "Rules that can never match, and the earlier rule shadowing them", - "Report it as a tidiness problem and rate it low, because a dead rule cannot expose anything"], + "Rate a dead permit low, as a tidiness problem; a dead deny keeps its check's severity, because when a permit shadows it the traffic it was meant to block gets through"], ["undefinedReferences", "Things the configuration mentions but never defines", "Report it directly. This is a silent failure risk"], ["traceroute", "The hop-by-hop path and whether the traffic arrived",