From df594bdfce8e56272d19f0089321ce67b6b7f468 Mon Sep 17 00:00:00 2001 From: patelankeet2 Date: Sun, 8 Nov 2026 22:48:23 +1300 Subject: [PATCH 1/3] Dead-rule fact: stop claiming a universal outcome Batfish cannot guarantee (#453) _compute_dead_rule_outcome()'s contradicted-action branch (own action vs. blocking line's action disagree) told the model, as already verified and certain, that 'traffic matching it is actually {outcome}' for the dead line's ENTIRE matching pattern. Measured live (same on main db59b00 and -- specifically, a line sharing the DEAD line's own action is not always listed, and when one exists it can decide a narrower slice of traffic the opposite way from what the named, opposite-action blocking line alone predicts. A real config had permit host 10.1.1.5 any before both the named deny and the dead permit; the fact said the whole /24 is denied, but testFilters showed host .5 is actually permitted. Confirmed only the contradicted branch is unsafe. The redundant branch (blocking line agrees with the dead line's own action) is unaffected -- break it, 6 returned None (Batfish named the overlapping line), 2 had no such overlap and the sentence was true. Worked out why: if an omission is always a same-action-as-the-dead-line line, it cannot change the outcome when every alternative (named or not) already agrees. Also confirmed naming the blocking line's own text is not, by itself, enough (the issue's suggested wording): the omitted line in the measured case is earlier than the NAMED blocking line, not narrower than the dead line's pattern -- host .5 matches the named deny's /24 too. The fix drops the universal 'actually {outcome}' claim for this branch entirely, stating only the two facts that stay true regardless of what Blocking_Lines could have left out: the dead line's own action never fires (filterLineReachability verifies that exhaustively), and the named earlier line genuinely is evaluated first and genuinely does have the opposite action. Stacked on #401's branch (feature/384-dead-rule-own-action) rather than main, since this function does not exist in its post-#401 shape on main yet. Tests: updated the four tests pinning the old universal-outcome wording to the new one; added a new test reproducing #453's own measured shape directly. Mutation-tested: reverting to the old wording breaks all 7 affected tests for the right reason; the two redundant-branch tests are unaffected either way, confirming that branch genuinely does not need this fix. Full suite: 2266 passed, 13 skipped, 0 failed. Live-tested against the real model: 15 generations on the measured contradicted-case finding, 0/15 fallbacks, wording correctly follows the new, narrower fact in every non-copy case. Re-captured and re-rated AC-002 in the held-out evaluation set (its prompt hash moved since this changes the exact text fed to the model for that finding) -- docs/evaluation.md and the holdout fixture both updated, with a request for Shubham and Arsh to rate the new capture. --- ai/explain.py | 52 +++++++++++++++---- docs/evaluation.md | 7 ++- .../fixtures/explanation_quality/holdout.json | 2 +- tests/test_explain_safety_net.py | 44 ++++++++++++++-- 4 files changed, 90 insertions(+), 15 deletions(-) diff --git a/ai/explain.py b/ai/explain.py index 81c8f34..a652f54 100644 --- a/ai/explain.py +++ b/ai/explain.py @@ -327,6 +327,15 @@ def _compute_dead_rule_outcome(detail: str) -> Optional[str]: generates an explanation from the raw finding either way. This function only ever ADDS confidence; it never blocks a finding from being explained. + + WHAT "THE REAL OUTCOME" MEANS FOR THE CONTRADICTED CASE (#453) + When the blocking line's action disagrees with the dead line's + own action, this no longer claims a single outcome for ALL + traffic matching the dead line's pattern -- see the comment + above that branch's return statement for why that claim is not + always true and what is stated instead. The redundant case + (blocking line agrees with the dead line) is unaffected -- see + that branch's own comment for why. """ match = _DEAD_RULE_DETAIL_PATTERN.search(detail) if not match: @@ -367,8 +376,10 @@ def _compute_dead_rule_outcome(detail: str) -> Optional[str]: # syntax (found in review, 9 November -- raw syntax measurably traded # that misdescription for a different one, confusing the model about # which address was "a specific host"; see _describe_acl_line()'s own - # docstring). If any blocking line does not parse, fall back to - # naming them in raw syntax rather than withholding the fact. + # docstring). This path only ever states what the blocking line itself + # does, never a claim about the dead line's whole pattern, so it does + # not carry #453's overclaim risk either (see the fallback below for + # what that risk was). descriptions = [_describe_acl_line(line) for line in blocking_lines] if all(d is not None for d in descriptions): if len(descriptions) == 1: @@ -385,16 +396,39 @@ def _compute_dead_rule_outcome(detail: str) -> Optional[str]: f"decide instead -- {cases}." ) + # If any blocking line does not parse, fall back to naming them in raw + # syntax -- but NOT to a universal-outcome claim (#453). Blocking_Lines + # does not always name every earlier line, specifically one sharing the + # DEAD line's own action rather than the named blocking line's opposite + # one, so "traffic matching it is actually {outcome}" can be false for + # a subset of the dead line's own pattern. Measured: a real config had + # `permit host 10.1.1.5 any` before both `deny 10.1.1.0/24 any` (the + # named blocker) and the dead `permit 10.1.1.0/24 ... eq 443` -- + # Batfish named only the deny, so the old sentence said "actually + # denied" for the whole /24, but testFilters showed host .5 is + # actually PERMITted, by the earlier, unnamed, same-action-as-the- + # dead-line permit. + # + # What stays true regardless of anything Blocking_Lines could have + # left out: filterLineReachability verifies the dead line's own action + # never fires, across its whole match space, exhaustively -- that part + # of "unreachable" is sound. And whatever IS named here genuinely is + # evaluated before the dead line and genuinely does have the opposite + # action -- that is just what Blocking_Lines means. Stating only those + # two facts, and nothing about what therefore happens to "traffic + # matching it", is what stays true no matter what is missing from this + # list. named = " ".join(match.group("blocking").split()) # Batfish pads its spaces - lines, verb, decide_verb = ( - ("earlier lines", "are", "decide") if len(blocking_lines) > 1 - else ("an earlier line", "is", "decides") + outcome_action = "DENY" if outcome == "denied" else "PERMIT" + lines, decide_verb, verb = ( + ("earlier lines", "are", "have") if len(blocking_lines) > 1 + else ("an earlier line", "is", "has") ) return ( - f"the unreachable line's own stated action is {dead_action}, but it " - f"never takes effect: traffic matching it is actually {outcome}, " - f"because {lines} ({named}) {verb} evaluated first and {decide_verb} " - f"instead." + f"the unreachable line's own stated action is {dead_action}, but " + f"that action never takes effect: {lines} ({named}), which " + f"{decide_verb} evaluated first, {verb} the opposite action " + f"({outcome_action}) instead." ) diff --git a/docs/evaluation.md b/docs/evaluation.md index dcd3bfa..5f9c47b 100644 --- a/docs/evaluation.md +++ b/docs/evaluation.md @@ -797,7 +797,10 @@ Ankeet, 3 November, after #401; AC-002 re-captured and re-rated by Ankeet again, 8 November, after review found the 3 November capture's own wording was itself a misdescription; AC-002 re-captured and re-rated a third time, 9 November, after review found the 8 November wording traded that -misdescription for a different one) - see +misdescription for a different one -- #453 also changed this computed +fact's fallback wording the same day, rebased onto this fix once pushed, +and does not change this particular capture, since this finding's +blocking line takes the unaffected plain-English path either way) - see [`tests/fixtures/explanation_quality/holdout.json`](../tests/fixtures/explanation_quality/holdout.json) for the full captured findings, explanations and reasoning behind each score. Same 1-5 scale as Part 3 (accuracy: does the text claim only what the @@ -809,7 +812,7 @@ read `evidence.detail` what is wrong). | RT-050 | fallback | 5 | 4 | Deterministic, cannot be wrong by construction - **reported separately below, not part of the model score.** | | AC-001 | model | 5 | 4 | Correct, no inversion. | | PC-001 | model | 5 | 4 | Regenerated post-#145: inversion gone. | -| AC-002 | model | 5 | 5 | **Re-captured a third time, 9 November, after review found that naming the blocking line in raw ACL syntax (the 8 November fix) traded one misdescription for another.** 30 live generations on the 8 November wording: 0/30 called the permit line a block, but 16/30 called the dead line's own /24 source "a specific host" - a confusion neither the pre-fix wording nor `main` produced, caused by two raw ACL snippets sitting next to each other in the prompt. Fixed by describing the blocking line in plain English instead of raw syntax, refusing rather than guessing on any shape it does not recognise. Re-measured, same finding, 30 generations: 0/30 call the permit line a block, 1/30 clearly calls the source a specific host (down from 16/30), 1/30 ambiguous, 0/30 copy Modelfile Example 1. This capture states both source and destination explicitly and correctly. Two things found along the way, reported but out of scope here: 2/30 invert the flow direction, 2/30 echo Example 1's "unencrypted" vocabulary without copying the phrase exactly. **@ARSH871-bot @shubhamkataria2005, fresh capture against a changed fact - your earlier ratings were of the 8 November text. Please rate again.** | +| AC-002 | model | 5 | 5 | **Re-captured a third time, 9 November, after review found that naming the blocking line in raw ACL syntax (the 8 November fix) traded one misdescription for another.** 30 live generations on the 8 November wording: 0/30 called the permit line a block, but 16/30 called the dead line's own /24 source "a specific host" - a confusion neither the pre-fix wording nor `main` produced, caused by two raw ACL snippets sitting next to each other in the prompt. Fixed by describing the blocking line in plain English instead of raw syntax, refusing rather than guessing on any shape it does not recognise. Re-measured, same finding, 30 generations: 0/30 call the permit line a block, 1/30 clearly calls the source a specific host (down from 16/30), 1/30 ambiguous, 0/30 copy Modelfile Example 1. This capture states both source and destination explicitly and correctly. Two things found along the way, reported but out of scope here: 2/30 invert the flow direction, 2/30 echo Example 1's "unencrypted" vocabulary without copying the phrase exactly. #453 (dead-rule fact completeness) also changed this function's fallback wording the same day, for the case where a blocking line cannot be parsed in plain English - this finding's blocking line always could, so that change does not affect this capture. **@ARSH871-bot @shubhamkataria2005, fresh capture against a changed fact - your earlier ratings were of the 8 November text. Please rate again.** | | PC-005 | model | 5 | 3 | Regenerated post-#145: inversion gone. First-sentence phrasing still asks the reader to resolve an apparent contradiction before it resolves itself - a readability risk, not a false claim this time. | @ARSH871-bot @shubhamkataria2005 @SamikaPerera - a second independent rating diff --git a/tests/fixtures/explanation_quality/holdout.json b/tests/fixtures/explanation_quality/holdout.json index d3a3bcb..2ebaf90 100644 --- a/tests/fixtures/explanation_quality/holdout.json +++ b/tests/fixtures/explanation_quality/holdout.json @@ -113,7 +113,7 @@ "rater": "Ankeet", "accuracy": 5, "usefulness": 5, - "notes": "RE-CAPTURED a fourth time (9 November), after review found naming the blocking line in raw ACL syntax (the 5 November fix) measurably traded that misdescription for a different one: 30 live generations went from 0/30 calling the dead PERMIT line a block to 16/30 calling the /24 source a specific host. Fixed by describing the blocking line in plain English instead of raw syntax (_describe_acl_line()), refusing rather than guessing on any shape it does not recognise. Re-measured, same finding, 30 generations: 0/30 call the PERMIT line a block, 1/30 clearly calls the source a specific host (down from 16/30), 1/30 ambiguous, 0/30 copy Modelfile Example 1 verbatim. This capture states both source and destination explicitly and correctly, names the real cause (an earlier deny rule), and invents nothing beyond the evidence. Two things found along the way, reported but out of scope for this fix: 2/30 invert the flow direction (says traffic runs from the destination to the source), 2/30 echo Example 1's \"unencrypted\" vocabulary without copying the phrase exactly -- filed as follow-up observations, not fixed here. @ARSH871-bot @shubhamkataria2005, fresh capture against a changed fact -- please rate again." + "notes": "RE-CAPTURED a fourth time (9 November), after review found naming the blocking line in raw ACL syntax (the 5 November fix) measurably traded that misdescription for a different one: 30 live generations went from 0/30 calling the dead PERMIT line a block to 16/30 calling the /24 source a specific host. Fixed by describing the blocking line in plain English instead of raw syntax (_describe_acl_line()), refusing rather than guessing on any shape it does not recognise. Re-measured, same finding, 30 generations: 0/30 call the PERMIT line a block, 1/30 clearly calls the source a specific host (down from 16/30), 1/30 ambiguous, 0/30 copy Modelfile Example 1 verbatim. This capture states both source and destination explicitly and correctly, names the real cause (an earlier deny rule), and invents nothing beyond the evidence. Two things found along the way, reported but out of scope for this fix: 2/30 invert the flow direction (says traffic runs from the destination to the source), 2/30 echo Example 1's \"unencrypted\" vocabulary without copying the phrase exactly -- filed as follow-up observations, not fixed here. #453 (dead-rule fact completeness) also changed this function's fallback wording the same day, for a blocking line that cannot be parsed in plain English -- this finding's blocking line always could, so that change does not affect this capture. @ARSH871-bot @shubhamkataria2005, fresh capture against a changed fact -- please rate again." } ] }, diff --git a/tests/test_explain_safety_net.py b/tests/test_explain_safety_net.py index ad80e3f..3512bc3 100644 --- a/tests/test_explain_safety_net.py +++ b/tests/test_explain_safety_net.py @@ -168,6 +168,7 @@ def test_dead_deny_shadowed_by_permit_means_traffic_is_permitted(): assert "own stated action is DENY" in outcome assert "permits all traffic from anywhere to 2.128.0.0/16" in outcome assert "denies" not in outcome + assert "own stated action is PERMIT" not in outcome def test_dead_permit_shadowed_by_deny_means_traffic_is_denied(): @@ -182,6 +183,7 @@ def test_dead_permit_shadowed_by_deny_means_traffic_is_denied(): assert "own stated action is PERMIT" in outcome assert "denies all traffic from anywhere to any destination" in outcome assert "permits" not in outcome + assert "own stated action is DENY" not in outcome def test_returns_none_for_a_finding_that_is_not_a_dead_rule(): @@ -269,9 +271,9 @@ def test_the_dead_lines_own_action_is_stated_explicitly_deny(): def test_the_dead_lines_own_action_and_the_outcome_can_differ(): """The whole point: the dead line's own action and the real outcome are two DIFFERENT facts, and a PERMIT line shadowed by a DENY line has them - disagree -- own action PERMIT, real outcome denied. Both must be - stated, not just whichever one the function used to consider more - important.""" + disagree -- own action PERMIT, the blocking line's action DENY. Both + must be stated, not just whichever one the function used to consider + more important.""" detail = ( "Unreachable line: permit udp 10.10.10.0 0.0.0.255 host 218.8.104.58 " "eq domain (action PERMIT). Blocked by: deny ip 10.10.10.0 " @@ -291,6 +293,13 @@ def test_the_dead_lines_own_action_and_the_outcome_can_differ(): assert "same destination" not in outcome assert "a specific host" not in outcome assert "0.0.0.255" not in outcome, "raw wildcard-mask syntax must not leak into the fact" + # #453: must not claim a universal outcome for every packet the dead + # line's pattern would match either -- this plain-English path never + # did (it only ever describes what the blocking line itself does), but + # pin it anyway since it is exactly the phrasing that used to appear. + assert "traffic matching it is actually" not in outcome + assert "actually denied" not in outcome + assert "actually permitted" not in outcome # --- _describe_acl_line (#401 review, @shubhamkataria2005, 9 November) ------- @@ -335,6 +344,35 @@ def test_returns_none_for_too_few_tokens(): assert _describe_acl_line("permit ip any") is None +def test_an_earlier_same_action_line_omitted_from_blocking_lines_is_not_overclaimed(): + """#453, measured live. Blocking_Lines named only the opposite-action + deny; a host-specific PERMIT even earlier than that deny, sharing the + dead line's own PERMIT action, was not named at all and decides that + host's traffic the other way. The computed fact must not claim the + deny's outcome holds for every packet the dead line's wider pattern + would have matched -- it has no way to know that from this detail + string alone. This fixture's blocking line parses through + _describe_acl_line() (#401), so it takes that plain-English path, not + the raw-syntax fallback #453 was originally written against -- updated + accordingly rather than left pinning wording the code no longer + produces.""" + detail = ( + "Unreachable line: permit tcp 10.1.1.0 0.0.0.255 any eq 443 " + "(action PERMIT). Blocked by: deny tcp 10.1.1.0 0.0.0.255 any. " + "Reason: BLOCKING_LINES" + ) + outcome = _compute_dead_rule_outcome(detail) + assert outcome is not None + assert "own stated action is PERMIT" in outcome + assert "denies TCP traffic from 10.1.1.0/24 to any destination" in outcome + # The overclaim #453 found: asserting this for ALL traffic matching + # the dead line's own pattern is false for host 10.1.1.5 specifically + # (real testFilters: PERMIT), because of a line even earlier than the + # one named here. Confirm the fact no longer makes that claim. + assert "actually denied" not in outcome + assert "traffic matching it" not in outcome + + def _real_dead_rule_detail(dead_action: str, blocking_action: str) -> str: """The genuine evidence string, from the genuine check -- not a literal. From e3ec2fdbd50aab3436d009c521da3100e7836470 Mon Sep 17 00:00:00 2001 From: patelankeet2 Date: Tue, 10 Nov 2026 13:13:39 +1300 Subject: [PATCH 2/3] Pin the fallback path's wording directly, not just the primary one (#453) Rebasing onto #401's plain-English blocking-line fix meant every existing fixture for the contradicted branch now takes that primary path, leaving #453's own fix -- the dropped universal-outcome claim in the raw-syntax fallback -- with no test reaching it at all. Added one that does, using a blocking line shaped so _describe_acl_line() refuses it (a source-port clause). Mutation tested: reverting the fallback to the old 'traffic matching it is actually {outcome}' wording fails this new test for the right reason; the existing 'omitted' test is unaffected, since it takes the primary path either way. --- tests/test_explain_safety_net.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/test_explain_safety_net.py b/tests/test_explain_safety_net.py index 3512bc3..22fb4a0 100644 --- a/tests/test_explain_safety_net.py +++ b/tests/test_explain_safety_net.py @@ -373,6 +373,29 @@ def test_an_earlier_same_action_line_omitted_from_blocking_lines_is_not_overclai assert "traffic matching it" not in outcome +def test_an_unparseable_blocking_line_falls_back_without_a_universal_claim(): + """#453's own fix, exercised directly through its actual code path. + When the blocking line's shape _describe_acl_line() refuses (here, a + source-port clause #401's parser does not recognise), the contradicted + branch must still fall back to naming it in raw syntax -- but never to + the dropped universal-outcome claim ("traffic matching it is actually + {outcome}") #453 found false for a subset of the dead line's own + pattern. Nothing else in this file reaches this fallback, since every + other fixture's blocking line happens to parse -- without this test, + #453's fix would be silently untested.""" + detail = ( + "Unreachable line: permit tcp any any eq 443 (action PERMIT). " + "Blocked by: deny tcp any eq 1024 any eq 443. Reason: BLOCKING_LINES" + ) + outcome = _compute_dead_rule_outcome(detail) + assert outcome is not None + assert "own stated action is PERMIT" in outcome + assert "deny tcp any eq 1024 any eq 443" in outcome + assert "opposite action (DENY)" in outcome + assert "traffic matching it is actually" not in outcome + assert "actually denied" not in outcome + + def _real_dead_rule_detail(dead_action: str, blocking_action: str) -> str: """The genuine evidence string, from the genuine check -- not a literal. From 039986f349836ffa6e58613a3a1e94eec7813cd9 Mon Sep 17 00:00:00 2001 From: patelankeet2 Date: Wed, 11 Nov 2026 11:01:58 +1300 Subject: [PATCH 3/3] Return None for an unparseable blocking line instead of a weaker fallback claim Shubham measured the raw-syntax fallback #453 added for a blocking line _describe_acl_line() can't render in plain English: 10/20 live generations copied the Modelfile's Example 1 almost verbatim, against 1/20 for main's prior wording and 0/20 for no computed fact at all. A sentence with no outcome in it still leaves a gap, and the model filled it with Example 1 more than either alternative. Return None instead, the same refusal this function already makes for unrecognised syntax and disagreeing blocking lines. Rewrote the test pinning the old wording to assert None, and added a mixed-list case (one parseable blocking line, one not) that nothing pinned before. --- ai/explain.py | 68 ++++++++++++++++---------------- tests/test_explain_safety_net.py | 50 +++++++++++++++-------- 2 files changed, 68 insertions(+), 50 deletions(-) diff --git a/ai/explain.py b/ai/explain.py index a652f54..9d261c9 100644 --- a/ai/explain.py +++ b/ai/explain.py @@ -336,6 +336,11 @@ def _compute_dead_rule_outcome(detail: str) -> Optional[str]: always true and what is stated instead. The redundant case (blocking line agrees with the dead line) is unaffected -- see that branch's own comment for why. + + If a blocking line's action is known but its shape cannot be + rendered in plain English, this now returns None instead of + naming it in raw syntax (#461) -- see the comment above that + return statement for the measurement behind it. """ match = _DEAD_RULE_DETAIL_PATTERN.search(detail) if not match: @@ -396,40 +401,37 @@ def _compute_dead_rule_outcome(detail: str) -> Optional[str]: f"decide instead -- {cases}." ) - # If any blocking line does not parse, fall back to naming them in raw - # syntax -- but NOT to a universal-outcome claim (#453). Blocking_Lines - # does not always name every earlier line, specifically one sharing the - # DEAD line's own action rather than the named blocking line's opposite - # one, so "traffic matching it is actually {outcome}" can be false for - # a subset of the dead line's own pattern. Measured: a real config had - # `permit host 10.1.1.5 any` before both `deny 10.1.1.0/24 any` (the - # named blocker) and the dead `permit 10.1.1.0/24 ... eq 443` -- - # Batfish named only the deny, so the old sentence said "actually - # denied" for the whole /24, but testFilters showed host .5 is - # actually PERMITted, by the earlier, unnamed, same-action-as-the- - # dead-line permit. + # If any blocking line does not parse, refuse rather than fall back to + # naming it in raw syntax (#461 review, @shubhamkataria2005). #453's own + # fix tried a raw-syntax fallback here, stating only that the named + # line has the opposite action -- no claim about "traffic matching it", + # specifically to avoid the universal-outcome overclaim described below. + # Measured anyway, 20 live generations on the exact fixture that + # fallback exists for: 10/20 copied the Modelfile's Example 1 almost + # verbatim, against 1/20 for #401's main-head wording and 0/20 for no + # computed fact at all. A sentence with no outcome in it still leaves a + # gap for the model to fill, and it filled it with Example 1 more often + # than either alternative. Returning None here is not a weaker version + # of that fallback; it is the same refusal this function already makes + # a few lines up for unrecognised syntax and disagreeing blocking + # lines -- "not confident enough to say" covers this case too. # - # What stays true regardless of anything Blocking_Lines could have - # left out: filterLineReachability verifies the dead line's own action - # never fires, across its whole match space, exhaustively -- that part - # of "unreachable" is sound. And whatever IS named here genuinely is - # evaluated before the dead line and genuinely does have the opposite - # action -- that is just what Blocking_Lines means. Stating only those - # two facts, and nothing about what therefore happens to "traffic - # matching it", is what stays true no matter what is missing from this - # list. - named = " ".join(match.group("blocking").split()) # Batfish pads its spaces - outcome_action = "DENY" if outcome == "denied" else "PERMIT" - lines, decide_verb, verb = ( - ("earlier lines", "are", "have") if len(blocking_lines) > 1 - else ("an earlier line", "is", "has") - ) - return ( - f"the unreachable line's own stated action is {dead_action}, but " - f"that action never takes effect: {lines} ({named}), which " - f"{decide_verb} evaluated first, {verb} the opposite action " - f"({outcome_action}) instead." - ) + # What the overclaim this used to guard against actually was, kept for + # the record: Blocking_Lines does not always name every earlier line, + # specifically one sharing the DEAD line's own action rather than the + # named blocking line's opposite one, so a claim like "traffic matching + # it is actually {outcome}" can be false for a subset of the dead + # line's own pattern. Measured: a real config had `permit host 10.1.1.5 + # any` before both `deny 10.1.1.0/24 any` (the named blocker) and the + # dead `permit 10.1.1.0/24 ... eq 443` -- Batfish named only the deny, + # so that sentence said "actually denied" for the whole /24, but + # testFilters showed host .5 is actually PERMITted, by the earlier, + # unnamed, same-action-as-the-dead-line permit. A fallback stating only + # the named line's own action, with no claim about what therefore + # happens to "traffic matching it", would have been safe from that + # specific overclaim -- it is just also, separately, the wording that + # measurably invites Example 1 copying. Refusing avoids both. + return None # ------------------------------------------------------------------------------ diff --git a/tests/test_explain_safety_net.py b/tests/test_explain_safety_net.py index 22fb4a0..2225667 100644 --- a/tests/test_explain_safety_net.py +++ b/tests/test_explain_safety_net.py @@ -373,27 +373,43 @@ def test_an_earlier_same_action_line_omitted_from_blocking_lines_is_not_overclai assert "traffic matching it" not in outcome -def test_an_unparseable_blocking_line_falls_back_without_a_universal_claim(): - """#453's own fix, exercised directly through its actual code path. - When the blocking line's shape _describe_acl_line() refuses (here, a - source-port clause #401's parser does not recognise), the contradicted - branch must still fall back to naming it in raw syntax -- but never to - the dropped universal-outcome claim ("traffic matching it is actually - {outcome}") #453 found false for a subset of the dead line's own - pattern. Nothing else in this file reaches this fallback, since every - other fixture's blocking line happens to parse -- without this test, - #453's fix would be silently untested.""" +def test_an_unparseable_blocking_line_returns_none_rather_than_a_weaker_claim(): + """#461 review (@shubhamkataria2005), measured live. #453's own fix + fell back to naming an unparseable blocking line in raw syntax, + stating only that it has the opposite action -- no "traffic matching + it" claim, specifically to avoid the overclaim the test below this one + pins. Measured anyway: 20 live generations on this exact fixture gave + 10/20 Modelfile Example 1 copies for that fallback sentence, against + 1/20 for #401's main-head wording and 0/20 for no computed fact at + all. A sentence with no outcome in it still leaves a gap, and the + model filled it with Example 1 more often than either alternative. + This must now return None -- the same refusal this function already + makes for unrecognised syntax and disagreeing blocking lines, not a + softer version of the dropped fallback sentence. Nothing else in this + file reaches this path, since every other fixture's blocking line + happens to parse -- without this test, #461's fix would be silently + untested.""" detail = ( "Unreachable line: permit tcp any any eq 443 (action PERMIT). " "Blocked by: deny tcp any eq 1024 any eq 443. Reason: BLOCKING_LINES" ) - outcome = _compute_dead_rule_outcome(detail) - assert outcome is not None - assert "own stated action is PERMIT" in outcome - assert "deny tcp any eq 1024 any eq 443" in outcome - assert "opposite action (DENY)" in outcome - assert "traffic matching it is actually" not in outcome - assert "actually denied" not in outcome + assert _compute_dead_rule_outcome(detail) is None + + +def test_one_unparseable_line_among_several_blocking_lines_also_returns_none(): + """The `all(d is not None for d in descriptions)` check already + requires every named blocking line to parse before taking the + plain-English path, so a mix of one parseable and one unparseable + line should fall through to the same refusal as a single unparseable + line -- not a partial description naming only the one that parsed. + Nothing pinned this specifically before #461; added alongside the + single-line case rather than left as an untested assumption.""" + detail = ( + "Unreachable line: permit tcp any any eq 443 (action PERMIT). " + "Blocked by: deny tcp 10.1.1.0 0.0.0.255 any, deny tcp any eq " + "1024 any eq 443. Reason: BLOCKING_LINES" + ) + assert _compute_dead_rule_outcome(detail) is None def _real_dead_rule_detail(dead_action: str, blocking_action: str) -> str: