diff --git a/ai/explain.py b/ai/explain.py index 81c8f34..9d261c9 100644 --- a/ai/explain.py +++ b/ai/explain.py @@ -327,6 +327,20 @@ def _compute_dead_rule_outcome(detail: str) -> Optional[str]: generates an explanation from the raw finding either way. This function only ever ADDS confidence; it never blocks a finding from being explained. + + WHAT "THE REAL OUTCOME" MEANS FOR THE CONTRADICTED CASE (#453) + When the blocking line's action disagrees with the dead line's + own action, this no longer claims a single outcome for ALL + traffic matching the dead line's pattern -- see the comment + above that branch's return statement for why that claim is not + always true and what is stated instead. The redundant case + (blocking line agrees with the dead line) is unaffected -- see + that branch's own comment for why. + + If a blocking line's action is known but its shape cannot be + rendered in plain English, this now returns None instead of + naming it in raw syntax (#461) -- see the comment above that + return statement for the measurement behind it. """ match = _DEAD_RULE_DETAIL_PATTERN.search(detail) if not match: @@ -367,8 +381,10 @@ def _compute_dead_rule_outcome(detail: str) -> Optional[str]: # syntax (found in review, 9 November -- raw syntax measurably traded # that misdescription for a different one, confusing the model about # which address was "a specific host"; see _describe_acl_line()'s own - # docstring). If any blocking line does not parse, fall back to - # naming them in raw syntax rather than withholding the fact. + # docstring). This path only ever states what the blocking line itself + # does, never a claim about the dead line's whole pattern, so it does + # not carry #453's overclaim risk either (see the fallback below for + # what that risk was). descriptions = [_describe_acl_line(line) for line in blocking_lines] if all(d is not None for d in descriptions): if len(descriptions) == 1: @@ -385,17 +401,37 @@ def _compute_dead_rule_outcome(detail: str) -> Optional[str]: f"decide instead -- {cases}." ) - named = " ".join(match.group("blocking").split()) # Batfish pads its spaces - lines, verb, decide_verb = ( - ("earlier lines", "are", "decide") if len(blocking_lines) > 1 - else ("an earlier line", "is", "decides") - ) - return ( - f"the unreachable line's own stated action is {dead_action}, but it " - f"never takes effect: traffic matching it is actually {outcome}, " - f"because {lines} ({named}) {verb} evaluated first and {decide_verb} " - f"instead." - ) + # If any blocking line does not parse, refuse rather than fall back to + # naming it in raw syntax (#461 review, @shubhamkataria2005). #453's own + # fix tried a raw-syntax fallback here, stating only that the named + # line has the opposite action -- no claim about "traffic matching it", + # specifically to avoid the universal-outcome overclaim described below. + # Measured anyway, 20 live generations on the exact fixture that + # fallback exists for: 10/20 copied the Modelfile's Example 1 almost + # verbatim, against 1/20 for #401's main-head wording and 0/20 for no + # computed fact at all. A sentence with no outcome in it still leaves a + # gap for the model to fill, and it filled it with Example 1 more often + # than either alternative. Returning None here is not a weaker version + # of that fallback; it is the same refusal this function already makes + # a few lines up for unrecognised syntax and disagreeing blocking + # lines -- "not confident enough to say" covers this case too. + # + # What the overclaim this used to guard against actually was, kept for + # the record: Blocking_Lines does not always name every earlier line, + # specifically one sharing the DEAD line's own action rather than the + # named blocking line's opposite one, so a claim like "traffic matching + # it is actually {outcome}" can be false for a subset of the dead + # line's own pattern. Measured: a real config had `permit host 10.1.1.5 + # any` before both `deny 10.1.1.0/24 any` (the named blocker) and the + # dead `permit 10.1.1.0/24 ... eq 443` -- Batfish named only the deny, + # so that sentence said "actually denied" for the whole /24, but + # testFilters showed host .5 is actually PERMITted, by the earlier, + # unnamed, same-action-as-the-dead-line permit. A fallback stating only + # the named line's own action, with no claim about what therefore + # happens to "traffic matching it", would have been safe from that + # specific overclaim -- it is just also, separately, the wording that + # measurably invites Example 1 copying. Refusing avoids both. + return None # ------------------------------------------------------------------------------ diff --git a/docs/evaluation.md b/docs/evaluation.md index dcd3bfa..5f9c47b 100644 --- a/docs/evaluation.md +++ b/docs/evaluation.md @@ -797,7 +797,10 @@ Ankeet, 3 November, after #401; AC-002 re-captured and re-rated by Ankeet again, 8 November, after review found the 3 November capture's own wording was itself a misdescription; AC-002 re-captured and re-rated a third time, 9 November, after review found the 8 November wording traded that -misdescription for a different one) - see +misdescription for a different one -- #453 also changed this computed +fact's fallback wording the same day, rebased onto this fix once pushed, +and does not change this particular capture, since this finding's +blocking line takes the unaffected plain-English path either way) - see [`tests/fixtures/explanation_quality/holdout.json`](../tests/fixtures/explanation_quality/holdout.json) for the full captured findings, explanations and reasoning behind each score. Same 1-5 scale as Part 3 (accuracy: does the text claim only what the @@ -809,7 +812,7 @@ read `evidence.detail` what is wrong). | RT-050 | fallback | 5 | 4 | Deterministic, cannot be wrong by construction - **reported separately below, not part of the model score.** | | AC-001 | model | 5 | 4 | Correct, no inversion. | | PC-001 | model | 5 | 4 | Regenerated post-#145: inversion gone. | -| AC-002 | model | 5 | 5 | **Re-captured a third time, 9 November, after review found that naming the blocking line in raw ACL syntax (the 8 November fix) traded one misdescription for another.** 30 live generations on the 8 November wording: 0/30 called the permit line a block, but 16/30 called the dead line's own /24 source "a specific host" - a confusion neither the pre-fix wording nor `main` produced, caused by two raw ACL snippets sitting next to each other in the prompt. Fixed by describing the blocking line in plain English instead of raw syntax, refusing rather than guessing on any shape it does not recognise. Re-measured, same finding, 30 generations: 0/30 call the permit line a block, 1/30 clearly calls the source a specific host (down from 16/30), 1/30 ambiguous, 0/30 copy Modelfile Example 1. This capture states both source and destination explicitly and correctly. Two things found along the way, reported but out of scope here: 2/30 invert the flow direction, 2/30 echo Example 1's "unencrypted" vocabulary without copying the phrase exactly. **@ARSH871-bot @shubhamkataria2005, fresh capture against a changed fact - your earlier ratings were of the 8 November text. Please rate again.** | +| AC-002 | model | 5 | 5 | **Re-captured a third time, 9 November, after review found that naming the blocking line in raw ACL syntax (the 8 November fix) traded one misdescription for another.** 30 live generations on the 8 November wording: 0/30 called the permit line a block, but 16/30 called the dead line's own /24 source "a specific host" - a confusion neither the pre-fix wording nor `main` produced, caused by two raw ACL snippets sitting next to each other in the prompt. Fixed by describing the blocking line in plain English instead of raw syntax, refusing rather than guessing on any shape it does not recognise. Re-measured, same finding, 30 generations: 0/30 call the permit line a block, 1/30 clearly calls the source a specific host (down from 16/30), 1/30 ambiguous, 0/30 copy Modelfile Example 1. This capture states both source and destination explicitly and correctly. Two things found along the way, reported but out of scope here: 2/30 invert the flow direction, 2/30 echo Example 1's "unencrypted" vocabulary without copying the phrase exactly. #453 (dead-rule fact completeness) also changed this function's fallback wording the same day, for the case where a blocking line cannot be parsed in plain English - this finding's blocking line always could, so that change does not affect this capture. **@ARSH871-bot @shubhamkataria2005, fresh capture against a changed fact - your earlier ratings were of the 8 November text. Please rate again.** | | PC-005 | model | 5 | 3 | Regenerated post-#145: inversion gone. First-sentence phrasing still asks the reader to resolve an apparent contradiction before it resolves itself - a readability risk, not a false claim this time. | @ARSH871-bot @shubhamkataria2005 @SamikaPerera - a second independent rating diff --git a/tests/fixtures/explanation_quality/holdout.json b/tests/fixtures/explanation_quality/holdout.json index d3a3bcb..2ebaf90 100644 --- a/tests/fixtures/explanation_quality/holdout.json +++ b/tests/fixtures/explanation_quality/holdout.json @@ -113,7 +113,7 @@ "rater": "Ankeet", "accuracy": 5, "usefulness": 5, - "notes": "RE-CAPTURED a fourth time (9 November), after review found naming the blocking line in raw ACL syntax (the 5 November fix) measurably traded that misdescription for a different one: 30 live generations went from 0/30 calling the dead PERMIT line a block to 16/30 calling the /24 source a specific host. Fixed by describing the blocking line in plain English instead of raw syntax (_describe_acl_line()), refusing rather than guessing on any shape it does not recognise. Re-measured, same finding, 30 generations: 0/30 call the PERMIT line a block, 1/30 clearly calls the source a specific host (down from 16/30), 1/30 ambiguous, 0/30 copy Modelfile Example 1 verbatim. This capture states both source and destination explicitly and correctly, names the real cause (an earlier deny rule), and invents nothing beyond the evidence. Two things found along the way, reported but out of scope for this fix: 2/30 invert the flow direction (says traffic runs from the destination to the source), 2/30 echo Example 1's \"unencrypted\" vocabulary without copying the phrase exactly -- filed as follow-up observations, not fixed here. @ARSH871-bot @shubhamkataria2005, fresh capture against a changed fact -- please rate again." + "notes": "RE-CAPTURED a fourth time (9 November), after review found naming the blocking line in raw ACL syntax (the 5 November fix) measurably traded that misdescription for a different one: 30 live generations went from 0/30 calling the dead PERMIT line a block to 16/30 calling the /24 source a specific host. Fixed by describing the blocking line in plain English instead of raw syntax (_describe_acl_line()), refusing rather than guessing on any shape it does not recognise. Re-measured, same finding, 30 generations: 0/30 call the PERMIT line a block, 1/30 clearly calls the source a specific host (down from 16/30), 1/30 ambiguous, 0/30 copy Modelfile Example 1 verbatim. This capture states both source and destination explicitly and correctly, names the real cause (an earlier deny rule), and invents nothing beyond the evidence. Two things found along the way, reported but out of scope for this fix: 2/30 invert the flow direction (says traffic runs from the destination to the source), 2/30 echo Example 1's \"unencrypted\" vocabulary without copying the phrase exactly -- filed as follow-up observations, not fixed here. #453 (dead-rule fact completeness) also changed this function's fallback wording the same day, for a blocking line that cannot be parsed in plain English -- this finding's blocking line always could, so that change does not affect this capture. @ARSH871-bot @shubhamkataria2005, fresh capture against a changed fact -- please rate again." } ] }, diff --git a/tests/test_explain_safety_net.py b/tests/test_explain_safety_net.py index ad80e3f..2225667 100644 --- a/tests/test_explain_safety_net.py +++ b/tests/test_explain_safety_net.py @@ -168,6 +168,7 @@ def test_dead_deny_shadowed_by_permit_means_traffic_is_permitted(): assert "own stated action is DENY" in outcome assert "permits all traffic from anywhere to 2.128.0.0/16" in outcome assert "denies" not in outcome + assert "own stated action is PERMIT" not in outcome def test_dead_permit_shadowed_by_deny_means_traffic_is_denied(): @@ -182,6 +183,7 @@ def test_dead_permit_shadowed_by_deny_means_traffic_is_denied(): assert "own stated action is PERMIT" in outcome assert "denies all traffic from anywhere to any destination" in outcome assert "permits" not in outcome + assert "own stated action is DENY" not in outcome def test_returns_none_for_a_finding_that_is_not_a_dead_rule(): @@ -269,9 +271,9 @@ def test_the_dead_lines_own_action_is_stated_explicitly_deny(): def test_the_dead_lines_own_action_and_the_outcome_can_differ(): """The whole point: the dead line's own action and the real outcome are two DIFFERENT facts, and a PERMIT line shadowed by a DENY line has them - disagree -- own action PERMIT, real outcome denied. Both must be - stated, not just whichever one the function used to consider more - important.""" + disagree -- own action PERMIT, the blocking line's action DENY. Both + must be stated, not just whichever one the function used to consider + more important.""" detail = ( "Unreachable line: permit udp 10.10.10.0 0.0.0.255 host 218.8.104.58 " "eq domain (action PERMIT). Blocked by: deny ip 10.10.10.0 " @@ -291,6 +293,13 @@ def test_the_dead_lines_own_action_and_the_outcome_can_differ(): assert "same destination" not in outcome assert "a specific host" not in outcome assert "0.0.0.255" not in outcome, "raw wildcard-mask syntax must not leak into the fact" + # #453: must not claim a universal outcome for every packet the dead + # line's pattern would match either -- this plain-English path never + # did (it only ever describes what the blocking line itself does), but + # pin it anyway since it is exactly the phrasing that used to appear. + assert "traffic matching it is actually" not in outcome + assert "actually denied" not in outcome + assert "actually permitted" not in outcome # --- _describe_acl_line (#401 review, @shubhamkataria2005, 9 November) ------- @@ -335,6 +344,74 @@ def test_returns_none_for_too_few_tokens(): assert _describe_acl_line("permit ip any") is None +def test_an_earlier_same_action_line_omitted_from_blocking_lines_is_not_overclaimed(): + """#453, measured live. Blocking_Lines named only the opposite-action + deny; a host-specific PERMIT even earlier than that deny, sharing the + dead line's own PERMIT action, was not named at all and decides that + host's traffic the other way. The computed fact must not claim the + deny's outcome holds for every packet the dead line's wider pattern + would have matched -- it has no way to know that from this detail + string alone. This fixture's blocking line parses through + _describe_acl_line() (#401), so it takes that plain-English path, not + the raw-syntax fallback #453 was originally written against -- updated + accordingly rather than left pinning wording the code no longer + produces.""" + detail = ( + "Unreachable line: permit tcp 10.1.1.0 0.0.0.255 any eq 443 " + "(action PERMIT). Blocked by: deny tcp 10.1.1.0 0.0.0.255 any. " + "Reason: BLOCKING_LINES" + ) + outcome = _compute_dead_rule_outcome(detail) + assert outcome is not None + assert "own stated action is PERMIT" in outcome + assert "denies TCP traffic from 10.1.1.0/24 to any destination" in outcome + # The overclaim #453 found: asserting this for ALL traffic matching + # the dead line's own pattern is false for host 10.1.1.5 specifically + # (real testFilters: PERMIT), because of a line even earlier than the + # one named here. Confirm the fact no longer makes that claim. + assert "actually denied" not in outcome + assert "traffic matching it" not in outcome + + +def test_an_unparseable_blocking_line_returns_none_rather_than_a_weaker_claim(): + """#461 review (@shubhamkataria2005), measured live. #453's own fix + fell back to naming an unparseable blocking line in raw syntax, + stating only that it has the opposite action -- no "traffic matching + it" claim, specifically to avoid the overclaim the test below this one + pins. Measured anyway: 20 live generations on this exact fixture gave + 10/20 Modelfile Example 1 copies for that fallback sentence, against + 1/20 for #401's main-head wording and 0/20 for no computed fact at + all. A sentence with no outcome in it still leaves a gap, and the + model filled it with Example 1 more often than either alternative. + This must now return None -- the same refusal this function already + makes for unrecognised syntax and disagreeing blocking lines, not a + softer version of the dropped fallback sentence. Nothing else in this + file reaches this path, since every other fixture's blocking line + happens to parse -- without this test, #461's fix would be silently + untested.""" + detail = ( + "Unreachable line: permit tcp any any eq 443 (action PERMIT). " + "Blocked by: deny tcp any eq 1024 any eq 443. Reason: BLOCKING_LINES" + ) + assert _compute_dead_rule_outcome(detail) is None + + +def test_one_unparseable_line_among_several_blocking_lines_also_returns_none(): + """The `all(d is not None for d in descriptions)` check already + requires every named blocking line to parse before taking the + plain-English path, so a mix of one parseable and one unparseable + line should fall through to the same refusal as a single unparseable + line -- not a partial description naming only the one that parsed. + Nothing pinned this specifically before #461; added alongside the + single-line case rather than left as an untested assumption.""" + detail = ( + "Unreachable line: permit tcp any any eq 443 (action PERMIT). " + "Blocked by: deny tcp 10.1.1.0 0.0.0.255 any, deny tcp any eq " + "1024 any eq 443. Reason: BLOCKING_LINES" + ) + assert _compute_dead_rule_outcome(detail) is None + + def _real_dead_rule_detail(dead_action: str, blocking_action: str) -> str: """The genuine evidence string, from the genuine check -- not a literal.