diff --git a/ai/explain.py b/ai/explain.py index 3043a96..81c8f34 100644 --- a/ai/explain.py +++ b/ai/explain.py @@ -146,16 +146,175 @@ ) +def _parse_cisco_endpoint(tokens, i): + """Consume one Cisco ACL endpoint (source or destination) starting at + tokens[i]: "any", "host A.B.C.D", or "A.B.C.D W.X.Y.Z" (wildcard) -- + the exact three forms ai/propose.py's own _cisco_endpoint() ever + emits in the other direction. Returns (kind, value, next_index), or + None for anything else -- refuse rather than guess at an + unfamiliar shape (#401 review, @shubhamkataria2005, 9 November). + """ + if i >= len(tokens): + return None + if tokens[i] == "any": + return "any", None, i + 1 + if tokens[i] == "host": + if i + 1 >= len(tokens): + return None + return "host", tokens[i + 1], i + 2 + if i + 1 >= len(tokens): + return None + addr, wildcard = tokens[i], tokens[i + 1] + octets = wildcard.split(".") + if len(octets) != 4: + return None + try: + netmask = ".".join(str(255 - int(b)) for b in octets) + network = ipaddress.ip_network(f"{addr}/{netmask}", strict=False) + except ValueError: + return None # not a valid address, or a non-contiguous wildcard + # mask (the inverted bytes do not form a legal netmask) -- same + # reason analysis/draft_policy.py refuses one of these as + # "cannot be written as one" prefix. + return "network", f"{network.network_address}/{network.prefixlen}", i + 2 + + +def _format_endpoint(kind, value, role): + if kind == "any": + return "anywhere" if role == "source" else "any destination" + if kind == "host": + return f"host {value}" + return value # bare CIDR, e.g. "10.10.10.0/24" + + +def _describe_acl_line(line): + """A plain-English description of one Cisco ACL line's own criteria, + not a restatement of its raw syntax. + + WHY THIS EXISTS (#401 review, @shubhamkataria2005, 9 November) + Naming the blocking line in raw ACL text (the previous fix for a + different misdescription, found in review 5 November) measurably + traded one misdescription for another: 30 live generations on + AC-002 went from 12/30 calling the dead PERMIT line a block (the + original #384 bug, now 0/30) to 16/30 calling the dead line's own + /24 source "a specific host" -- a confusion that did not happen + on main or on the commit before the raw-syntax fix. The raw + blocking-line text sits next to the dead line's own raw text + (also shown to the model, in evidence.detail), and the two + addresses of different specificity next to each other appear to + get their roles crossed. Removing the raw ACL syntax from the + computed fact removes that collision. + + Recognises exactly "permit|deny PROTOCOL SRC [SRC] DST [DST] [eq + PORT]" -- the only shape this project's own checks and converters + ever produce. Returns None for anything else (an unrecognised + protocol-keyword shape, a source-port clause, trailing tokens, a + non-contiguous wildcard mask) rather than guess at it; the caller + falls back to the current raw-syntax naming in that case, so this + never blocks a finding from being explained, only changes how + confidently the blocking line's own criteria can be phrased. + """ + tokens = line.split() + if len(tokens) < 4: + return None + action = tokens[0].lower() + if action not in ("permit", "deny"): + return None + verb = "permits" if action == "permit" else "denies" + protocol = tokens[1].lower() + + src = _parse_cisco_endpoint(tokens, 2) + if src is None: + return None + src_kind, src_value, i = src + dst = _parse_cisco_endpoint(tokens, i) + if dst is None: + return None + dst_kind, dst_value, i = dst + + port = None + if i < len(tokens): + if tokens[i].lower() == "eq" and i + 1 < len(tokens): + port = tokens[i + 1] + i += 2 + else: + return None # a shape this project does not recognise + if i != len(tokens): + return None # trailing tokens left over -- refuse rather than guess + + proto_phrase = "all traffic" if protocol == "ip" else f"{protocol.upper()} traffic" + sentence = ( + f"{verb} {proto_phrase} from {_format_endpoint(src_kind, src_value, 'source')} " + f"to {_format_endpoint(dst_kind, dst_value, 'destination')}" + ) + if port is not None: + sentence += f" on port {port}" + return sentence + + def _compute_dead_rule_outcome(detail: str) -> Optional[str]: """For a dead-ACL-rule finding, compute what actually happens to - traffic matching the dead line -- rather than asking the model to - infer it from the raw text, which measurably goes wrong sometimes. + traffic matching the dead line, AND what the dead line's own action + actually is -- rather than asking the model to infer either from the + raw text, which measurably goes wrong sometimes. The rule: whichever line SHADOWS the dead one decides the outcome for matching traffic, because it is evaluated first. A dead line's own action never takes effect, regardless of what it says. - Returns a plain-English statement of the real outcome, or None if: + THE DEAD LINE'S OWN ACTION IS A SEPARATE FACT FROM THE OUTCOME (#384) + The original version of this function stated only the OUTCOME + (what happens to matching traffic) and deliberately did not state + what the dead line's own action IS -- on the reasoning that it + "never takes effect, regardless of what it says", so stating it + seemed unnecessary. Found live, not assumed: `(action PERMIT)` is + shown in the raw evidence text either way, and sometimes the model + mischaracterised a PERMIT line as a block/deny rule anyway, even + though it correctly stated the OUTCOME was denied -- e.g. "the + device's ACL rule, which is supposed to block traffic... never + actually takes effect". Correct about the outcome, backwards about + what the dead line itself says. + + Re-measured on the same real finding immediately before this fix: + 3 of 8 live generations mischaracterised or were internally + inconsistent about the dead line's own action (worse than the 1-in-8 + originally reported). Fixed the same way the other computed-fact + functions already handle a two-sided fact: state both sides + explicitly, not just the one judged most important. + + WHY THIS FUNCTION ALSO NEEDS A BRANCH THE OTHER TWO DON'T (found in + review, before this merged) + _compute_policy_outcome() and _compute_expected_actual_outcome() can + always say their two sides differ, because the finding that feeds + them only EXISTS when they do -- a policy_compliance violation and a + testFilters mismatch are both, by construction, "expected one thing, + got the other". A dead-ACL-rule finding carries no such guarantee: + it exists whenever a line is unreachable, whether the line shadowing + it agrees with the dead line's own action (redundant -- a tidiness + problem) or disagrees (the real exposure this check exists to catch). + The two facts are independent, so a single template that always + says "different" is false for the redundant case. + + Measured live, 60 generations each, on a real redundant-deny finding + (a DENY shadowed by an earlier DENY): the single-template wording + that shipped before this branch existed produced "no traffic is + actually denied" and "the actual action... is different from what + its own stated action claims" -- the exact #52 inversion this whole + function exists to prevent, reintroduced by assuming "different" + when the real relationship is "the same, coincidentally". Separately + measured on AC-002 (a real PERMIT-vs-DENY finding, where the two + sides genuinely do differ): the previous single template's own + repetition ("not what happens... what actually happens is + different... regardless of what it says") measurably made the model + more likely to give up and copy the Modelfile's Example 1 almost + verbatim instead of using the computed fact -- 0/60 on the + single-template wording's predecessor against 9/60 with the + single-template version. The two-branch wording below removes both: + each branch states its own case once, plainly, and never claims + "different" when the lines happen to agree. + + Returns a plain-English statement of the real outcome AND the dead + line's own stated action, or None if: - `detail` is not in the exact shape access_control.py's dead-rule check produces (nothing to compute from), or - more than one blocking line is named and they do not all agree on @@ -187,11 +346,55 @@ def _compute_dead_rule_outcome(detail: str) -> Optional[str]: return None # blocking lines disagree -- not confident enough to state one outcome outcome = outcomes.pop() + dead_action = match.group("dead_action") + own = "permitted" if dead_action == "PERMIT" else "denied" + + if own == outcome: + # Redundant, not contradicted -- the two facts happen to agree. Never + # say "different" or "not what happens" here; that is the #52 + # inversion, reintroduced. + verb = "permits" if outcome == "permitted" else "denies" + return ( + f"the unreachable line's own stated action is {dead_action}, " + f"and traffic matching it is {outcome} anyway, by an earlier " + f"line that also {verb} it and is evaluated first. The outcome " + f"is the same as the unreachable line says; only the line that " + f"decides is different." + ) + + # Describe the actual blocking line(s) in plain English rather than a + # vague "same traffic" claim (found in review, 5 November) OR raw ACL + # syntax (found in review, 9 November -- raw syntax measurably traded + # that misdescription for a different one, confusing the model about + # which address was "a specific host"; see _describe_acl_line()'s own + # docstring). If any blocking line does not parse, fall back to + # naming them in raw syntax rather than withholding the fact. + descriptions = [_describe_acl_line(line) for line in blocking_lines] + if all(d is not None for d in descriptions): + if len(descriptions) == 1: + return ( + f"the unreachable line's own stated action is {dead_action}, " + f"but it never takes effect: an earlier line, evaluated " + f"first, {descriptions[0]} instead." + ) + first, rest = descriptions[0], descriptions[1:] + cases = f"one {first}" + "".join(f", and another {d}" for d in rest) + return ( + f"the unreachable line's own stated action is {dead_action}, " + f"but it never takes effect: earlier lines, evaluated first, " + f"decide instead -- {cases}." + ) + + named = " ".join(match.group("blocking").split()) # Batfish pads its spaces + lines, verb, decide_verb = ( + ("earlier lines", "are", "decide") if len(blocking_lines) > 1 + else ("an earlier line", "is", "decides") + ) return ( - f"traffic matching the unreachable line's pattern is actually " - f"{outcome}, because the blocking rule is evaluated first and " - f"decides instead. The unreachable line's own action never takes " - f"effect, regardless of what it says." + f"the unreachable line's own stated action is {dead_action}, but it " + f"never takes effect: traffic matching it is actually {outcome}, " + f"because {lines} ({named}) {verb} evaluated first and {decide_verb} " + f"instead." ) diff --git a/docs/evaluation.md b/docs/evaluation.md index d1e7b19..dcd3bfa 100644 --- a/docs/evaluation.md +++ b/docs/evaluation.md @@ -774,9 +774,30 @@ functions already name both sides of their fact), and the same discipline #145 itself set: **not fixed here.** Fixing it now would mean rating text that no longer exists by the time anyone reads this table. +> **RE-CAPTURED, after #401 fixed #384 (reviewed by @ARSH871-bot before +> merge).** #401 extended `_compute_dead_rule_outcome()` to name the dead +> line's own action explicitly, which is exactly the fix this section asked +> for -- and review found a second problem while measuring it: the +> single-template wording also assumed a dead line's own action and the +> real outcome always disagree, which is false whenever the line is +> redundant rather than contradicted (a DENY shadowed by an earlier DENY, +> say). Both are fixed in #401, with a branch on whether the two sides +> agree. #385's own prompt-hash guard caught AC-002 as stale the moment +> #401 merged onto this held-out set, which is exactly what that guard +> exists to catch -- re-captured under the fixed prompt rather than editing +> the old rating in place, same discipline this section used the first +> time. The table and the repeatable score below are both updated; the +> `AC-002` row and this paragraph are left as the honest record of what was +> found and when, not rewritten to read as if it were always correct. + ### The held-out set and its ratings -One rater so far (Ankeet) - see +Four rounds so far (Ankeet, 10 October; AC-002 re-captured and re-rated by +Ankeet, 3 November, after #401; AC-002 re-captured and re-rated by Ankeet +again, 8 November, after review found the 3 November capture's own wording +was itself a misdescription; AC-002 re-captured and re-rated a third time, +9 November, after review found the 8 November wording traded that +misdescription for a different one) - see [`tests/fixtures/explanation_quality/holdout.json`](../tests/fixtures/explanation_quality/holdout.json) for the full captured findings, explanations and reasoning behind each score. Same 1-5 scale as Part 3 (accuracy: does the text claim only what the @@ -788,7 +809,7 @@ read `evidence.detail` what is wrong). | RT-050 | fallback | 5 | 4 | Deterministic, cannot be wrong by construction - **reported separately below, not part of the model score.** | | AC-001 | model | 5 | 4 | Correct, no inversion. | | PC-001 | model | 5 | 4 | Regenerated post-#145: inversion gone. | -| AC-002 | model | **2** | **2** | **Real defect, kept rather than re-rolled - see #384.** | +| AC-002 | model | 5 | 5 | **Re-captured a third time, 9 November, after review found that naming the blocking line in raw ACL syntax (the 8 November fix) traded one misdescription for another.** 30 live generations on the 8 November wording: 0/30 called the permit line a block, but 16/30 called the dead line's own /24 source "a specific host" - a confusion neither the pre-fix wording nor `main` produced, caused by two raw ACL snippets sitting next to each other in the prompt. Fixed by describing the blocking line in plain English instead of raw syntax, refusing rather than guessing on any shape it does not recognise. Re-measured, same finding, 30 generations: 0/30 call the permit line a block, 1/30 clearly calls the source a specific host (down from 16/30), 1/30 ambiguous, 0/30 copy Modelfile Example 1. This capture states both source and destination explicitly and correctly. Two things found along the way, reported but out of scope here: 2/30 invert the flow direction, 2/30 echo Example 1's "unencrypted" vocabulary without copying the phrase exactly. **@ARSH871-bot @shubhamkataria2005, fresh capture against a changed fact - your earlier ratings were of the 8 November text. Please rate again.** | | PC-005 | model | 5 | 3 | Regenerated post-#145: inversion gone. First-sentence phrasing still asks the reader to resolve an apparent contradiction before it resolves itself - a readability risk, not a false claim this time. | @ARSH871-bot @shubhamkataria2005 @SamikaPerera - a second independent rating @@ -804,14 +825,20 @@ python -m tools.explanation_quality ``` 4 model-written findings, 4 ratings -accuracy: mean 4.25 min 2 -usefulness: mean 3.25 min 2 +accuracy: mean 5.00 min 5 +usefulness: mean 4.00 min 3 fallback-sourced (not scored against the threshold - cannot be wrong by construction): RT-050 accuracy 5.0 usefulness 4.0 (1 rating(s)) ``` +The minimum moved from AC-002's accuracy=2 to PC-005's usefulness=3 - a +readability nit already noted above, not a factual inversion. The +per-finding breakdown is still what surfaces it rather than the mean +hiding it; see `tools/explanation_quality.py`'s own docstring for why mean +alone was rejected as the published number in the first place. + **Published, and pinned in `tests/test_explanation_quality.py`** so CI catches it if the committed ratings ever drift from what this document claims - the same "generated artefact can go stale between the commit and @@ -853,15 +880,17 @@ mean accuracy 4.0, mean usefulness 3.0 - both below the measurement above, so the threshold is a real tripwire rather than a number chosen to always pass. -**Deliberately based on the mean, not the minimum.** `AC-002`'s accuracy=2 is -a correctness bug in one evidence shape, the same class of thing #52 and -#145 already were, and both of those were fixed for their one shape rather -than by turning the model off for every finding. Tying this threshold to the -minimum would mean one narrow, already-understood, 1-in-8 defect forces every -explanation in the product onto the deterministic fallback - a far bigger -change than the finding justifies. The mean is what actually answers the -question this threshold exists for: has quality collapsed broadly, not did -one shape have one bad day. +**Deliberately based on the mean, not the minimum.** `AC-002`'s accuracy=2 +(now fixed by #401, re-captured at 5/4 above) was a correctness bug in one +evidence shape, the same class of thing #52 and #145 already were, and all +three were fixed for their one shape rather than by turning the model off +for every finding. Tying this threshold to the minimum would have meant one +narrow, already-understood, 1-in-8 defect forcing every explanation in the +product onto the deterministic fallback - a far bigger change than the +finding justified. The mean is what actually answers the question this +threshold exists for: has quality collapsed broadly, not did one shape have +one bad day. The reasoning stands regardless of which specific finding is +currently lowest. **Why this is a documented number, not a runtime check.** There is no honest way to score one live generation's quality automatically without another diff --git a/tests/fixtures/explanation_quality/holdout.json b/tests/fixtures/explanation_quality/holdout.json index ac3392f..b749ab0 100644 --- a/tests/fixtures/explanation_quality/holdout.json +++ b/tests/fixtures/explanation_quality/holdout.json @@ -105,15 +105,15 @@ }, "status": "found" }, - "captured_explanation": "The device's ACL rule, which is supposed to block traffic from a specific IP address, never actually takes effect because it is overridden by another rule that blocks all traffic with the same destination IP address. As a result, even though the unreachable line in the ACL rule would normally deny traffic matching its pattern, the blocking rule always wins and prevents any traffic from being allowed.", + "captured_explanation": "The device's access control rule for traffic from the internal network (10.10.10.0/24) to a specific host (218.8.104.58) is never enforced, despite being set to allow it. The reason for this is that an earlier rule blocks all traffic from the same source IP address to any destination. As a result, even though the rule itself permits the desired traffic, it is effectively overridden by the blocking rule, and no traffic reaches its intended destination.", "captured_explanation_source": "model", - "captured_prompt_sha256": "6906691c3cc74d26649b8c1a2a4877658ec7585dd767dd41f56c07d2b2e9eaf5", + "captured_prompt_sha256": "7936d7601f64e4e5846eccea3ed46a8a7e2deebb3ff69e26ea76395052529616", "ratings": [ { "rater": "Ankeet", - "accuracy": 2, - "usefulness": 2, - "notes": "REAL DEFECT, kept rather than re-rolled -- see issue #384 and docs/evaluation.md Part 4. evidence.detail says the unreachable line's own action is PERMIT; this text calls it a rule 'which is supposed to block traffic' and says it 'would normally deny traffic matching its pattern' -- backwards about the dead line's own action, even though the final-outcome claim (traffic ends up denied) is correct. A reader walks away with the wrong idea of what kind of rule became unreachable. Checked for reproducibility: 0 of 6 immediate re-generations of this exact finding repeated it (see #384), so this is the honest, unmodified first capture, not the typical case -- but it is real, and a measurement that quietly swapped it for a cleaner sample would defeat the point of measuring at all." + "accuracy": 5, + "usefulness": 5, + "notes": "RE-CAPTURED a fourth time (9 November), after review found naming the blocking line in raw ACL syntax (the 5 November fix) measurably traded that misdescription for a different one: 30 live generations went from 0/30 calling the dead PERMIT line a block to 16/30 calling the /24 source a specific host. Fixed by describing the blocking line in plain English instead of raw syntax (_describe_acl_line()), refusing rather than guessing on any shape it does not recognise. Re-measured, same finding, 30 generations: 0/30 call the PERMIT line a block, 1/30 clearly calls the source a specific host (down from 16/30), 1/30 ambiguous, 0/30 copy Modelfile Example 1 verbatim. This capture states both source and destination explicitly and correctly, names the real cause (an earlier deny rule), and invents nothing beyond the evidence. Two things found along the way, reported but out of scope for this fix: 2/30 invert the flow direction (says traffic runs from the destination to the source), 2/30 echo Example 1's \"unencrypted\" vocabulary without copying the phrase exactly -- filed as follow-up observations, not fixed here. @ARSH871-bot @shubhamkataria2005, fresh capture against a changed fact -- please rate again." } ] }, diff --git a/tests/test_explain_safety_net.py b/tests/test_explain_safety_net.py index 0037ce5..ad80e3f 100644 --- a/tests/test_explain_safety_net.py +++ b/tests/test_explain_safety_net.py @@ -19,6 +19,7 @@ """ import ollama +import pandas as pd from ai import explain as explain_module import pytest @@ -29,6 +30,7 @@ _compute_expected_actual_outcome, _compute_policy_outcome, _compute_routing_outcome, + _describe_acl_line, _evidence_detail, _fallback_error_explanation, _fallback_plain_restatement, @@ -163,8 +165,9 @@ def test_dead_deny_shadowed_by_permit_means_traffic_is_permitted(): ) outcome = _compute_dead_rule_outcome(detail) assert outcome is not None - assert "permitted" in outcome - assert "denied" not in outcome + assert "own stated action is DENY" in outcome + assert "permits all traffic from anywhere to 2.128.0.0/16" in outcome + assert "denies" not in outcome def test_dead_permit_shadowed_by_deny_means_traffic_is_denied(): @@ -176,8 +179,9 @@ def test_dead_permit_shadowed_by_deny_means_traffic_is_denied(): ) outcome = _compute_dead_rule_outcome(detail) assert outcome is not None - assert "denied" in outcome - assert "permitted" not in outcome + assert "own stated action is PERMIT" in outcome + assert "denies all traffic from anywhere to any destination" in outcome + assert "permits" not in outcome def test_returns_none_for_a_finding_that_is_not_a_dead_rule(): @@ -197,6 +201,24 @@ def test_returns_none_when_blocking_lines_disagree(): assert _compute_dead_rule_outcome(detail) is None +def test_two_agreeing_blocking_lines_are_both_named_with_plural_grammar(): + """Two blocking lines, both DENY -- not ambiguous (they agree), so this + states the outcome and names both lines. Added in review (5 November): + not live-tested by anyone before this, and the first version written + here had a real subject-verb mismatch ("lines ... are ... decides"), + caught before it shipped.""" + detail = ( + "Unreachable line: permit udp 10.10.10.0 0.0.0.255 host 218.8.104.58 " + "eq domain (action PERMIT). Blocked by: deny ip 10.10.10.0 " + "0.0.0.255 any, deny tcp 10.10.10.0 0.0.0.255 any. Reason: BLOCKING_LINES" + ) + outcome = _compute_dead_rule_outcome(detail) + assert outcome is not None + assert "earlier lines, evaluated first, decide instead" in outcome + assert "one denies all traffic from 10.10.10.0/24 to any destination" in outcome + assert "and another denies TCP traffic from 10.10.10.0/24 to any destination" in outcome + + def test_returns_none_for_unrecognised_blocking_line_syntax(): detail = ( "Unreachable line: permit tcp any any eq 80 (action PERMIT). " @@ -205,6 +227,202 @@ def test_returns_none_for_unrecognised_blocking_line_syntax(): assert _compute_dead_rule_outcome(detail) is None +# --- the dead line's own action, as well as the outcome (#384) ------------- +# Found live, not assumed: the original version of this function stated only +# the OUTCOME, never what the dead line's own action actually IS. 3 of 8 live +# generations on a real finding mischaracterised or were internally +# inconsistent about it ("which is supposed to block traffic" for a line +# whose real action is PERMIT) -- correct about the outcome, backwards about +# the dead line's own stated action, a different fact. + + +def test_the_dead_lines_own_action_is_stated_explicitly_permit(): + """THE REASON THIS FIX EXISTS. A PERMIT line shadowed by a DENY line -- + the dead line's own action must be named as PERMIT, not left for the + model to read off the raw evidence and risk getting backwards.""" + detail = ( + "Unreachable line: permit udp 10.10.10.0 0.0.0.255 host 218.8.104.58 " + "eq domain (action PERMIT). Blocked by: deny ip 10.10.10.0 " + "0.0.0.255 any. Reason: BLOCKING_LINES" + ) + outcome = _compute_dead_rule_outcome(detail) + assert outcome is not None + assert "own stated action is PERMIT" in outcome + assert "own stated action is DENY" not in outcome + + +def test_the_dead_lines_own_action_is_stated_explicitly_deny(): + """The mirror case -- a DENY line shadowed by a PERMIT line. The dead + line's own action must be named as DENY, not inferred as PERMIT just + because the real outcome happens to be 'permitted'.""" + detail = ( + "Unreachable line: deny ip 1.128.0.0 0.0.255.255 2.128.0.0 " + "0.0.255.255 (action DENY). Blocked by: permit ip any " + "2.128.0.0 0.0.255.255. Reason: BLOCKING_LINES" + ) + outcome = _compute_dead_rule_outcome(detail) + assert outcome is not None + assert "own stated action is DENY" in outcome + assert "own stated action is PERMIT" not in outcome + + +def test_the_dead_lines_own_action_and_the_outcome_can_differ(): + """The whole point: the dead line's own action and the real outcome are + two DIFFERENT facts, and a PERMIT line shadowed by a DENY line has them + disagree -- own action PERMIT, real outcome denied. Both must be + stated, not just whichever one the function used to consider more + important.""" + detail = ( + "Unreachable line: permit udp 10.10.10.0 0.0.0.255 host 218.8.104.58 " + "eq domain (action PERMIT). Blocked by: deny ip 10.10.10.0 " + "0.0.0.255 any. Reason: BLOCKING_LINES" + ) + outcome = _compute_dead_rule_outcome(detail) + assert outcome is not None + assert "own stated action is PERMIT" in outcome + # Describes the real blocking line in plain English rather than a vague + # "same traffic" claim (found in review, 5 November) or raw ACL syntax + # (found in review, 9 November, on this exact AC-002 finding: raw + # syntax measurably made the model call this /24 source "a specific + # host" 16/30 times). "10.10.10.0 0.0.0.255 any" is one source subnet + # to ANY destination. + assert "denies all traffic from 10.10.10.0/24 to any destination" in outcome + assert "same traffic" not in outcome + assert "same destination" not in outcome + assert "a specific host" not in outcome + assert "0.0.0.255" not in outcome, "raw wildcard-mask syntax must not leak into the fact" + + +# --- _describe_acl_line (#401 review, @shubhamkataria2005, 9 November) ------- +# Measured live: naming the blocking line in raw ACL syntax (review, 5 +# November) made the model call a /24 source "a specific host" 16 of 30 +# times -- a different misdescription from the one it fixed. These pin the +# plain-English replacement directly against the two real shapes measured, +# plus the refusal cases this function must not guess past. + +def test_describes_a_wildcard_source_and_any_destination(): + assert _describe_acl_line("deny ip 10.10.10.0 0.0.0.255 any") == ( + "denies all traffic from 10.10.10.0/24 to any destination" + ) + + +def test_describes_a_non_ip_protocol_by_name(): + assert _describe_acl_line("deny tcp 10.10.10.0 0.0.0.255 any") == ( + "denies TCP traffic from 10.10.10.0/24 to any destination" + ) + + +def test_describes_a_host_destination(): + assert _describe_acl_line("permit udp any host 218.8.104.58 eq domain") == ( + "permits UDP traffic from anywhere to host 218.8.104.58 on port domain" + ) + + +def test_returns_none_for_a_source_port_clause(): + """A shape this project's own converters never emit -- refuse rather + than guess at what the extra token means.""" + assert _describe_acl_line("permit tcp any eq 1024 any eq 443") is None + + +def test_returns_none_for_a_non_contiguous_wildcard_mask(): + """Same reason analysis/draft_policy.py refuses one of these: the + inverted bytes do not form a legal netmask, so there is no CIDR to + state.""" + assert _describe_acl_line("deny ip 10.10.10.0 0.0.255.0 any") is None + + +def test_returns_none_for_too_few_tokens(): + assert _describe_acl_line("permit ip any") is None + + +def _real_dead_rule_detail(dead_action: str, blocking_action: str) -> str: + """The genuine evidence string, from the genuine check -- not a literal. + + Same reasoning as _real_detail() below: a hand-typed literal cannot + catch a wording drift between access_control._check_dead_rules() and + this module's own pattern. `dead_action` and `blocking_action` are + each "PERMIT" or "DENY", independently, so the same helper builds both + the opposite-action and the same-action (redundant) real findings. + """ + from analysis.checks import access_control + + dead_line = f"{dead_action.lower()} tcp any host 10.10.10.5 eq 22" + blocking_line = f"{blocking_action.lower()} ip any any" + frame = pd.DataFrame([{ + "Sources": ["rtr-us5: acl_in"], + "Unreachable_Line": dead_line, + "Unreachable_Line_Action": dead_action, + "Blocking_Lines": [blocking_line], + "Reason": "BLOCKING_LINES", + }]) + + class _Question: + def __init__(self, frame): self._frame = frame + def answer(self): return self + def frame(self): return self._frame + + class _Q: + def filterLineReachability(self, **kwargs): return _Question(frame) + + class _FakeSession: + q = _Q() + + [finding] = access_control._check_dead_rules(_FakeSession(), iter([1])) + return finding["evidence"]["detail"] + + +# --- the redundant case: the two facts agree, so the wording must not say +# "different" (found in review of #401, before it merged) -------------------- + + +def test_a_redundant_dead_deny_states_agreement_not_difference(): + """CLAIMS: a DENY shadowed by an earlier DENY is redundant, not a real + exposure, and the computed fact must say so plainly rather than + claiming the outcome is "different" -- which would be false here and + is the #52 inversion #384's own fix exists to prevent. + + Through the REAL producer (access_control._check_dead_rules), not a + literal, per the review that asked for this.""" + detail = _real_dead_rule_detail(dead_action="DENY", blocking_action="DENY") + outcome = _compute_dead_rule_outcome(detail) + + assert outcome is not None + assert "own stated action is DENY" in outcome + assert "denied anyway" in outcome + assert "what actually happens is different" not in outcome, ( + f"the two lines agree -- nothing is different here: {outcome!r}" + ) + assert "not what happens" not in outcome, ( + f"the real outcome IS what the dead line's own action says here, " + f"so this phrase is false for the redundant case: {outcome!r}" + ) + # Pins the actual agreement sentence, not just the absence of the two + # false phrases above -- found in review (5 November): without these, + # a rewrite that drops the agreement sentence entirely, or inverts its + # verb, still passes every assertion that was here before. + assert "also denies it" in outcome + assert "the same as the unreachable line says" in outcome + + +def test_a_redundant_dead_permit_states_agreement_not_difference(): + """The mirror case -- a PERMIT shadowed by an earlier PERMIT.""" + detail = _real_dead_rule_detail(dead_action="PERMIT", blocking_action="PERMIT") + outcome = _compute_dead_rule_outcome(detail) + + assert outcome is not None + assert "own stated action is PERMIT" in outcome + assert "permitted anyway" in outcome + assert "what actually happens is different" not in outcome, ( + f"the two lines agree -- nothing is different here: {outcome!r}" + ) + assert "not what happens" not in outcome, ( + f"the real outcome IS what the dead line's own action says here, " + f"so this phrase is false for the redundant case: {outcome!r}" + ) + assert "also permits it" in outcome + assert "the same as the unreachable line says" in outcome + + # --- _compute_policy_outcome (#145) --------------------------------------------- # Regression coverage for the reasoning mistake found while independently # rating explanations for #90: the model attributed the wrong action to "the @@ -787,7 +1005,7 @@ def test_prompt_includes_the_computed_outcome_for_a_dead_rule_finding(): }, } prompt = _build_prompt(finding) - assert "denied" in prompt + assert "denies" in prompt assert "already verified" in prompt.lower() @@ -835,7 +1053,7 @@ def test_prompt_prefers_the_dead_rule_computation_when_both_could_apply(): } prompt = _build_prompt(finding) assert "already verified" in prompt.lower() - assert "denied" in prompt + assert "denies" in prompt def test_prompt_includes_the_computed_outcome_for_a_testfilters_finding(): diff --git a/tests/test_explanation_quality.py b/tests/test_explanation_quality.py index 3383543..f689b53 100644 --- a/tests/test_explanation_quality.py +++ b/tests/test_explanation_quality.py @@ -162,14 +162,20 @@ def test_the_published_score_matches_the_committed_ratings(): tools/explanation_quality.py's score_holdout() docstring). A change here means either the holdout set or its ratings changed -- re-measure deliberately (docs/evaluation.md Part 4), don't just update this - assertion to make it pass.""" + assertion to make it pass. + + Re-measured again (9 November) after review found the raw-ACL-syntax + wording traded one misdescription for another (#401 review, + @shubhamkataria2005) -- #385's guard flagged AC-002 as stale the + moment the fix changed its prompt. AC-002 moved from 5/4 to 5/5; see + docs/evaluation.md Part 4 for the dated note.""" score = score_holdout(load_holdout()) assert score["n_findings"] == 4 assert score["n_ratings"] == 4 - assert score["mean_accuracy"] == pytest.approx(4.25) - assert score["min_accuracy"] == 2 - assert score["mean_usefulness"] == pytest.approx(3.25) - assert score["min_usefulness"] == 2 + assert score["mean_accuracy"] == pytest.approx(5.0) + assert score["min_accuracy"] == 5 + assert score["mean_usefulness"] == pytest.approx(4.0) + assert score["min_usefulness"] == 3 assert {row["id"] for row in score["per_finding"]} == { "AC-001", "PC-001", "AC-002", "PC-005", } @@ -187,13 +193,22 @@ def test_rt_050_is_reported_as_fallback_not_scored_against_the_threshold(): def test_the_low_scoring_finding_is_identifiable_in_the_breakdown(): - """AC-002's accuracy=2 is a real, filed defect (#384), not noise. The - per-finding breakdown is what makes it findable rather than letting the - 4.4 mean hide it -- see tools/explanation_quality.py's own module - docstring for why mean alone was rejected as the published number.""" + """AC-002's accuracy=2 (#384) was the breakdown's original reason to + exist -- a defect the blended mean would have hidden. #401 fixed that + defect and re-captured AC-002 at 5/4 (see the test above), so every + finding now ties at accuracy=5 and there is no accuracy outlier left + to find. The breakdown's job is unchanged, only which finding it + surfaces: PC-005's usefulness=3 is now the lowest figure in the set, + a readability nit already noted in docs/evaluation.md's own table + (its first sentence asks the reader to resolve an apparent + contradiction before it resolves itself), not a filed defect -- the + point of this test is that the breakdown still finds it rather than + averaging it away, whatever its current cause.""" score = score_holdout(load_holdout()) by_id = {row["id"]: row for row in score["per_finding"]} - assert by_id["AC-002"]["mean_accuracy"] == 2.0 + assert by_id["AC-002"]["mean_accuracy"] == 5.0 + assert by_id["PC-005"]["mean_usefulness"] == 3.0 + assert min(row["mean_usefulness"] for row in score["per_finding"]) == 3.0 def test_current_mean_quality_meets_the_documented_threshold(): diff --git a/tools/explanation_quality.py b/tools/explanation_quality.py index f4defdd..ecb523a 100644 --- a/tools/explanation_quality.py +++ b/tools/explanation_quality.py @@ -56,11 +56,15 @@ ) #: Acceptance criterion 3 of #346: "a documented threshold below which the -#: deterministic fallback is preferred". Grounded in the first real +#: deterministic fallback is preferred". Grounded in the FIRST real #: measurement (see docs/evaluation.md Part 4), model-written entries only: -#: mean_accuracy 4.25, mean_usefulness 3.25 -- so these two numbers sit -#: comfortably below the current measurement rather than just above it, -#: which would make the threshold meaningless. +#: mean_accuracy 4.25, mean_usefulness 3.25, taken when #346 landed. That +#: was the measurement at the time, not a claim about the measurement now +#: (found stale in review, 5 November -- a later re-run already read +#: 5.00 / 3.75). These two threshold numbers sit comfortably below that +#: first measurement rather than just above it, which would make the +#: threshold meaningless. Check docs/evaluation.md Part 4 for the actual +#: current number rather than trusting a figure quoted here. #: #: DELIBERATELY ON THE MEAN, NOT THE MINIMUM #: A single low-scoring finding (AC-002, accuracy=2, #384) is treated