From c7cf47e7d10bcd66533366729c8e062b38f74223 Mon Sep 17 00:00:00 2001 From: patelankeet2 Date: Wed, 14 Oct 2026 09:42:33 +1300 Subject: [PATCH 1/7] Name the dead line's own stated action explicitly, not just the outcome (#384) _compute_dead_rule_outcome() already captured the dead line's own action via its regex ('(?PPERMIT|DENY)') but never used it -- the computed fact stated only the real outcome, never what the dead line itself says, on the reasoning that the action 'never takes effect, regardless of what it says' made stating it unnecessary. Found live, not assumed (#384): the model sometimes mischaracterised the dead line's own action anyway, even while correctly stating the outcome -- 'which is supposed to block traffic' for a line whose real action is PERMIT. Correct about the outcome, backwards about a different fact. Re-measured on the same real finding (AC-002, rtr-us5-messy) immediately before fixing: 3 of 8 live generations mischaracterised or were internally inconsistent about the dead line's own action, worse than the 1-in-8 originally reported in #384. Re-measured again with the fix in place: 0 of 8. Both facts are now stated explicitly, the same discipline the other three computed-fact functions already use for their own two-sided facts -- the dead line's own stated action, and the real outcome, named as two separate things rather than leaving the model to read one of them off the raw evidence and risk inverting it. python -m pytest tests/test_explain_safety_net.py -q -> 81 passed (was 78). --- ai/explain.py | 39 ++++++++++++++++++---- tests/test_explain_safety_net.py | 56 ++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 7 deletions(-) diff --git a/ai/explain.py b/ai/explain.py index fcfa8b1..8126c07 100644 --- a/ai/explain.py +++ b/ai/explain.py @@ -148,14 +148,36 @@ def _compute_dead_rule_outcome(detail: str) -> Optional[str]: """For a dead-ACL-rule finding, compute what actually happens to - traffic matching the dead line -- rather than asking the model to - infer it from the raw text, which measurably goes wrong sometimes. + traffic matching the dead line, AND what the dead line's own action + actually is -- rather than asking the model to infer either from the + raw text, which measurably goes wrong sometimes. The rule: whichever line SHADOWS the dead one decides the outcome for matching traffic, because it is evaluated first. A dead line's own action never takes effect, regardless of what it says. - Returns a plain-English statement of the real outcome, or None if: + THE DEAD LINE'S OWN ACTION IS A SEPARATE FACT FROM THE OUTCOME (#384) + The original version of this function stated only the OUTCOME + (what happens to matching traffic) and deliberately did not state + what the dead line's own action IS -- on the reasoning that it + "never takes effect, regardless of what it says", so stating it + seemed unnecessary. Found live, not assumed: `(action PERMIT)` is + shown in the raw evidence text either way, and sometimes the model + mischaracterised a PERMIT line as a block/deny rule anyway, even + though it correctly stated the OUTCOME was denied -- e.g. "the + device's ACL rule, which is supposed to block traffic... never + actually takes effect". Correct about the outcome, backwards about + what the dead line itself says. + + Re-measured on the same real finding immediately before this fix: + 3 of 8 live generations mischaracterised or were internally + inconsistent about the dead line's own action (worse than the 1-in-8 + originally reported). Fixed the same way the other computed-fact + functions already handle a two-sided fact: state both sides + explicitly, not just the one judged most important. + + Returns a plain-English statement of the real outcome AND the dead + line's own stated action, or None if: - `detail` is not in the exact shape access_control.py's dead-rule check produces (nothing to compute from), or - more than one blocking line is named and they do not all agree on @@ -187,11 +209,14 @@ def _compute_dead_rule_outcome(detail: str) -> Optional[str]: return None # blocking lines disagree -- not confident enough to state one outcome outcome = outcomes.pop() + dead_action = match.group("dead_action") return ( - f"traffic matching the unreachable line's pattern is actually " - f"{outcome}, because the blocking rule is evaluated first and " - f"decides instead. The unreachable line's own action never takes " - f"effect, regardless of what it says." + f"the unreachable line's own stated action is {dead_action} -- that " + f"is what it says, not what happens. What actually happens is " + f"different: traffic matching the unreachable line's pattern is " + f"actually {outcome}, because the blocking rule is evaluated first " + f"and decides instead. The unreachable line's own action never " + f"takes effect, regardless of what it says." ) diff --git a/tests/test_explain_safety_net.py b/tests/test_explain_safety_net.py index bf99281..f119440 100644 --- a/tests/test_explain_safety_net.py +++ b/tests/test_explain_safety_net.py @@ -204,6 +204,62 @@ def test_returns_none_for_unrecognised_blocking_line_syntax(): assert _compute_dead_rule_outcome(detail) is None +# --- the dead line's own action, as well as the outcome (#384) ------------- +# Found live, not assumed: the original version of this function stated only +# the OUTCOME, never what the dead line's own action actually IS. 3 of 8 live +# generations on a real finding mischaracterised or were internally +# inconsistent about it ("which is supposed to block traffic" for a line +# whose real action is PERMIT) -- correct about the outcome, backwards about +# the dead line's own stated action, a different fact. + + +def test_the_dead_lines_own_action_is_stated_explicitly_permit(): + """THE REASON THIS FIX EXISTS. A PERMIT line shadowed by a DENY line -- + the dead line's own action must be named as PERMIT, not left for the + model to read off the raw evidence and risk getting backwards.""" + detail = ( + "Unreachable line: permit udp 10.10.10.0 0.0.0.255 host 218.8.104.58 " + "eq domain (action PERMIT). Blocked by: deny ip 10.10.10.0 " + "0.0.0.255 any. Reason: BLOCKING_LINES" + ) + outcome = _compute_dead_rule_outcome(detail) + assert outcome is not None + assert "own stated action is PERMIT" in outcome + assert "own stated action is DENY" not in outcome + + +def test_the_dead_lines_own_action_is_stated_explicitly_deny(): + """The mirror case -- a DENY line shadowed by a PERMIT line. The dead + line's own action must be named as DENY, not inferred as PERMIT just + because the real outcome happens to be 'permitted'.""" + detail = ( + "Unreachable line: deny ip 1.128.0.0 0.0.255.255 2.128.0.0 " + "0.0.255.255 (action DENY). Blocked by: permit ip any " + "2.128.0.0 0.0.255.255. Reason: BLOCKING_LINES" + ) + outcome = _compute_dead_rule_outcome(detail) + assert outcome is not None + assert "own stated action is DENY" in outcome + assert "own stated action is PERMIT" not in outcome + + +def test_the_dead_lines_own_action_and_the_outcome_can_differ(): + """The whole point: the dead line's own action and the real outcome are + two DIFFERENT facts, and a PERMIT line shadowed by a DENY line has them + disagree -- own action PERMIT, real outcome denied. Both must be + stated, not just whichever one the function used to consider more + important.""" + detail = ( + "Unreachable line: permit udp 10.10.10.0 0.0.0.255 host 218.8.104.58 " + "eq domain (action PERMIT). Blocked by: deny ip 10.10.10.0 " + "0.0.0.255 any. Reason: BLOCKING_LINES" + ) + outcome = _compute_dead_rule_outcome(detail) + assert outcome is not None + assert "own stated action is PERMIT" in outcome + assert "actually denied" in outcome + + # --- _compute_policy_outcome (#145) --------------------------------------------- # Regression coverage for the reasoning mistake found while independently # rating explanations for #90: the model attributed the wrong action to "the From ae84624e2ab06f3eba851eec6b7fdde6a0382fdd Mon Sep 17 00:00:00 2001 From: patelankeet2 Date: Wed, 4 Nov 2026 11:33:40 +1300 Subject: [PATCH 2/7] Branch the dead-line computed fact on whether the two actions agree A dead ACL line's own stated action and the real outcome are independent facts, unlike the other two computed-fact functions, whose two sides are guaranteed to differ by construction of when their finding even exists. The single template that always said "what actually happens is different" was false whenever the dead line was redundant rather than contradicted (a DENY shadowed by an earlier DENY, say) -- reintroducing the exact #52 inversion this function exists to prevent, measured live going from ~6/60 wrong to ~23/60 wrong on that case. The old template's own repetition also measurably made the model more likely to give up and copy the Modelfile's Example 1 verbatim on AC-002 (a genuine opposite-action case): 9/60 against 0/60 before. The simpler, single-statement wording for the differing-action branch removes that without changing the fact it states. Two new tests exercise the redundant case through the real producer (access_control._check_dead_rules), not a literal, asserting the output never claims the outcome is different when it agrees. Found and specified by review before this PR merged. --- ai/explain.py | 56 +++++++++++++++++++--- tests/test_explain_safety_net.py | 81 ++++++++++++++++++++++++++++++++ 2 files changed, 131 insertions(+), 6 deletions(-) diff --git a/ai/explain.py b/ai/explain.py index 8126c07..847b9a3 100644 --- a/ai/explain.py +++ b/ai/explain.py @@ -176,6 +176,37 @@ def _compute_dead_rule_outcome(detail: str) -> Optional[str]: functions already handle a two-sided fact: state both sides explicitly, not just the one judged most important. + WHY THIS FUNCTION ALSO NEEDS A BRANCH THE OTHER TWO DON'T (found in + review, before this merged) + _compute_policy_outcome() and _compute_expected_actual_outcome() can + always say their two sides differ, because the finding that feeds + them only EXISTS when they do -- a policy_compliance violation and a + testFilters mismatch are both, by construction, "expected one thing, + got the other". A dead-ACL-rule finding carries no such guarantee: + it exists whenever a line is unreachable, whether the line shadowing + it agrees with the dead line's own action (redundant -- a tidiness + problem) or disagrees (the real exposure this check exists to catch). + The two facts are independent, so a single template that always + says "different" is false for the redundant case. + + Measured live, 60 generations each, on a real redundant-deny finding + (a DENY shadowed by an earlier DENY): the single-template wording + that shipped before this branch existed produced "no traffic is + actually denied" and "the actual action... is different from what + its own stated action claims" -- the exact #52 inversion this whole + function exists to prevent, reintroduced by assuming "different" + when the real relationship is "the same, coincidentally". Separately + measured on AC-002 (a real PERMIT-vs-DENY finding, where the two + sides genuinely do differ): the previous single template's own + repetition ("not what happens... what actually happens is + different... regardless of what it says") measurably made the model + more likely to give up and copy the Modelfile's Example 1 almost + verbatim instead of using the computed fact -- 0/60 on the + single-template wording's predecessor against 9/60 with the + single-template version. The two-branch wording below removes both: + each branch states its own case once, plainly, and never claims + "different" when the lines happen to agree. + Returns a plain-English statement of the real outcome AND the dead line's own stated action, or None if: - `detail` is not in the exact shape access_control.py's dead-rule @@ -210,13 +241,26 @@ def _compute_dead_rule_outcome(detail: str) -> Optional[str]: outcome = outcomes.pop() dead_action = match.group("dead_action") + own = "permitted" if dead_action == "PERMIT" else "denied" + + if own == outcome: + # Redundant, not contradicted -- the two facts happen to agree. Never + # say "different" or "not what happens" here; that is #444's exact + # shape, reintroduced. + verb = "permits" if outcome == "permitted" else "denies" + return ( + f"the unreachable line's own stated action is {dead_action}, " + f"and traffic matching it is {outcome} anyway, by an earlier " + f"line that also {verb} it and is evaluated first. The outcome " + f"is the same as the unreachable line says; only the line that " + f"decides is different." + ) + return ( - f"the unreachable line's own stated action is {dead_action} -- that " - f"is what it says, not what happens. What actually happens is " - f"different: traffic matching the unreachable line's pattern is " - f"actually {outcome}, because the blocking rule is evaluated first " - f"and decides instead. The unreachable line's own action never " - f"takes effect, regardless of what it says." + f"the unreachable line's own stated action is {dead_action}, but it " + f"never takes effect: traffic matching it is actually {outcome}, " + f"because an earlier line that matches the same traffic is " + f"evaluated first and decides instead." ) diff --git a/tests/test_explain_safety_net.py b/tests/test_explain_safety_net.py index f119440..f63b6e9 100644 --- a/tests/test_explain_safety_net.py +++ b/tests/test_explain_safety_net.py @@ -19,6 +19,7 @@ """ import ollama +import pandas as pd from ai import explain as explain_module import pytest @@ -260,6 +261,86 @@ def test_the_dead_lines_own_action_and_the_outcome_can_differ(): assert "actually denied" in outcome +def _real_dead_rule_detail(dead_action: str, blocking_action: str) -> str: + """The genuine evidence string, from the genuine check -- not a literal. + + Same reasoning as _real_detail() below: a hand-typed literal cannot + catch a wording drift between access_control._check_dead_rules() and + this module's own pattern. `dead_action` and `blocking_action` are + each "PERMIT" or "DENY", independently, so the same helper builds both + the opposite-action and the same-action (redundant) real findings. + """ + from analysis.checks import access_control + + dead_line = f"{dead_action.lower()} tcp any host 10.10.10.5 eq 22" + blocking_line = f"{blocking_action.lower()} ip any any" + frame = pd.DataFrame([{ + "Sources": ["rtr-us5: acl_in"], + "Unreachable_Line": dead_line, + "Unreachable_Line_Action": dead_action, + "Blocking_Lines": [blocking_line], + "Reason": "BLOCKING_LINES", + }]) + + class _Question: + def __init__(self, frame): self._frame = frame + def answer(self): return self + def frame(self): return self._frame + + class _Q: + def filterLineReachability(self, **kwargs): return _Question(frame) + + class _FakeSession: + q = _Q() + + [finding] = access_control._check_dead_rules(_FakeSession(), iter([1])) + return finding["evidence"]["detail"] + + +# --- the redundant case: the two facts agree, so the wording must not say +# "different" (found in review of #401, before it merged) -------------------- + + +def test_a_redundant_dead_deny_states_agreement_not_difference(): + """CLAIMS: a DENY shadowed by an earlier DENY is redundant, not a real + exposure, and the computed fact must say so plainly rather than + claiming the outcome is "different" -- which would be false here and + is the #52 inversion #384's own fix exists to prevent. + + Through the REAL producer (access_control._check_dead_rules), not a + literal, per the review that asked for this.""" + detail = _real_dead_rule_detail(dead_action="DENY", blocking_action="DENY") + outcome = _compute_dead_rule_outcome(detail) + + assert outcome is not None + assert "own stated action is DENY" in outcome + assert "denied anyway" in outcome + assert "what actually happens is different" not in outcome, ( + f"the two lines agree -- nothing is different here: {outcome!r}" + ) + assert "not what happens" not in outcome, ( + f"the real outcome IS what the dead line's own action says here, " + f"so this phrase is false for the redundant case: {outcome!r}" + ) + + +def test_a_redundant_dead_permit_states_agreement_not_difference(): + """The mirror case -- a PERMIT shadowed by an earlier PERMIT.""" + detail = _real_dead_rule_detail(dead_action="PERMIT", blocking_action="PERMIT") + outcome = _compute_dead_rule_outcome(detail) + + assert outcome is not None + assert "own stated action is PERMIT" in outcome + assert "permitted anyway" in outcome + assert "what actually happens is different" not in outcome, ( + f"the two lines agree -- nothing is different here: {outcome!r}" + ) + assert "not what happens" not in outcome, ( + f"the real outcome IS what the dead line's own action says here, " + f"so this phrase is false for the redundant case: {outcome!r}" + ) + + # --- _compute_policy_outcome (#145) --------------------------------------------- # Regression coverage for the reasoning mistake found while independently # rating explanations for #90: the model attributed the wrong action to "the From 246db8efcde680e325a2ea876fae3fd2f2a17174 Mon Sep 17 00:00:00 2001 From: patelankeet2 Date: Wed, 4 Nov 2026 11:57:42 +1300 Subject: [PATCH 3/7] Re-capture and re-rate AC-002 under the fixed prompt (#385's guard) #385's per-finding prompt hash correctly flagged AC-002 as stale the moment main (with #385 and #399) merged onto this branch -- the prompt _build_prompt() builds for it moved when the previous commit fixed #384. Re-captured one live generation under the new prompt and rated it by hand (Ankeet): accuracy 5, usefulness 4, up from 2/2. The old rating's defect is gone -- the dead line's own PERMIT action is now named correctly, and the real blocked outcome is stated without inverting which rule is which. Updated docs/evaluation.md Part 4 with a dated note recording the re-capture rather than silently rewriting the AC-002 row, and updated the two tests that pin the published numbers (mean accuracy 4.25 -> 5.0, mean usefulness 3.25 -> 3.75; the breakdown's low-scorer demonstration now points at PC-005's usefulness, the new minimum, since AC-002 no longer has an outlier to show). Full suite: 2245 passed, 13 skipped, 0 failed. --- docs/evaluation.md | 51 ++++++++++++++----- .../fixtures/explanation_quality/holdout.json | 10 ++-- tests/test_explanation_quality.py | 34 +++++++++---- 3 files changed, 67 insertions(+), 28 deletions(-) diff --git a/docs/evaluation.md b/docs/evaluation.md index d1e7b19..699bcb5 100644 --- a/docs/evaluation.md +++ b/docs/evaluation.md @@ -774,9 +774,26 @@ functions already name both sides of their fact), and the same discipline #145 itself set: **not fixed here.** Fixing it now would mean rating text that no longer exists by the time anyone reads this table. +> **RE-CAPTURED, after #401 fixed #384 (reviewed by @ARSH871-bot before +> merge).** #401 extended `_compute_dead_rule_outcome()` to name the dead +> line's own action explicitly, which is exactly the fix this section asked +> for -- and review found a second problem while measuring it: the +> single-template wording also assumed a dead line's own action and the +> real outcome always disagree, which is false whenever the line is +> redundant rather than contradicted (a DENY shadowed by an earlier DENY, +> say). Both are fixed in #401, with a branch on whether the two sides +> agree. #385's own prompt-hash guard caught AC-002 as stale the moment +> #401 merged onto this held-out set, which is exactly what that guard +> exists to catch -- re-captured under the fixed prompt rather than editing +> the old rating in place, same discipline this section used the first +> time. The table and the repeatable score below are both updated; the +> `AC-002` row and this paragraph are left as the honest record of what was +> found and when, not rewritten to read as if it were always correct. + ### The held-out set and its ratings -One rater so far (Ankeet) - see +Two rounds so far (Ankeet, 10 October; AC-002 re-captured and re-rated by +Ankeet, 3 November, after #401) - see [`tests/fixtures/explanation_quality/holdout.json`](../tests/fixtures/explanation_quality/holdout.json) for the full captured findings, explanations and reasoning behind each score. Same 1-5 scale as Part 3 (accuracy: does the text claim only what the @@ -788,7 +805,7 @@ read `evidence.detail` what is wrong). | RT-050 | fallback | 5 | 4 | Deterministic, cannot be wrong by construction - **reported separately below, not part of the model score.** | | AC-001 | model | 5 | 4 | Correct, no inversion. | | PC-001 | model | 5 | 4 | Regenerated post-#145: inversion gone. | -| AC-002 | model | **2** | **2** | **Real defect, kept rather than re-rolled - see #384.** | +| AC-002 | model | 5 | 4 | **Re-captured under #401's fix (was 2/2, see #384 above).** Correctly names the dead line's own action as PERMIT and the real outcome as blocked, no inversion either way. | | PC-005 | model | 5 | 3 | Regenerated post-#145: inversion gone. First-sentence phrasing still asks the reader to resolve an apparent contradiction before it resolves itself - a readability risk, not a false claim this time. | @ARSH871-bot @shubhamkataria2005 @SamikaPerera - a second independent rating @@ -804,14 +821,20 @@ python -m tools.explanation_quality ``` 4 model-written findings, 4 ratings -accuracy: mean 4.25 min 2 -usefulness: mean 3.25 min 2 +accuracy: mean 5.00 min 5 +usefulness: mean 3.75 min 3 fallback-sourced (not scored against the threshold - cannot be wrong by construction): RT-050 accuracy 5.0 usefulness 4.0 (1 rating(s)) ``` +The minimum moved from AC-002's accuracy=2 to PC-005's usefulness=3 - a +readability nit already noted above, not a factual inversion. The +per-finding breakdown is still what surfaces it rather than the mean +hiding it; see `tools/explanation_quality.py`'s own docstring for why mean +alone was rejected as the published number in the first place. + **Published, and pinned in `tests/test_explanation_quality.py`** so CI catches it if the committed ratings ever drift from what this document claims - the same "generated artefact can go stale between the commit and @@ -853,15 +876,17 @@ mean accuracy 4.0, mean usefulness 3.0 - both below the measurement above, so the threshold is a real tripwire rather than a number chosen to always pass. -**Deliberately based on the mean, not the minimum.** `AC-002`'s accuracy=2 is -a correctness bug in one evidence shape, the same class of thing #52 and -#145 already were, and both of those were fixed for their one shape rather -than by turning the model off for every finding. Tying this threshold to the -minimum would mean one narrow, already-understood, 1-in-8 defect forces every -explanation in the product onto the deterministic fallback - a far bigger -change than the finding justifies. The mean is what actually answers the -question this threshold exists for: has quality collapsed broadly, not did -one shape have one bad day. +**Deliberately based on the mean, not the minimum.** `AC-002`'s accuracy=2 +(now fixed by #401, re-captured at 5/4 above) was a correctness bug in one +evidence shape, the same class of thing #52 and #145 already were, and all +three were fixed for their one shape rather than by turning the model off +for every finding. Tying this threshold to the minimum would have meant one +narrow, already-understood, 1-in-8 defect forcing every explanation in the +product onto the deterministic fallback - a far bigger change than the +finding justified. The mean is what actually answers the question this +threshold exists for: has quality collapsed broadly, not did one shape have +one bad day. The reasoning stands regardless of which specific finding is +currently lowest. **Why this is a documented number, not a runtime check.** There is no honest way to score one live generation's quality automatically without another diff --git a/tests/fixtures/explanation_quality/holdout.json b/tests/fixtures/explanation_quality/holdout.json index ac3392f..ead991f 100644 --- a/tests/fixtures/explanation_quality/holdout.json +++ b/tests/fixtures/explanation_quality/holdout.json @@ -105,15 +105,15 @@ }, "status": "found" }, - "captured_explanation": "The device's ACL rule, which is supposed to block traffic from a specific IP address, never actually takes effect because it is overridden by another rule that blocks all traffic with the same destination IP address. As a result, even though the unreachable line in the ACL rule would normally deny traffic matching its pattern, the blocking rule always wins and prevents any traffic from being allowed.", + "captured_explanation": "The ACL rule on the RTR-US5 device, which allows traffic to reach host 218.8.104.58 over UDP, is never actually applied because an earlier rule blocks all traffic with the same destination IP address. As a result, this specific rule has no effect, even though it explicitly states that it should permit traffic.", "captured_explanation_source": "model", - "captured_prompt_sha256": "6906691c3cc74d26649b8c1a2a4877658ec7585dd767dd41f56c07d2b2e9eaf5", + "captured_prompt_sha256": "8ea2eeb9812dacac19d1f6079f6ac5e7d79c65c82506b8657fb238d77b76afc3", "ratings": [ { "rater": "Ankeet", - "accuracy": 2, - "usefulness": 2, - "notes": "REAL DEFECT, kept rather than re-rolled -- see issue #384 and docs/evaluation.md Part 4. evidence.detail says the unreachable line's own action is PERMIT; this text calls it a rule 'which is supposed to block traffic' and says it 'would normally deny traffic matching its pattern' -- backwards about the dead line's own action, even though the final-outcome claim (traffic ends up denied) is correct. A reader walks away with the wrong idea of what kind of rule became unreachable. Checked for reproducibility: 0 of 6 immediate re-generations of this exact finding repeated it (see #384), so this is the honest, unmodified first capture, not the typical case -- but it is real, and a measurement that quietly swapped it for a cleaner sample would defeat the point of measuring at all." + "accuracy": 5, + "usefulness": 4, + "notes": "RE-CAPTURED after #384/#401 rewrote _compute_dead_rule_outcome() -- the prompt-hash guard (#385) correctly flagged this entry as stale the moment #401 merged onto this set. Old capture named the dead line a rule \"which is supposed to block traffic\" when its real action is PERMIT (the #384 defect, accuracy 2/2, see the superseded note in git history). This capture correctly says the line \"allows traffic... should permit traffic\" -- own action named right, outcome (no effect, blocked by the earlier deny) also right, no inversion. Checked for reproducibility before rating: 16 live generations across this exact finding and the mirror redundant-dead-rule case (see #401), 0 of 16 inverted the dead line's own action and 0 of 16 copied ai/Modelfile's Example 1, which #401's own review measured happening 9/60 under the wording this replaces." } ] }, diff --git a/tests/test_explanation_quality.py b/tests/test_explanation_quality.py index 3383543..a1bef8d 100644 --- a/tests/test_explanation_quality.py +++ b/tests/test_explanation_quality.py @@ -162,14 +162,19 @@ def test_the_published_score_matches_the_committed_ratings(): tools/explanation_quality.py's score_holdout() docstring). A change here means either the holdout set or its ratings changed -- re-measure deliberately (docs/evaluation.md Part 4), don't just update this - assertion to make it pass.""" + assertion to make it pass. + + Re-measured after #401 re-captured AC-002 under its fixed prompt -- + #385's guard flagged it stale the moment #401 merged onto this set, + since #384's defect is exactly what #401 fixes. AC-002 moved from 2/2 + to 5/4; see docs/evaluation.md Part 4 for the dated note.""" score = score_holdout(load_holdout()) assert score["n_findings"] == 4 assert score["n_ratings"] == 4 - assert score["mean_accuracy"] == pytest.approx(4.25) - assert score["min_accuracy"] == 2 - assert score["mean_usefulness"] == pytest.approx(3.25) - assert score["min_usefulness"] == 2 + assert score["mean_accuracy"] == pytest.approx(5.0) + assert score["min_accuracy"] == 5 + assert score["mean_usefulness"] == pytest.approx(3.75) + assert score["min_usefulness"] == 3 assert {row["id"] for row in score["per_finding"]} == { "AC-001", "PC-001", "AC-002", "PC-005", } @@ -187,13 +192,22 @@ def test_rt_050_is_reported_as_fallback_not_scored_against_the_threshold(): def test_the_low_scoring_finding_is_identifiable_in_the_breakdown(): - """AC-002's accuracy=2 is a real, filed defect (#384), not noise. The - per-finding breakdown is what makes it findable rather than letting the - 4.4 mean hide it -- see tools/explanation_quality.py's own module - docstring for why mean alone was rejected as the published number.""" + """AC-002's accuracy=2 (#384) was the breakdown's original reason to + exist -- a defect the blended mean would have hidden. #401 fixed that + defect and re-captured AC-002 at 5/4 (see the test above), so every + finding now ties at accuracy=5 and there is no accuracy outlier left + to find. The breakdown's job is unchanged, only which finding it + surfaces: PC-005's usefulness=3 is now the lowest figure in the set, + a readability nit already noted in docs/evaluation.md's own table + (its first sentence asks the reader to resolve an apparent + contradiction before it resolves itself), not a filed defect -- the + point of this test is that the breakdown still finds it rather than + averaging it away, whatever its current cause.""" score = score_holdout(load_holdout()) by_id = {row["id"]: row for row in score["per_finding"]} - assert by_id["AC-002"]["mean_accuracy"] == 2.0 + assert by_id["AC-002"]["mean_accuracy"] == 5.0 + assert by_id["PC-005"]["mean_usefulness"] == 3.0 + assert min(row["mean_usefulness"] for row in score["per_finding"]) == 3.0 def test_current_mean_quality_meets_the_documented_threshold(): From ecd7e055bcf816d25404374940424aebf1645bca Mon Sep 17 00:00:00 2001 From: patelankeet2 Date: Sun, 8 Nov 2026 15:46:42 +1300 Subject: [PATCH 4/7] Pin the redundant-case agreement sentence, not just the absent phrases Arsh's 5 November review: the existing asserts only rejected the two old false phrases, so a rewrite that drops the agreement sentence entirely, or inverts its verb, still passed. Added the two asserts he gave, mutation tested against both of his exact adversarial wordings (sentence removed, verb inverted) -- both now fail for the right reason. Also fixed a stale reference found in the same review: the redundant-case comment said this was #444's shape; #444 is R-1's word match, this is the #52 inversion. The docstring already had it right, only this comment did not. --- ai/explain.py | 4 ++-- tests/test_explain_safety_net.py | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/ai/explain.py b/ai/explain.py index 9716462..c813889 100644 --- a/ai/explain.py +++ b/ai/explain.py @@ -245,8 +245,8 @@ def _compute_dead_rule_outcome(detail: str) -> Optional[str]: if own == outcome: # Redundant, not contradicted -- the two facts happen to agree. Never - # say "different" or "not what happens" here; that is #444's exact - # shape, reintroduced. + # say "different" or "not what happens" here; that is the #52 + # inversion, reintroduced. verb = "permits" if outcome == "permitted" else "denies" return ( f"the unreachable line's own stated action is {dead_action}, " diff --git a/tests/test_explain_safety_net.py b/tests/test_explain_safety_net.py index 4e464f6..771e93b 100644 --- a/tests/test_explain_safety_net.py +++ b/tests/test_explain_safety_net.py @@ -323,6 +323,12 @@ def test_a_redundant_dead_deny_states_agreement_not_difference(): f"the real outcome IS what the dead line's own action says here, " f"so this phrase is false for the redundant case: {outcome!r}" ) + # Pins the actual agreement sentence, not just the absence of the two + # false phrases above -- found in review (5 November): without these, + # a rewrite that drops the agreement sentence entirely, or inverts its + # verb, still passes every assertion that was here before. + assert "also denies it" in outcome + assert "the same as the unreachable line says" in outcome def test_a_redundant_dead_permit_states_agreement_not_difference(): @@ -340,6 +346,8 @@ def test_a_redundant_dead_permit_states_agreement_not_difference(): f"the real outcome IS what the dead line's own action says here, " f"so this phrase is false for the redundant case: {outcome!r}" ) + assert "also permits it" in outcome + assert "the same as the unreachable line says" in outcome # --- _compute_policy_outcome (#145) --------------------------------------------- From 08345b356121392db1e31730497f1543ced71939 Mon Sep 17 00:00:00 2001 From: patelankeet2 Date: Sun, 8 Nov 2026 16:14:09 +1300 Subject: [PATCH 5/7] Name the real blocking line in the opposite-action fact, not a vague claim Arsh's 5 November review: the wording said an earlier line blocks 'the same traffic', which the real AC-002 finding's blocking line (deny ip 10.10.10.0 0.0.0.255 any) does not actually support -- it is one source subnet to ANY destination, not a destination match, and 6 of 29 live generations under the old wording said so anyway ('same destination IP address'). Now names the real blocking line text, singular and plural grammar both pinned by new tests. Found and fixed a real subject-verb mismatch in the plural branch while writing it (Arsh's own suggested wording for that branch was not live tested by either of us) -- 'lines ... are ... decides' corrected to '... decide'. Re-captured AC-002 in the held-out set under the new wording and rated it myself (30 live generations: 1/30 still said same destination, 0/30 called the permit line a block, 1/30 copied Example 1 -- normal model variance, not hidden by the new wording). docs/evaluation.md and the holdout fixture both updated, with an explicit request for Shubham and Arsh to rate the new capture since their earlier 4s rated the old one. Also fixed the second stale reference from the same review: tools/explanation_quality.py called the 4.25/3.25 threshold basis 'the current measurement' when a later re-run already reads higher -- reworded to say when it was taken and to point at docs/evaluation.md for the real current number instead of repeating one here. --- ai/explain.py | 15 +++++++++-- docs/evaluation.md | 8 +++--- .../fixtures/explanation_quality/holdout.json | 8 +++--- tests/test_explain_safety_net.py | 26 +++++++++++++++++++ tools/explanation_quality.py | 12 ++++++--- 5 files changed, 56 insertions(+), 13 deletions(-) diff --git a/ai/explain.py b/ai/explain.py index c813889..6be19f2 100644 --- a/ai/explain.py +++ b/ai/explain.py @@ -256,11 +256,22 @@ def _compute_dead_rule_outcome(detail: str) -> Optional[str]: f"decides is different." ) + # Name the actual blocking line(s) rather than a vague "same traffic" + # claim (found in review, 5 November). Measured on AC-002: the vague + # wording was read as "blocks all traffic with the same destination IP + # address" when the real blocker (deny ip 10.10.10.0 0.0.0.255 any) is + # one source subnet to ANY destination -- a real misdescription, not a + # one-off. Naming the line leaves nothing for the model to invent. + named = " ".join(match.group("blocking").split()) # Batfish pads its spaces + lines, verb, decide_verb = ( + ("earlier lines", "are", "decide") if len(blocking_lines) > 1 + else ("an earlier line", "is", "decides") + ) return ( f"the unreachable line's own stated action is {dead_action}, but it " f"never takes effect: traffic matching it is actually {outcome}, " - f"because an earlier line that matches the same traffic is " - f"evaluated first and decides instead." + f"because {lines} ({named}) {verb} evaluated first and {decide_verb} " + f"instead." ) diff --git a/docs/evaluation.md b/docs/evaluation.md index 699bcb5..ddf8b30 100644 --- a/docs/evaluation.md +++ b/docs/evaluation.md @@ -792,8 +792,10 @@ that no longer exists by the time anyone reads this table. ### The held-out set and its ratings -Two rounds so far (Ankeet, 10 October; AC-002 re-captured and re-rated by -Ankeet, 3 November, after #401) - see +Three rounds so far (Ankeet, 10 October; AC-002 re-captured and re-rated by +Ankeet, 3 November, after #401; AC-002 re-captured and re-rated by Ankeet +again, 8 November, after review found the 3 November capture's own wording +was itself a misdescription) - see [`tests/fixtures/explanation_quality/holdout.json`](../tests/fixtures/explanation_quality/holdout.json) for the full captured findings, explanations and reasoning behind each score. Same 1-5 scale as Part 3 (accuracy: does the text claim only what the @@ -805,7 +807,7 @@ read `evidence.detail` what is wrong). | RT-050 | fallback | 5 | 4 | Deterministic, cannot be wrong by construction - **reported separately below, not part of the model score.** | | AC-001 | model | 5 | 4 | Correct, no inversion. | | PC-001 | model | 5 | 4 | Regenerated post-#145: inversion gone. | -| AC-002 | model | 5 | 4 | **Re-captured under #401's fix (was 2/2, see #384 above).** Correctly names the dead line's own action as PERMIT and the real outcome as blocked, no inversion either way. | +| AC-002 | model | 5 | 4 | **Re-captured 8 November, after the computed fact was changed to name the real blocking line instead of describing it loosely (review, 5 November).** The 3 November capture said the blocker "blocks all traffic with the same destination IP address" - false, the real blocker (`deny ip 10.10.10.0 0.0.0.255 any`) is one source subnet to ANY destination, which is what cost that capture its accuracy=4 from two independent raters. This capture says "blocks all traffic through that same interface" instead, which is true and does not repeat the destination claim. 30 live generations measured on this exact finding: 1/30 still said "same destination", 0/30 called the permit line a block, 1/30 copied Modelfile Example 1 - normal model variance, not something this wording hides. **@ARSH871-bot @shubhamkataria2005, this is a fresh capture - your earlier 4s rated the 3 November text, not this one. Please rate again.** | | PC-005 | model | 5 | 3 | Regenerated post-#145: inversion gone. First-sentence phrasing still asks the reader to resolve an apparent contradiction before it resolves itself - a readability risk, not a false claim this time. | @ARSH871-bot @shubhamkataria2005 @SamikaPerera - a second independent rating diff --git a/tests/fixtures/explanation_quality/holdout.json b/tests/fixtures/explanation_quality/holdout.json index ead991f..61374de 100644 --- a/tests/fixtures/explanation_quality/holdout.json +++ b/tests/fixtures/explanation_quality/holdout.json @@ -105,15 +105,15 @@ }, "status": "found" }, - "captured_explanation": "The ACL rule on the RTR-US5 device, which allows traffic to reach host 218.8.104.58 over UDP, is never actually applied because an earlier rule blocks all traffic with the same destination IP address. As a result, this specific rule has no effect, even though it explicitly states that it should permit traffic.", + "captured_explanation": "The ACL rule on the RTR-US5 device, which allows unencrypted UDP traffic to reach host 218.8.104.58 from the internal network (10.10.10.0/24), is never actually enforced because an earlier deny rule takes precedence and blocks all traffic through that same interface. As a result, this specific permit action is effectively ignored by the device's access control system.", "captured_explanation_source": "model", - "captured_prompt_sha256": "8ea2eeb9812dacac19d1f6079f6ac5e7d79c65c82506b8657fb238d77b76afc3", + "captured_prompt_sha256": "f037179380f1eb6904c3686eab657ca1045ce1559fd83008d9950a7014454643", "ratings": [ { "rater": "Ankeet", "accuracy": 5, "usefulness": 4, - "notes": "RE-CAPTURED after #384/#401 rewrote _compute_dead_rule_outcome() -- the prompt-hash guard (#385) correctly flagged this entry as stale the moment #401 merged onto this set. Old capture named the dead line a rule \"which is supposed to block traffic\" when its real action is PERMIT (the #384 defect, accuracy 2/2, see the superseded note in git history). This capture correctly says the line \"allows traffic... should permit traffic\" -- own action named right, outcome (no effect, blocked by the earlier deny) also right, no inversion. Checked for reproducibility before rating: 16 live generations across this exact finding and the mirror redundant-dead-rule case (see #401), 0 of 16 inverted the dead line's own action and 0 of 16 copied ai/Modelfile's Example 1, which #401's own review measured happening 9/60 under the wording this replaces." + "notes": "RE-CAPTURED again after review (5 November) found the previous capture's 'same destination IP address' claim was false -- the real blocker (deny ip 10.10.10.0 0.0.0.255 any) is one source subnet to ANY destination, not a destination match. The computed fact now names the real blocking line text instead of describing it loosely, so there is less room for the model to invent a relationship between the two rules. This capture says the deny rule 'blocks all traffic through that same interface', which is true (the ACL applies to the whole interface) and does not repeat the destination-IP claim. 30 live generations on this exact finding, measured: 1/30 still said 'same destination', 0/30 called the permit line a block, 1/30 copied Modelfile Example 1 near-verbatim -- all three are the model's own variance, not something this capture's wording hides. Posted for Shubham and Arsh to add their own rating of THIS capture, since their earlier 4s rated the previous one." } ] }, @@ -145,4 +145,4 @@ ] } ] -} +} \ No newline at end of file diff --git a/tests/test_explain_safety_net.py b/tests/test_explain_safety_net.py index 771e93b..d206a1f 100644 --- a/tests/test_explain_safety_net.py +++ b/tests/test_explain_safety_net.py @@ -198,6 +198,24 @@ def test_returns_none_when_blocking_lines_disagree(): assert _compute_dead_rule_outcome(detail) is None +def test_two_agreeing_blocking_lines_are_both_named_with_plural_grammar(): + """Two blocking lines, both DENY -- not ambiguous (they agree), so this + states the outcome and names both lines. Added in review (5 November): + not live-tested by anyone before this, and the first version written + here had a real subject-verb mismatch ("lines ... are ... decides"), + caught before it shipped.""" + detail = ( + "Unreachable line: permit udp 10.10.10.0 0.0.0.255 host 218.8.104.58 " + "eq domain (action PERMIT). Blocked by: deny ip 10.10.10.0 " + "0.0.0.255 any, deny tcp 10.10.10.0 0.0.0.255 any. Reason: BLOCKING_LINES" + ) + outcome = _compute_dead_rule_outcome(detail) + assert outcome is not None + assert "earlier lines (" in outcome + assert "deny ip 10.10.10.0 0.0.0.255 any, deny tcp 10.10.10.0 0.0.0.255 any" in outcome + assert "are evaluated first and decide instead" in outcome + + def test_returns_none_for_unrecognised_blocking_line_syntax(): detail = ( "Unreachable line: permit tcp any any eq 80 (action PERMIT). " @@ -260,6 +278,14 @@ def test_the_dead_lines_own_action_and_the_outcome_can_differ(): assert outcome is not None assert "own stated action is PERMIT" in outcome assert "actually denied" in outcome + # Names the real blocking line rather than a vague "same traffic" claim + # (found in review, 5 November, on this exact AC-002 finding): the old + # wording read as "blocks all traffic with the same destination IP + # address", which is false -- this blocker is one source subnet to ANY + # destination. + assert "deny ip 10.10.10.0 0.0.0.255 any" in outcome + assert "same traffic" not in outcome + assert "same destination" not in outcome def _real_dead_rule_detail(dead_action: str, blocking_action: str) -> str: diff --git a/tools/explanation_quality.py b/tools/explanation_quality.py index f4defdd..ecb523a 100644 --- a/tools/explanation_quality.py +++ b/tools/explanation_quality.py @@ -56,11 +56,15 @@ ) #: Acceptance criterion 3 of #346: "a documented threshold below which the -#: deterministic fallback is preferred". Grounded in the first real +#: deterministic fallback is preferred". Grounded in the FIRST real #: measurement (see docs/evaluation.md Part 4), model-written entries only: -#: mean_accuracy 4.25, mean_usefulness 3.25 -- so these two numbers sit -#: comfortably below the current measurement rather than just above it, -#: which would make the threshold meaningless. +#: mean_accuracy 4.25, mean_usefulness 3.25, taken when #346 landed. That +#: was the measurement at the time, not a claim about the measurement now +#: (found stale in review, 5 November -- a later re-run already read +#: 5.00 / 3.75). These two threshold numbers sit comfortably below that +#: first measurement rather than just above it, which would make the +#: threshold meaningless. Check docs/evaluation.md Part 4 for the actual +#: current number rather than trusting a figure quoted here. #: #: DELIBERATELY ON THE MEAN, NOT THE MINIMUM #: A single low-scoring finding (AC-002, accuracy=2, #384) is treated From e22d970e7b3a4f50a765cd55f668c2000a660e2d Mon Sep 17 00:00:00 2001 From: patelankeet2 Date: Tue, 10 Nov 2026 12:54:20 +1300 Subject: [PATCH 6/7] Describe the blocking line in plain English, not raw ACL syntax (#401) Shubham's review (9 November): naming the blocking line in raw ACL syntax (the previous fix, 5 November) measurably traded that misdescription for a different one. 30 live generations on AC-002: calls the dead PERMIT line a block (the #384 bug) 12/30 -> 0/30 calls the source a specific host (new) 0/30 -> 16/30 The dead line's source is the subnet 10.10.10.0 0.0.0.255 (a /24); the raw blocking-line text sitting next to the dead line's own raw text in evidence.detail (which contains the literal word 'host' for its destination) appears to make the model misattribute which address plays which role. Adds _describe_acl_line(), a narrow Cisco-ACL-line-to-plain-English translator reusing this project's own established ipaddress-based wildcard/netmask conversion (same technique ai/propose.py's _cisco_endpoint() already uses in the opposite direction, same contiguous-mask validation analysis/draft_policy.py already applies). Recognises exactly the three endpoint forms this project's own checks and converters ever produce (any, host A.B.C.D, A.B.C.D wildcard) and refuses -- returns None -- on anything else: a source-port clause, an unrecognised shape, a non-contiguous wildcard mask. The contradicted branch of _compute_dead_rule_outcome() uses it when every named blocking line parses; if any one does not, falls back to the current raw-syntax naming for all of them, so this never blocks a finding from being explained. Tests: six new unit tests for _describe_acl_line() directly (both real fixtures, a host destination, and three refusal cases); updated the four existing tests that pinned the old wording. Mutation-tested: forcing the helper to always return None breaks all 9 dependent tests for the right reason (back to raw-syntax wording); the three refusal-path tests are unaffected either way, confirming they test real refusal behaviour, not a side effect of the mutation. Full suite: 2271 passed, 13 skipped, 0 failed. Live-tested against the real model: reproduced Shubham's exact 30-run measurement on the same AC-002 finding, read every output by hand rather than trust a keyword heuristic (an early pattern-matching attempt wildly overcounted, since 'a specific host' correctly describes the real destination in most outputs). Real result: calls the dead PERMIT line a block: 0/30 calls the source a specific host: 1/30 clear, 1/30 ambiguous (down from 16/30) copies Modelfile Example 1 verbatim: 0/30 Two things found while reading, reported but out of scope for this fix: 2/30 invert the flow direction (source and destination swapped), 2/30 echo Example 1's 'unencrypted' vocabulary without copying the phrase exactly. Re-captured and re-rated AC-002 in the held-out evaluation set a third time (its prompt hash moved again), with a request for Shubham and Arsh to rate the new capture. --- ai/explain.py | 135 +++++++++++++++++- docs/evaluation.md | 10 +- .../fixtures/explanation_quality/holdout.json | 8 +- tests/test_explain_safety_net.py | 79 +++++++--- tests/test_explanation_quality.py | 11 +- 5 files changed, 208 insertions(+), 35 deletions(-) diff --git a/ai/explain.py b/ai/explain.py index 6be19f2..81c8f34 100644 --- a/ai/explain.py +++ b/ai/explain.py @@ -146,6 +146,112 @@ ) +def _parse_cisco_endpoint(tokens, i): + """Consume one Cisco ACL endpoint (source or destination) starting at + tokens[i]: "any", "host A.B.C.D", or "A.B.C.D W.X.Y.Z" (wildcard) -- + the exact three forms ai/propose.py's own _cisco_endpoint() ever + emits in the other direction. Returns (kind, value, next_index), or + None for anything else -- refuse rather than guess at an + unfamiliar shape (#401 review, @shubhamkataria2005, 9 November). + """ + if i >= len(tokens): + return None + if tokens[i] == "any": + return "any", None, i + 1 + if tokens[i] == "host": + if i + 1 >= len(tokens): + return None + return "host", tokens[i + 1], i + 2 + if i + 1 >= len(tokens): + return None + addr, wildcard = tokens[i], tokens[i + 1] + octets = wildcard.split(".") + if len(octets) != 4: + return None + try: + netmask = ".".join(str(255 - int(b)) for b in octets) + network = ipaddress.ip_network(f"{addr}/{netmask}", strict=False) + except ValueError: + return None # not a valid address, or a non-contiguous wildcard + # mask (the inverted bytes do not form a legal netmask) -- same + # reason analysis/draft_policy.py refuses one of these as + # "cannot be written as one" prefix. + return "network", f"{network.network_address}/{network.prefixlen}", i + 2 + + +def _format_endpoint(kind, value, role): + if kind == "any": + return "anywhere" if role == "source" else "any destination" + if kind == "host": + return f"host {value}" + return value # bare CIDR, e.g. "10.10.10.0/24" + + +def _describe_acl_line(line): + """A plain-English description of one Cisco ACL line's own criteria, + not a restatement of its raw syntax. + + WHY THIS EXISTS (#401 review, @shubhamkataria2005, 9 November) + Naming the blocking line in raw ACL text (the previous fix for a + different misdescription, found in review 5 November) measurably + traded one misdescription for another: 30 live generations on + AC-002 went from 12/30 calling the dead PERMIT line a block (the + original #384 bug, now 0/30) to 16/30 calling the dead line's own + /24 source "a specific host" -- a confusion that did not happen + on main or on the commit before the raw-syntax fix. The raw + blocking-line text sits next to the dead line's own raw text + (also shown to the model, in evidence.detail), and the two + addresses of different specificity next to each other appear to + get their roles crossed. Removing the raw ACL syntax from the + computed fact removes that collision. + + Recognises exactly "permit|deny PROTOCOL SRC [SRC] DST [DST] [eq + PORT]" -- the only shape this project's own checks and converters + ever produce. Returns None for anything else (an unrecognised + protocol-keyword shape, a source-port clause, trailing tokens, a + non-contiguous wildcard mask) rather than guess at it; the caller + falls back to the current raw-syntax naming in that case, so this + never blocks a finding from being explained, only changes how + confidently the blocking line's own criteria can be phrased. + """ + tokens = line.split() + if len(tokens) < 4: + return None + action = tokens[0].lower() + if action not in ("permit", "deny"): + return None + verb = "permits" if action == "permit" else "denies" + protocol = tokens[1].lower() + + src = _parse_cisco_endpoint(tokens, 2) + if src is None: + return None + src_kind, src_value, i = src + dst = _parse_cisco_endpoint(tokens, i) + if dst is None: + return None + dst_kind, dst_value, i = dst + + port = None + if i < len(tokens): + if tokens[i].lower() == "eq" and i + 1 < len(tokens): + port = tokens[i + 1] + i += 2 + else: + return None # a shape this project does not recognise + if i != len(tokens): + return None # trailing tokens left over -- refuse rather than guess + + proto_phrase = "all traffic" if protocol == "ip" else f"{protocol.upper()} traffic" + sentence = ( + f"{verb} {proto_phrase} from {_format_endpoint(src_kind, src_value, 'source')} " + f"to {_format_endpoint(dst_kind, dst_value, 'destination')}" + ) + if port is not None: + sentence += f" on port {port}" + return sentence + + def _compute_dead_rule_outcome(detail: str) -> Optional[str]: """For a dead-ACL-rule finding, compute what actually happens to traffic matching the dead line, AND what the dead line's own action @@ -256,12 +362,29 @@ def _compute_dead_rule_outcome(detail: str) -> Optional[str]: f"decides is different." ) - # Name the actual blocking line(s) rather than a vague "same traffic" - # claim (found in review, 5 November). Measured on AC-002: the vague - # wording was read as "blocks all traffic with the same destination IP - # address" when the real blocker (deny ip 10.10.10.0 0.0.0.255 any) is - # one source subnet to ANY destination -- a real misdescription, not a - # one-off. Naming the line leaves nothing for the model to invent. + # Describe the actual blocking line(s) in plain English rather than a + # vague "same traffic" claim (found in review, 5 November) OR raw ACL + # syntax (found in review, 9 November -- raw syntax measurably traded + # that misdescription for a different one, confusing the model about + # which address was "a specific host"; see _describe_acl_line()'s own + # docstring). If any blocking line does not parse, fall back to + # naming them in raw syntax rather than withholding the fact. + descriptions = [_describe_acl_line(line) for line in blocking_lines] + if all(d is not None for d in descriptions): + if len(descriptions) == 1: + return ( + f"the unreachable line's own stated action is {dead_action}, " + f"but it never takes effect: an earlier line, evaluated " + f"first, {descriptions[0]} instead." + ) + first, rest = descriptions[0], descriptions[1:] + cases = f"one {first}" + "".join(f", and another {d}" for d in rest) + return ( + f"the unreachable line's own stated action is {dead_action}, " + f"but it never takes effect: earlier lines, evaluated first, " + f"decide instead -- {cases}." + ) + named = " ".join(match.group("blocking").split()) # Batfish pads its spaces lines, verb, decide_verb = ( ("earlier lines", "are", "decide") if len(blocking_lines) > 1 diff --git a/docs/evaluation.md b/docs/evaluation.md index ddf8b30..dcd3bfa 100644 --- a/docs/evaluation.md +++ b/docs/evaluation.md @@ -792,10 +792,12 @@ that no longer exists by the time anyone reads this table. ### The held-out set and its ratings -Three rounds so far (Ankeet, 10 October; AC-002 re-captured and re-rated by +Four rounds so far (Ankeet, 10 October; AC-002 re-captured and re-rated by Ankeet, 3 November, after #401; AC-002 re-captured and re-rated by Ankeet again, 8 November, after review found the 3 November capture's own wording -was itself a misdescription) - see +was itself a misdescription; AC-002 re-captured and re-rated a third time, +9 November, after review found the 8 November wording traded that +misdescription for a different one) - see [`tests/fixtures/explanation_quality/holdout.json`](../tests/fixtures/explanation_quality/holdout.json) for the full captured findings, explanations and reasoning behind each score. Same 1-5 scale as Part 3 (accuracy: does the text claim only what the @@ -807,7 +809,7 @@ read `evidence.detail` what is wrong). | RT-050 | fallback | 5 | 4 | Deterministic, cannot be wrong by construction - **reported separately below, not part of the model score.** | | AC-001 | model | 5 | 4 | Correct, no inversion. | | PC-001 | model | 5 | 4 | Regenerated post-#145: inversion gone. | -| AC-002 | model | 5 | 4 | **Re-captured 8 November, after the computed fact was changed to name the real blocking line instead of describing it loosely (review, 5 November).** The 3 November capture said the blocker "blocks all traffic with the same destination IP address" - false, the real blocker (`deny ip 10.10.10.0 0.0.0.255 any`) is one source subnet to ANY destination, which is what cost that capture its accuracy=4 from two independent raters. This capture says "blocks all traffic through that same interface" instead, which is true and does not repeat the destination claim. 30 live generations measured on this exact finding: 1/30 still said "same destination", 0/30 called the permit line a block, 1/30 copied Modelfile Example 1 - normal model variance, not something this wording hides. **@ARSH871-bot @shubhamkataria2005, this is a fresh capture - your earlier 4s rated the 3 November text, not this one. Please rate again.** | +| AC-002 | model | 5 | 5 | **Re-captured a third time, 9 November, after review found that naming the blocking line in raw ACL syntax (the 8 November fix) traded one misdescription for another.** 30 live generations on the 8 November wording: 0/30 called the permit line a block, but 16/30 called the dead line's own /24 source "a specific host" - a confusion neither the pre-fix wording nor `main` produced, caused by two raw ACL snippets sitting next to each other in the prompt. Fixed by describing the blocking line in plain English instead of raw syntax, refusing rather than guessing on any shape it does not recognise. Re-measured, same finding, 30 generations: 0/30 call the permit line a block, 1/30 clearly calls the source a specific host (down from 16/30), 1/30 ambiguous, 0/30 copy Modelfile Example 1. This capture states both source and destination explicitly and correctly. Two things found along the way, reported but out of scope here: 2/30 invert the flow direction, 2/30 echo Example 1's "unencrypted" vocabulary without copying the phrase exactly. **@ARSH871-bot @shubhamkataria2005, fresh capture against a changed fact - your earlier ratings were of the 8 November text. Please rate again.** | | PC-005 | model | 5 | 3 | Regenerated post-#145: inversion gone. First-sentence phrasing still asks the reader to resolve an apparent contradiction before it resolves itself - a readability risk, not a false claim this time. | @ARSH871-bot @shubhamkataria2005 @SamikaPerera - a second independent rating @@ -824,7 +826,7 @@ python -m tools.explanation_quality ``` 4 model-written findings, 4 ratings accuracy: mean 5.00 min 5 -usefulness: mean 3.75 min 3 +usefulness: mean 4.00 min 3 fallback-sourced (not scored against the threshold - cannot be wrong by construction): diff --git a/tests/fixtures/explanation_quality/holdout.json b/tests/fixtures/explanation_quality/holdout.json index 61374de..d3a3bcb 100644 --- a/tests/fixtures/explanation_quality/holdout.json +++ b/tests/fixtures/explanation_quality/holdout.json @@ -105,15 +105,15 @@ }, "status": "found" }, - "captured_explanation": "The ACL rule on the RTR-US5 device, which allows unencrypted UDP traffic to reach host 218.8.104.58 from the internal network (10.10.10.0/24), is never actually enforced because an earlier deny rule takes precedence and blocks all traffic through that same interface. As a result, this specific permit action is effectively ignored by the device's access control system.", + "captured_explanation": "The device's access control rule for traffic from the internal network (10.10.10.0/24) to a specific host (218.8.104.58) is never enforced, despite being set to allow it. The reason for this is that an earlier rule blocks all traffic from the same source IP address to any destination. As a result, even though the rule itself permits the desired traffic, it is effectively overridden by the blocking rule, and no traffic reaches its intended destination.", "captured_explanation_source": "model", - "captured_prompt_sha256": "f037179380f1eb6904c3686eab657ca1045ce1559fd83008d9950a7014454643", + "captured_prompt_sha256": "7936d7601f64e4e5846eccea3ed46a8a7e2deebb3ff69e26ea76395052529616", "ratings": [ { "rater": "Ankeet", "accuracy": 5, - "usefulness": 4, - "notes": "RE-CAPTURED again after review (5 November) found the previous capture's 'same destination IP address' claim was false -- the real blocker (deny ip 10.10.10.0 0.0.0.255 any) is one source subnet to ANY destination, not a destination match. The computed fact now names the real blocking line text instead of describing it loosely, so there is less room for the model to invent a relationship between the two rules. This capture says the deny rule 'blocks all traffic through that same interface', which is true (the ACL applies to the whole interface) and does not repeat the destination-IP claim. 30 live generations on this exact finding, measured: 1/30 still said 'same destination', 0/30 called the permit line a block, 1/30 copied Modelfile Example 1 near-verbatim -- all three are the model's own variance, not something this capture's wording hides. Posted for Shubham and Arsh to add their own rating of THIS capture, since their earlier 4s rated the previous one." + "usefulness": 5, + "notes": "RE-CAPTURED a fourth time (9 November), after review found naming the blocking line in raw ACL syntax (the 5 November fix) measurably traded that misdescription for a different one: 30 live generations went from 0/30 calling the dead PERMIT line a block to 16/30 calling the /24 source a specific host. Fixed by describing the blocking line in plain English instead of raw syntax (_describe_acl_line()), refusing rather than guessing on any shape it does not recognise. Re-measured, same finding, 30 generations: 0/30 call the PERMIT line a block, 1/30 clearly calls the source a specific host (down from 16/30), 1/30 ambiguous, 0/30 copy Modelfile Example 1 verbatim. This capture states both source and destination explicitly and correctly, names the real cause (an earlier deny rule), and invents nothing beyond the evidence. Two things found along the way, reported but out of scope for this fix: 2/30 invert the flow direction (says traffic runs from the destination to the source), 2/30 echo Example 1's \"unencrypted\" vocabulary without copying the phrase exactly -- filed as follow-up observations, not fixed here. @ARSH871-bot @shubhamkataria2005, fresh capture against a changed fact -- please rate again." } ] }, diff --git a/tests/test_explain_safety_net.py b/tests/test_explain_safety_net.py index d206a1f..ad80e3f 100644 --- a/tests/test_explain_safety_net.py +++ b/tests/test_explain_safety_net.py @@ -30,6 +30,7 @@ _compute_expected_actual_outcome, _compute_policy_outcome, _compute_routing_outcome, + _describe_acl_line, _evidence_detail, _fallback_error_explanation, _fallback_plain_restatement, @@ -164,8 +165,9 @@ def test_dead_deny_shadowed_by_permit_means_traffic_is_permitted(): ) outcome = _compute_dead_rule_outcome(detail) assert outcome is not None - assert "permitted" in outcome - assert "denied" not in outcome + assert "own stated action is DENY" in outcome + assert "permits all traffic from anywhere to 2.128.0.0/16" in outcome + assert "denies" not in outcome def test_dead_permit_shadowed_by_deny_means_traffic_is_denied(): @@ -177,8 +179,9 @@ def test_dead_permit_shadowed_by_deny_means_traffic_is_denied(): ) outcome = _compute_dead_rule_outcome(detail) assert outcome is not None - assert "denied" in outcome - assert "permitted" not in outcome + assert "own stated action is PERMIT" in outcome + assert "denies all traffic from anywhere to any destination" in outcome + assert "permits" not in outcome def test_returns_none_for_a_finding_that_is_not_a_dead_rule(): @@ -211,9 +214,9 @@ def test_two_agreeing_blocking_lines_are_both_named_with_plural_grammar(): ) outcome = _compute_dead_rule_outcome(detail) assert outcome is not None - assert "earlier lines (" in outcome - assert "deny ip 10.10.10.0 0.0.0.255 any, deny tcp 10.10.10.0 0.0.0.255 any" in outcome - assert "are evaluated first and decide instead" in outcome + assert "earlier lines, evaluated first, decide instead" in outcome + assert "one denies all traffic from 10.10.10.0/24 to any destination" in outcome + assert "and another denies TCP traffic from 10.10.10.0/24 to any destination" in outcome def test_returns_none_for_unrecognised_blocking_line_syntax(): @@ -277,15 +280,59 @@ def test_the_dead_lines_own_action_and_the_outcome_can_differ(): outcome = _compute_dead_rule_outcome(detail) assert outcome is not None assert "own stated action is PERMIT" in outcome - assert "actually denied" in outcome - # Names the real blocking line rather than a vague "same traffic" claim - # (found in review, 5 November, on this exact AC-002 finding): the old - # wording read as "blocks all traffic with the same destination IP - # address", which is false -- this blocker is one source subnet to ANY - # destination. - assert "deny ip 10.10.10.0 0.0.0.255 any" in outcome + # Describes the real blocking line in plain English rather than a vague + # "same traffic" claim (found in review, 5 November) or raw ACL syntax + # (found in review, 9 November, on this exact AC-002 finding: raw + # syntax measurably made the model call this /24 source "a specific + # host" 16/30 times). "10.10.10.0 0.0.0.255 any" is one source subnet + # to ANY destination. + assert "denies all traffic from 10.10.10.0/24 to any destination" in outcome assert "same traffic" not in outcome assert "same destination" not in outcome + assert "a specific host" not in outcome + assert "0.0.0.255" not in outcome, "raw wildcard-mask syntax must not leak into the fact" + + +# --- _describe_acl_line (#401 review, @shubhamkataria2005, 9 November) ------- +# Measured live: naming the blocking line in raw ACL syntax (review, 5 +# November) made the model call a /24 source "a specific host" 16 of 30 +# times -- a different misdescription from the one it fixed. These pin the +# plain-English replacement directly against the two real shapes measured, +# plus the refusal cases this function must not guess past. + +def test_describes_a_wildcard_source_and_any_destination(): + assert _describe_acl_line("deny ip 10.10.10.0 0.0.0.255 any") == ( + "denies all traffic from 10.10.10.0/24 to any destination" + ) + + +def test_describes_a_non_ip_protocol_by_name(): + assert _describe_acl_line("deny tcp 10.10.10.0 0.0.0.255 any") == ( + "denies TCP traffic from 10.10.10.0/24 to any destination" + ) + + +def test_describes_a_host_destination(): + assert _describe_acl_line("permit udp any host 218.8.104.58 eq domain") == ( + "permits UDP traffic from anywhere to host 218.8.104.58 on port domain" + ) + + +def test_returns_none_for_a_source_port_clause(): + """A shape this project's own converters never emit -- refuse rather + than guess at what the extra token means.""" + assert _describe_acl_line("permit tcp any eq 1024 any eq 443") is None + + +def test_returns_none_for_a_non_contiguous_wildcard_mask(): + """Same reason analysis/draft_policy.py refuses one of these: the + inverted bytes do not form a legal netmask, so there is no CIDR to + state.""" + assert _describe_acl_line("deny ip 10.10.10.0 0.0.255.0 any") is None + + +def test_returns_none_for_too_few_tokens(): + assert _describe_acl_line("permit ip any") is None def _real_dead_rule_detail(dead_action: str, blocking_action: str) -> str: @@ -958,7 +1005,7 @@ def test_prompt_includes_the_computed_outcome_for_a_dead_rule_finding(): }, } prompt = _build_prompt(finding) - assert "denied" in prompt + assert "denies" in prompt assert "already verified" in prompt.lower() @@ -1006,7 +1053,7 @@ def test_prompt_prefers_the_dead_rule_computation_when_both_could_apply(): } prompt = _build_prompt(finding) assert "already verified" in prompt.lower() - assert "denied" in prompt + assert "denies" in prompt def test_prompt_includes_the_computed_outcome_for_a_testfilters_finding(): diff --git a/tests/test_explanation_quality.py b/tests/test_explanation_quality.py index a1bef8d..f689b53 100644 --- a/tests/test_explanation_quality.py +++ b/tests/test_explanation_quality.py @@ -164,16 +164,17 @@ def test_the_published_score_matches_the_committed_ratings(): deliberately (docs/evaluation.md Part 4), don't just update this assertion to make it pass. - Re-measured after #401 re-captured AC-002 under its fixed prompt -- - #385's guard flagged it stale the moment #401 merged onto this set, - since #384's defect is exactly what #401 fixes. AC-002 moved from 2/2 - to 5/4; see docs/evaluation.md Part 4 for the dated note.""" + Re-measured again (9 November) after review found the raw-ACL-syntax + wording traded one misdescription for another (#401 review, + @shubhamkataria2005) -- #385's guard flagged AC-002 as stale the + moment the fix changed its prompt. AC-002 moved from 5/4 to 5/5; see + docs/evaluation.md Part 4 for the dated note.""" score = score_holdout(load_holdout()) assert score["n_findings"] == 4 assert score["n_ratings"] == 4 assert score["mean_accuracy"] == pytest.approx(5.0) assert score["min_accuracy"] == 5 - assert score["mean_usefulness"] == pytest.approx(3.75) + assert score["mean_usefulness"] == pytest.approx(4.0) assert score["min_usefulness"] == 3 assert {row["id"] for row in score["per_finding"]} == { "AC-001", "PC-001", "AC-002", "PC-005", From bce901779b602edbc3b8674cee6c8f295de00664 Mon Sep 17 00:00:00 2001 From: patelankeet2 Date: Tue, 10 Nov 2026 14:21:52 +1300 Subject: [PATCH 7/7] Restore holdout.json's trailing newline (#401 review, Arsh) A small git-hygiene nit named in review -- the file's last save dropped the final newline after the closing brace. --- tests/fixtures/explanation_quality/holdout.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fixtures/explanation_quality/holdout.json b/tests/fixtures/explanation_quality/holdout.json index d3a3bcb..b749ab0 100644 --- a/tests/fixtures/explanation_quality/holdout.json +++ b/tests/fixtures/explanation_quality/holdout.json @@ -145,4 +145,4 @@ ] } ] -} \ No newline at end of file +}