From fff7b69fbdf73c7b4c406b0d7dd822f7ec9e316b Mon Sep 17 00:00:00 2001 From: patelankeet2 Date: Wed, 28 Oct 2026 09:23:34 +1300 Subject: [PATCH] A fourth computed-fact guard, for routing route assertions (#395) _compute_route_assertion_outcome() follows the exact pattern of the three existing computed-fact functions: routing.py's route assertions use the same two-sided "Expected X ... but Y" phrasing that #52 and #145 already found the model getting backwards on real findings, so the real outcome is computed in Python and handed to the model as an already-verified fact rather than something to re-derive. Wired into _build_prompt()'s or-chain alongside the other three, same ordering discipline (each check produces its own exact evidence.detail shape, so trying the next only when an earlier one returns None cannot mask one with another). Live-tested six times against a real finding (routing-missing-route's RT-001, Ollama + Batfish running) before adding the guard: every run correctly stated the flow was blocked, no inversion reproduced. Recorded as a clean but limited result, not proof of safety, the same honesty the third guard's own docstring already holds itself to. New tests follow test_explain_safety_net.py's own established conventions exactly: a _real_route_detail() helper building evidence text from the real routing._evaluate() producer (not a hardcoded literal, the #145/#169/#194 lesson), mutual-exclusion tests against the other three shapes, and a wiring test confirming the computed fact actually reaches the generated prompt text. 6 new tests, 95/95 passing in the directly affected files. --- ai/explain.py | 109 +++++++++++++++++++++++++++---- tests/test_explain_safety_net.py | 93 ++++++++++++++++++++++++++ 2 files changed, 190 insertions(+), 12 deletions(-) diff --git a/ai/explain.py b/ai/explain.py index fcfa8b1..417ae3a 100644 --- a/ai/explain.py +++ b/ai/explain.py @@ -78,6 +78,20 @@ testing. _compute_expected_actual_outcome() computes it the same way the other two do, and costs nothing when it does not match. + A fourth shape (#395) was added on the identical reasoning, in a + check this module had not touched before: routing.py's route + assertions use the same two-sided "Expected X ... but Y" phrasing, + asking whether a flow meant to be reachable actually is, or a flow + meant to be blocked actually got through. Live-tested six times + against a real finding (routing-missing-route's RT-001, an + Expected-REACHABLE assertion resolving to NO_ROUTE) before adding + the guard -- every run correctly stated the flow was blocked, no + inversion reproduced. Recorded the same way as the third shape: a + clean result on the one real finding tested, not proof the model + can never get this backwards. _compute_route_assertion_outcome() + computes it the same way the other three do, and costs nothing + when it does not match. + WHY status="error" ALSO GETS ITS OWN CHECK For "found"/"none", a wording slip is undesirable but not dangerous -- the underlying fact was still real, the phrasing was just off. For @@ -325,10 +339,79 @@ def _compute_expected_actual_outcome(detail: str) -> Optional[str]: # ------------------------------------------------------------------------------ -# 1d. Deterministic remediation text (#221) +# 1d. Deterministic route-assertion outcome computation (#395, US-62) +# ------------------------------------------------------------------------------ + +# Matches evidence.detail EXACTLY as produced by +# analysis.checks.routing._evaluate() (via run()'s route-statement loop): +# "Expected REACHABLE for a flow from to , but traceroute ended +# in . Path: " +# "Expected UNREACHABLE for a flow from to , but traceroute +# unexpectedly succeeded: . Path: " +# Deliberately narrow, same discipline as the three patterns above -- this +# must only match the two shapes it is confident about, never a loose +# approximation of them. +_ROUTE_ASSERTION_DETAIL_PATTERN = re.compile( + r"Expected (?PREACHABLE|UNREACHABLE) for a flow from " + r"(?P\S+) to (?P\S+), but " + r"(?:traceroute ended in (?P\w+)" + r"|traceroute unexpectedly succeeded: (?P\w+))" +) + + +def _compute_route_assertion_outcome(detail: str) -> Optional[str]: + """For a routing route-assertion finding, state -- in unambiguous + words -- what actually happened versus what was expected, rather than + asking the model to keep the two straight itself. + + WHY THIS EXISTS (#395) + The identical "which of two opposite states is which" attribution + task #52 (dead rules) and #145 (policy_compliance) already got + backwards on real findings, and access_control's own + `_compute_expected_actual_outcome()` was added on the same + structural risk before a third accident. `routing.py`'s route + assertions use the same two-sided phrasing ("Expected X ... but + Y") the other guarded shapes needed a fix for -- a route meant to + be reachable that is not, or a route meant to be blocked that + gets through -- so a model restating it could plausibly invert + which one is true. Live-tested against the local model before + adding this; see the PR this shipped in for the result, recorded + honestly whether or not an inversion reproduced on that occasion, + the same bar `_compute_expected_actual_outcome()` was added at. + + Returns a plain-English statement naming both sides explicitly, or + None if `detail` is not in one of the two exact shapes + `routing._evaluate()` produces -- nothing to compute from, so the + model gets no extra help and reasons from the raw finding alone, same + as any other finding shape. + """ + match = _ROUTE_ASSERTION_DETAIL_PATTERN.search(detail) + if not match: + return None + + expected = match.group("expected") + if expected == "REACHABLE": + disposition = match.group("failed_disposition") + return ( + f"this flow was expected to be REACHABLE, but it is not -- " + f"the actual result is {disposition}. The route or filter " + f"configuration is what is blocking it, not a requirement " + f"that says it should be." + ) + disposition = match.group("succeeded_disposition") + return ( + f"this flow was expected to be UNREACHABLE, but it got through -- " + f"the actual result is {disposition}. The route or filter " + f"configuration is what is letting it through, despite the " + f"requirement that it should not." + ) + + +# ------------------------------------------------------------------------------ +# 1e. Deterministic remediation text (#221) # -# A DIFFERENT QUESTION FROM SECTIONS 1a-1c ABOVE -# Those three compute what a finding MEANS, to correct the model's +# A DIFFERENT QUESTION FROM SECTIONS 1a-1d ABOVE +# Those four compute what a finding MEANS, to correct the model's # wording before it speaks. This computes what would FIX it, and never # goes near the model at all -- ai/Modelfile rule 6 explicitly forbids # recommending a fix "unless the evidence itself states what would @@ -624,17 +707,18 @@ def _build_prompt(finding: Dict[str, Any]) -> str: example's answer almost verbatim for an unrelated finding, rather than reasoning about the finding it was actually given. - 2. If one of the three deterministic outcome functions can compute the + 2. If one of the four deterministic outcome functions can compute the real answer from evidence.detail -- _compute_dead_rule_outcome() for a dead-ACL-rule finding, _compute_policy_outcome() for a - policy_compliance finding (#145), or _compute_expected_actual_outcome() - for an access_control policy-statement finding -- that computed fact, - labelled as already-verified. See each function's docstring for why - it exists and how confident it has to be before it says anything at - all. The three never more than one matches at once: each check - produces its own exact evidence.detail shape, so trying the next one - only when an earlier one returns None does not risk masking one with - another. + policy_compliance finding (#145), _compute_expected_actual_outcome() + for an access_control policy-statement finding, or + _compute_route_assertion_outcome() for a routing route-assertion + finding (#395) -- that computed fact, labelled as already-verified. + See each function's docstring for why it exists and how confident it + has to be before it says anything at all. The four never more than + one matches at once: each check produces its own exact + evidence.detail shape, so trying the next one only when an earlier + one returns None does not risk masking one with another. """ parts = [ "Explain ONLY the finding below. Do not reuse any wording from the " @@ -648,6 +732,7 @@ def _build_prompt(finding: Dict[str, Any]) -> str: _compute_dead_rule_outcome(detail) or _compute_policy_outcome(detail) or _compute_expected_actual_outcome(detail) + or _compute_route_assertion_outcome(detail) ) if computed_outcome is not None: # Deliberately NOT a distinctive, quotable label like "IMPORTANT diff --git a/tests/test_explain_safety_net.py b/tests/test_explain_safety_net.py index bf99281..c20b613 100644 --- a/tests/test_explain_safety_net.py +++ b/tests/test_explain_safety_net.py @@ -28,6 +28,7 @@ _compute_dead_rule_outcome, _compute_expected_actual_outcome, _compute_policy_outcome, + _compute_route_assertion_outcome, _evidence_detail, _fallback_error_explanation, _fallback_plain_restatement, @@ -326,6 +327,98 @@ def test_expected_actual_outcome_returns_none_for_a_policy_compliance_finding(): assert _compute_policy_outcome(detail) is not None # the OTHER guard still matches +# --- _compute_route_assertion_outcome (routing) ----------------------------- +# Same structural reasoning as the three guards above, a fourth producer of +# the identical "which of two opposite states is which" shape: a route +# assertion names what was expected and what actually happened, on real +# routing.py route-assertion findings. Live-tested six times against a real +# finding (routing-missing-route, RT-001) before adding the guard -- see the +# PR this shipped in for the recorded result. + + +def _real_route_detail(expected: str, trace): + """The genuine evidence string, from the genuine function -- same + discipline _real_detail() already uses for policy_compliance, so a + future rewording of routing._evaluate()'s text fails this test loudly + instead of leaving it green while it tests nothing (#145/#169).""" + from analysis.checks.routing import _evaluate + + class FakeHop: + def __init__(self, node): + self.node = node + + class FakeTrace: + def __init__(self, disposition, hops=None): + self.disposition = disposition + self.hops = hops or [FakeHop("some-node")] + + violation = _evaluate(expected, [FakeTrace(trace)]) + assert violation is not None, "test setup error: this trace should violate" + return f"Expected {expected} for a flow from 10.10.10.5 to 10.20.20.5, but {violation}" + + +def test_a_required_reachable_route_that_is_actually_blocked(): + """Real evidence.detail shape from routing._evaluate()'s REACHABLE + branch -- a flow that was supposed to get through did not.""" + detail = _real_route_detail("REACHABLE", "NO_ROUTE") + outcome = _compute_route_assertion_outcome(detail) + assert outcome is not None + assert "REACHABLE" in outcome + assert "NO_ROUTE" in outcome + + +def test_a_required_unreachable_route_that_actually_got_through(): + """The other direction -- a flow that was supposed to be blocked + succeeded instead.""" + detail = _real_route_detail("UNREACHABLE", "ACCEPTED") + outcome = _compute_route_assertion_outcome(detail) + assert outcome is not None + assert "UNREACHABLE" in outcome + assert "ACCEPTED" in outcome + + +def test_route_assertion_outcome_returns_none_for_an_unrelated_finding(): + detail = "BatfishException: Work terminated abnormally" + assert _compute_route_assertion_outcome(detail) is None + + +def test_route_assertion_outcome_returns_none_for_a_dead_rule_finding(): + """The four computations must not cross-match each other's shape.""" + detail = ( + "Unreachable line: permit icmp any any (action PERMIT). " + "Blocked by: deny ip any any. Reason: BLOCKING_LINES" + ) + assert _compute_route_assertion_outcome(detail) is None + + +def test_route_assertion_outcome_returns_none_for_an_expected_actual_finding(): + """Against the real producer, not a copy of its wording -- same + #145/#169 discipline as the cross-match test above it.""" + detail = "Expected PERMIT but got DENY, decided by: deny ip any any" + assert _compute_route_assertion_outcome(detail) is None + assert _compute_expected_actual_outcome(detail) is not None # the OTHER guard still matches + + +def test_route_assertion_outcome_reaches_the_generated_prompt(): + """The wiring, not just the function -- a computed fact that exists but + is never passed to _build_prompt() protects nothing. Matches the + "wiring, not just the function" discipline this project's other + computed-fact and discard-function tests already use.""" + detail = _real_route_detail("REACHABLE", "NO_ROUTE") + finding = { + "id": "RT-001", + "check": "routing", + "severity": "high", + "device": "rtr-hq", + "summary": "A route assertion does not hold", + "evidence": {"detail": detail, "source": "traceroute"}, + "status": "found", + } + prompt = _build_prompt(finding) + assert "NO_ROUTE" in prompt + assert "already verified by code and certain" in prompt + + # --- _fallback_plain_restatement -----------------------------------------------