From 8db4dec00fb9f877f62d9ab0f217ceb4af955b2a Mon Sep 17 00:00:00 2001 From: patelankeet2 Date: Tue, 13 Oct 2026 16:30:27 +1300 Subject: [PATCH 1/3] US-62: a computed fact for routing-assertion findings (#395) ai/explain.py had three computed-fact functions (dead rules, policy compliance, access_control policy statements) for one class of mistake: a model attributing which of two opposite states is which. Routing assertions had no equivalent, and this one found a different failure, not the one it was built looking for. Live-tested a real routing violation six times with no computed fact: the reachability direction itself (reachable vs not) was never inverted, but 3 of 6 runs inverted the flow's source and destination -- 'the HQ network cannot be reached FROM the branch network' instead of the real 'HQ cannot reach branch'. Both IPs were genuine in every run, so the existing invented-fact guard would not have caught it -- the addresses were just attributed to the wrong role in the relationship. _compute_routing_outcome() states the flow's direction explicitly (source before destination, 'that direction, not the reverse') as well as its real disposition, rather than only the disposition the way the other three functions state only their two-state fact. Re-ran the same live finding six times with the fix in place: 0 of 6 inverted the direction. Tests use the real routing._evaluate() to build the evidence string, not a hand-typed literal, same discipline test_the_safeguard_can_read_ what_policy_compliance_actually_writes already established (#145/#169) -- a future rewording of _evaluate() fails these tests loudly instead of leaving them green while testing a string that no longer exists. python -m pytest tests/test_explain_safety_net.py tests/test_routing_classification.py -q 96 passed. --- ai/explain.py | 99 +++++++++++++++++++++++++++--- tests/test_explain_safety_net.py | 102 +++++++++++++++++++++++++++++++ 2 files changed, 192 insertions(+), 9 deletions(-) diff --git a/ai/explain.py b/ai/explain.py index fcfa8b1..fc4c574 100644 --- a/ai/explain.py +++ b/ai/explain.py @@ -324,6 +324,85 @@ def _compute_expected_actual_outcome(detail: str) -> Optional[str]: ) +# ------------------------------------------------------------------------------ +# 1c-2. Deterministic routing-assertion outcome computation (US-62, #395) +# ------------------------------------------------------------------------------ + +# Matches evidence.detail EXACTLY as produced by +# analysis.checks.routing.py's assertion loop: +# "Expected REACHABLE for a flow from to , but traceroute ended +# in . Path: ..." +# "Expected UNREACHABLE for a flow from to , but traceroute +# unexpectedly succeeded: . Path: ..." +# Deliberately narrow, same discipline as the patterns above -- this must +# only match the two shapes _evaluate() in routing.py actually produces. +_ROUTING_ASSERTION_DETAIL_PATTERN = re.compile( + r"Expected (?PREACHABLE|UNREACHABLE) for a flow from " + r"(?P\S+) to (?P\S+), but " + r"(?:traceroute ended in (?P\w+)" + r"|traceroute unexpectedly succeeded: (?P\w+))" +) + + +def _compute_routing_outcome(detail: str) -> Optional[str]: + """For a routing-assertion finding, state -- in unambiguous words -- + which direction the flow runs and what actually happens to it, rather + than asking the model to keep both straight itself. + + WHY THIS EXISTS (found live, not assumed) + Live-tested against the local model before adding this, the same + way _compute_expected_actual_outcome() was: six runs of a real + finding in this shape (`Expected REACHABLE for a flow from + 10.10.10.5 to 10.20.20.5, but traceroute ended in NO_ROUTE`). + + The reachability direction itself (reachable vs not) was never + inverted in those six runs -- but 3 of 6 inverted something else: + the flow's SOURCE and DESTINATION. "The HQ network cannot reach the + branch network" (correct) became "The HQ network cannot be REACHED + FROM the branch network" in half the runs -- which describes the + opposite flow, branch-to-HQ rather than HQ-to-branch. Neither IP + address was invented, so `_names_an_invented_fact()` would not catch + it: both addresses are real, just attributed to the wrong role. + + This is a different failure from the "which of two states is which" + shape #52/#145/_compute_expected_actual_outcome() guard against -- + here both sides are directions of the SAME relationship (a flow + from A to B), not two different claims about one thing. So the + computed fact below states the direction explicitly ("the flow FROM + src TO dst, not the reverse") as well as the disposition, rather + than only the disposition. + + Returns a plain-English statement naming the flow's direction and its + real disposition explicitly, or None if `detail` is not in the exact + shape routing.py's assertion loop produces -- nothing to compute from, + so the model gets no extra help and reasons from the raw finding alone. + """ + match = _ROUTING_ASSERTION_DETAIL_PATTERN.search(detail) + if not match: + return None + + expected = match.group("expected") + src, dst = match.group("src"), match.group("dst") + if match.group("failed_disposition"): + disposition = match.group("failed_disposition") + return ( + f"the flow FROM {src} TO {dst} -- that direction, not the " + f"reverse -- currently does NOT reach {dst}; the trace ended in " + f"{disposition}. That is the real, observed outcome. The " + f"statement requires this flow to be {expected}; the network's " + f"current routing is what disagrees with that requirement, not " + f"the requirement itself." + ) + disposition = match.group("succeeded_disposition") + return ( + f"the flow FROM {src} TO {dst} -- that direction, not the " + f"reverse -- currently DOES reach {dst} ({disposition}). That is " + f"the real, observed outcome. The statement requires this flow to " + f"be {expected}; the network's current routing is what disagrees " + f"with that requirement, not the requirement itself." + ) + + # ------------------------------------------------------------------------------ # 1d. Deterministic remediation text (#221) # @@ -624,17 +703,18 @@ def _build_prompt(finding: Dict[str, Any]) -> str: example's answer almost verbatim for an unrelated finding, rather than reasoning about the finding it was actually given. - 2. If one of the three deterministic outcome functions can compute the + 2. If one of the four deterministic outcome functions can compute the real answer from evidence.detail -- _compute_dead_rule_outcome() for a dead-ACL-rule finding, _compute_policy_outcome() for a - policy_compliance finding (#145), or _compute_expected_actual_outcome() - for an access_control policy-statement finding -- that computed fact, - labelled as already-verified. See each function's docstring for why - it exists and how confident it has to be before it says anything at - all. The three never more than one matches at once: each check - produces its own exact evidence.detail shape, so trying the next one - only when an earlier one returns None does not risk masking one with - another. + policy_compliance finding (#145), _compute_expected_actual_outcome() + for an access_control policy-statement finding, or + _compute_routing_outcome() for a routing-assertion finding (#395) -- + that computed fact, labelled as already-verified. See each + function's docstring for why it exists and how confident it has to + be before it says anything at all. The four never more than one + matches at once: each check produces its own exact evidence.detail + shape, so trying the next one only when an earlier one returns None + does not risk masking one with another. """ parts = [ "Explain ONLY the finding below. Do not reuse any wording from the " @@ -648,6 +728,7 @@ def _build_prompt(finding: Dict[str, Any]) -> str: _compute_dead_rule_outcome(detail) or _compute_policy_outcome(detail) or _compute_expected_actual_outcome(detail) + or _compute_routing_outcome(detail) ) if computed_outcome is not None: # Deliberately NOT a distinctive, quotable label like "IMPORTANT diff --git a/tests/test_explain_safety_net.py b/tests/test_explain_safety_net.py index bf99281..353528a 100644 --- a/tests/test_explain_safety_net.py +++ b/tests/test_explain_safety_net.py @@ -28,6 +28,7 @@ _compute_dead_rule_outcome, _compute_expected_actual_outcome, _compute_policy_outcome, + _compute_routing_outcome, _evidence_detail, _fallback_error_explanation, _fallback_plain_restatement, @@ -326,6 +327,107 @@ def test_expected_actual_outcome_returns_none_for_a_policy_compliance_finding(): assert _compute_policy_outcome(detail) is not None # the OTHER guard still matches +# --- _compute_routing_outcome (routing assertions, US-62, #395) -------------- +# Found live, not assumed: six runs of a real routing-violation finding with +# no computed fact. The reachability direction itself (reachable vs not) was +# never inverted -- but 3 of 6 runs inverted the flow's SOURCE and +# DESTINATION ("the HQ network cannot be reached FROM the branch network" +# instead of the real "HQ cannot reach branch"). Both IPs were real in every +# run, so _names_an_invented_fact() would not have caught it -- the addresses +# were just attributed to the wrong role in the relationship. That is why the +# computed fact below states the direction explicitly, not only the +# disposition. + + +def _real_routing_detail(expected: str, disposition: str, src="10.10.10.5", + dst="10.20.20.5"): + """The genuine violation text, from the genuine function -- same + reasoning as _real_detail() above, so a future rewording of + routing._evaluate() fails this test loudly rather than leaving it green + while testing a string that no longer exists.""" + from analysis.checks.routing import _evaluate + + class _FakeHop: + def __init__(self, node): + self.node = node + + class _FakeTrace: + def __init__(self, disposition): + self.disposition = disposition + self.hops = [_FakeHop("rtr-a")] + + violation = _evaluate(expected, [_FakeTrace(disposition)]) + assert violation is not None, "fixture disposition did not violate the assertion" + return f"Expected {expected} for a flow from {src} to {dst}, but {violation}" + + +def test_a_required_reachable_flow_that_is_actually_blocked(): + """Real evidence.detail shape from routing.py's assertion loop -- a flow + required to reach its destination currently does not.""" + detail = _real_routing_detail("REACHABLE", "NO_ROUTE") + outcome = _compute_routing_outcome(detail) + assert outcome is not None + assert "REACHABLE" in outcome + assert "NO_ROUTE" in outcome + assert "NOT reach" in outcome + + +def test_a_required_unreachable_flow_that_is_actually_reaching(): + """The other direction -- a flow required to be blocked currently gets + through.""" + detail = _real_routing_detail("UNREACHABLE", "ACCEPTED") + outcome = _compute_routing_outcome(detail) + assert outcome is not None + assert "UNREACHABLE" in outcome + assert "ACCEPTED" in outcome + assert "DOES reach" in outcome + + +def test_the_flow_direction_is_stated_explicitly_not_only_the_disposition(): + """THE REASON THIS FUNCTION EXISTS. Both IPs must appear, in the real + source-then-destination order, so the model has the direction already + settled rather than free to invert it the way 3 of 6 live runs did + before this guard existed.""" + detail = _real_routing_detail("REACHABLE", "NO_ROUTE", src="10.10.10.5", dst="10.20.20.5") + outcome = _compute_routing_outcome(detail) + assert outcome is not None + assert "10.10.10.5" in outcome + assert "10.20.20.5" in outcome + assert outcome.index("10.10.10.5") < outcome.index("10.20.20.5"), ( + "source must be stated before destination, matching the real flow " + "direction, not left for the model to order" + ) + assert "not the reverse" in outcome + + +def test_routing_outcome_returns_none_for_an_unrelated_finding(): + detail = "BatfishException: Work terminated abnormally" + assert _compute_routing_outcome(detail) is None + + +def test_routing_outcome_returns_none_for_an_expected_actual_finding(): + """The four computations must not cross-match each other's shape.""" + detail = "Expected PERMIT but got DENY, decided by: deny ip any any" + assert _compute_routing_outcome(detail) is None + assert _compute_expected_actual_outcome(detail) is not None # the OTHER guard still matches + + +@pytest.mark.parametrize("expected,disposition", [ + ("REACHABLE", "NO_ROUTE"), + ("UNREACHABLE", "ACCEPTED"), +]) +def test_the_safeguard_can_read_what_routing_actually_writes(expected, disposition): + """Against the real producer, not a copy of its wording -- same + discipline test_the_safeguard_can_read_what_policy_compliance_actually_ + writes already uses.""" + detail = _real_routing_detail(expected, disposition) + outcome = _compute_routing_outcome(detail) + assert outcome is not None, ( + f"_compute_routing_outcome() did not match routing._evaluate()'s " + f"real output for {expected}/{disposition}: {detail!r}" + ) + + # --- _fallback_plain_restatement ----------------------------------------------- From 94bc75c8f8df620a39012aba214387d608a1eaf5 Mon Sep 17 00:00:00 2001 From: patelankeet2 Date: Tue, 13 Oct 2026 16:31:41 +1300 Subject: [PATCH 2/3] US-62: add the missing _build_prompt wiring test for routing (#395) Found by mutation-testing my own previous commit: removing _compute_routing_outcome() from _build_prompt()'s chain broke nothing in the direct unit tests, because they call the function in isolation rather than through the chain it is supposed to be wired into. Each of the other three computed-fact functions already has a dedicated test_prompt_includes_the_computed_outcome_for_a_X_finding test that exercises the real wiring -- routing was missing its equivalent. python -m pytest tests/test_explain_safety_net.py tests/test_routing_classification.py -q 97 passed (was 96). --- tests/test_explain_safety_net.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/test_explain_safety_net.py b/tests/test_explain_safety_net.py index 353528a..297f83b 100644 --- a/tests/test_explain_safety_net.py +++ b/tests/test_explain_safety_net.py @@ -805,6 +805,28 @@ def test_prompt_includes_the_computed_outcome_for_a_testfilters_finding(): assert "PERMIT" in prompt +def test_prompt_includes_the_computed_outcome_for_a_routing_finding(): + """US-62/#395 -- a routing assertion gets the same deterministic help as + the other three shapes. Caught by mutation testing: removing + _compute_routing_outcome() from _build_prompt()'s chain broke nothing + in the direct unit tests above, because those call the function + directly rather than through the chain it is supposed to be wired + into -- this is the test that actually exercises the wiring.""" + finding = { + "id": "RT-001", + "evidence": { + "detail": ( + "Expected REACHABLE for a flow from 10.10.10.5 to " + "10.20.20.5, but traceroute ended in NO_ROUTE. Path: rtr-hq" + ) + }, + } + prompt = _build_prompt(finding) + assert "already verified" in prompt.lower() + assert "10.10.10.5" in prompt + assert "10.20.20.5" in prompt + + def test_prompt_tells_the_model_not_to_echo_the_computed_fact_as_a_label(): """Regression coverage for a real failure: the model echoed the label on the injected fact verbatim as its own paragraph instead of folding From d2d12ae04b5ae980fde1f4b314ebcd5851366ba9 Mon Sep 17 00:00:00 2001 From: patelankeet2 Date: Sun, 1 Nov 2026 22:30:16 +1300 Subject: [PATCH 3/3] US-62: state a mixed multi-path result as a split, not a flat claim routing._evaluate() appends "(n of m paths failed/succeeded)" when more than one path was traced. The regex ignored that suffix, so a flow where only some paths disagree got the same flat "does NOT reach"/"DOES reach" statement as a unanimous result -- an overclaim on exactly the evidence this function exists to get right. When n < m (a real split), the computed fact now states the fraction explicitly, the same way ai/query.py's traceroute answer already states a mixed result rather than guessing a single yes/no, instead of returning None the way the other three computed-fact functions do on ambiguous evidence -- the fraction itself is the verified fact, and withholding it would leave the model to describe the split in its own words. When n == m every path agrees, so the flat statement is unchanged and still exactly accurate. Live-tested against the real model, 6 of 6 runs correctly conveyed the split rather than flattening it. Addresses the change requested on PR #399 (review by Shubham). --- ai/explain.py | 42 +++++++++++++++++++++++++++ tests/test_explain_safety_net.py | 50 ++++++++++++++++++++++++++++++-- 2 files changed, 90 insertions(+), 2 deletions(-) diff --git a/ai/explain.py b/ai/explain.py index fc4c574..3043a96 100644 --- a/ai/explain.py +++ b/ai/explain.py @@ -341,6 +341,10 @@ def _compute_expected_actual_outcome(detail: str) -> Optional[str]: r"(?P\S+) to (?P\S+), but " r"(?:traceroute ended in (?P\w+)" r"|traceroute unexpectedly succeeded: (?P\w+))" + # routing._evaluate() appends this when more than one path was traced + # (#399 review, @shubhamkataria2005) -- optional, since a single-path + # finding has no such suffix at all. + r"(?:.*\((?P\d+) of (?P\d+) paths (?:failed|succeeded)\))?" ) @@ -376,6 +380,21 @@ def _compute_routing_outcome(detail: str) -> Optional[str]: real disposition explicitly, or None if `detail` is not in the exact shape routing.py's assertion loop produces -- nothing to compute from, so the model gets no extra help and reasons from the raw finding alone. + + A PARTIAL multi-path result is stated as a split, not flattened (#399 + review, @shubhamkataria2005). routing._evaluate() appends "(n of m paths + failed/succeeded)" when more than one path was traced. If n < m, some + paths disagree with the rest, and a flat "does NOT reach"/"DOES reach" + would overclaim exactly what this function exists to prevent -- the + other three computed-fact functions already refuse to pick a side on + genuinely mixed evidence (_compute_dead_rule_outcome() on disagreeing + blocking lines), and ai/query.py's own traceroute answer states a mixed + result as a fraction rather than guessing a single yes/no. This mirrors + that rather than returning None, because the fraction itself IS the + verified fact -- withholding it would leave the model to describe the + split in its own words, the exact risk this function family exists to + remove. When n == m every path agrees, so the flat statement is still + exactly accurate and unchanged. """ match = _ROUTING_ASSERTION_DETAIL_PATTERN.search(detail) if not match: @@ -383,8 +402,21 @@ def _compute_routing_outcome(detail: str) -> Optional[str]: expected = match.group("expected") src, dst = match.group("src"), match.group("dst") + n, m = match.group("n"), match.group("m") + mixed = n is not None and int(n) < int(m) + if match.group("failed_disposition"): disposition = match.group("failed_disposition") + if mixed: + return ( + f"{n} of {m} traced paths for the flow FROM {src} TO {dst} " + f"-- that direction, not the reverse -- do NOT reach {dst} " + f"(one ended in {disposition}); the rest do. That is a " + f"real, mixed result, not a single yes or no -- state both " + f"parts, do not round it to one. The statement requires " + f"this flow to be {expected}; whether that holds depends on " + f"which path is taken." + ) return ( f"the flow FROM {src} TO {dst} -- that direction, not the " f"reverse -- currently does NOT reach {dst}; the trace ended in " @@ -394,6 +426,16 @@ def _compute_routing_outcome(detail: str) -> Optional[str]: f"the requirement itself." ) disposition = match.group("succeeded_disposition") + if mixed: + return ( + f"{n} of {m} traced paths for the flow FROM {src} TO {dst} -- " + f"that direction, not the reverse -- DO reach {dst} (one " + f"succeeded as {disposition}); the rest do not. That is a " + f"real, mixed result, not a single yes or no -- state both " + f"parts, do not round it to one. The statement requires this " + f"flow to be {expected}; whether that holds depends on which " + f"path is taken." + ) return ( f"the flow FROM {src} TO {dst} -- that direction, not the " f"reverse -- currently DOES reach {dst} ({disposition}). That is " diff --git a/tests/test_explain_safety_net.py b/tests/test_explain_safety_net.py index 297f83b..0037ce5 100644 --- a/tests/test_explain_safety_net.py +++ b/tests/test_explain_safety_net.py @@ -345,6 +345,15 @@ def _real_routing_detail(expected: str, disposition: str, src="10.10.10.5", reasoning as _real_detail() above, so a future rewording of routing._evaluate() fails this test loudly rather than leaving it green while testing a string that no longer exists.""" + return _real_routing_detail_multi(expected, [disposition], src=src, dst=dst) + + +def _real_routing_detail_multi(expected: str, dispositions, src="10.10.10.5", + dst="10.20.20.5"): + """Same as _real_routing_detail(), but for more than one traced path -- + exercises routing._evaluate()'s "(n of m paths failed/succeeded)" suffix + against the real function, for the mixed-result case (#399 review, + @shubhamkataria2005).""" from analysis.checks.routing import _evaluate class _FakeHop: @@ -356,8 +365,8 @@ def __init__(self, disposition): self.disposition = disposition self.hops = [_FakeHop("rtr-a")] - violation = _evaluate(expected, [_FakeTrace(disposition)]) - assert violation is not None, "fixture disposition did not violate the assertion" + violation = _evaluate(expected, [_FakeTrace(d) for d in dispositions]) + assert violation is not None, "fixture dispositions did not violate the assertion" return f"Expected {expected} for a flow from {src} to {dst}, but {violation}" @@ -400,6 +409,43 @@ def test_the_flow_direction_is_stated_explicitly_not_only_the_disposition(): assert "not the reverse" in outcome +def test_a_mixed_reachable_result_states_the_split_not_a_flat_claim(): + """#399 review, @shubhamkataria2005. One of two paths fails; the flow + DOES reach its destination half the time, so a flat "does NOT reach" + would overclaim exactly what this function exists to prevent.""" + detail = _real_routing_detail_multi("REACHABLE", ["NO_ROUTE", "ACCEPTED"]) + outcome = _compute_routing_outcome(detail) + assert outcome is not None + assert "1 of 2" in outcome + assert "do NOT reach" in outcome + assert "mixed" in outcome + assert "does NOT reach" not in outcome, "must not also state the flat claim" + + +def test_a_mixed_unreachable_result_states_the_split_not_a_flat_claim(): + """The other direction: one of two paths gets through, so a flat "DOES + reach" would claim more than the evidence proves.""" + detail = _real_routing_detail_multi("UNREACHABLE", ["ACCEPTED", "NO_ROUTE"]) + outcome = _compute_routing_outcome(detail) + assert outcome is not None + assert "1 of 2" in outcome + assert "DO reach" in outcome + assert "mixed" in outcome + assert "DOES reach" not in outcome, "must not also state the flat claim" + + +def test_a_unanimous_multi_path_result_still_states_the_flat_claim(): + """Every path agrees (2 of 2 failed) -- not a mixed result, so the flat + statement is still exactly accurate and must not be watered down into + a "mixed" hedge it does not need.""" + detail = _real_routing_detail_multi("REACHABLE", ["NO_ROUTE", "DENIED_IN"]) + outcome = _compute_routing_outcome(detail) + assert outcome is not None + assert "does NOT reach" in outcome + assert "mixed" not in outcome + assert "of 2" not in outcome + + def test_routing_outcome_returns_none_for_an_unrelated_finding(): detail = "BatfishException: Work terminated abnormally" assert _compute_routing_outcome(detail) is None