diff --git a/ai/explain.py b/ai/explain.py index fcfa8b1..3043a96 100644 --- a/ai/explain.py +++ b/ai/explain.py @@ -324,6 +324,127 @@ def _compute_expected_actual_outcome(detail: str) -> Optional[str]: ) +# ------------------------------------------------------------------------------ +# 1c-2. Deterministic routing-assertion outcome computation (US-62, #395) +# ------------------------------------------------------------------------------ + +# Matches evidence.detail EXACTLY as produced by +# analysis.checks.routing.py's assertion loop: +# "Expected REACHABLE for a flow from to , but traceroute ended +# in . Path: ..." +# "Expected UNREACHABLE for a flow from to , but traceroute +# unexpectedly succeeded: . Path: ..." +# Deliberately narrow, same discipline as the patterns above -- this must +# only match the two shapes _evaluate() in routing.py actually produces. +_ROUTING_ASSERTION_DETAIL_PATTERN = re.compile( + r"Expected (?PREACHABLE|UNREACHABLE) for a flow from " + r"(?P\S+) to (?P\S+), but " + r"(?:traceroute ended in (?P\w+)" + r"|traceroute unexpectedly succeeded: (?P\w+))" + # routing._evaluate() appends this when more than one path was traced + # (#399 review, @shubhamkataria2005) -- optional, since a single-path + # finding has no such suffix at all. + r"(?:.*\((?P\d+) of (?P\d+) paths (?:failed|succeeded)\))?" +) + + +def _compute_routing_outcome(detail: str) -> Optional[str]: + """For a routing-assertion finding, state -- in unambiguous words -- + which direction the flow runs and what actually happens to it, rather + than asking the model to keep both straight itself. + + WHY THIS EXISTS (found live, not assumed) + Live-tested against the local model before adding this, the same + way _compute_expected_actual_outcome() was: six runs of a real + finding in this shape (`Expected REACHABLE for a flow from + 10.10.10.5 to 10.20.20.5, but traceroute ended in NO_ROUTE`). + + The reachability direction itself (reachable vs not) was never + inverted in those six runs -- but 3 of 6 inverted something else: + the flow's SOURCE and DESTINATION. "The HQ network cannot reach the + branch network" (correct) became "The HQ network cannot be REACHED + FROM the branch network" in half the runs -- which describes the + opposite flow, branch-to-HQ rather than HQ-to-branch. Neither IP + address was invented, so `_names_an_invented_fact()` would not catch + it: both addresses are real, just attributed to the wrong role. + + This is a different failure from the "which of two states is which" + shape #52/#145/_compute_expected_actual_outcome() guard against -- + here both sides are directions of the SAME relationship (a flow + from A to B), not two different claims about one thing. So the + computed fact below states the direction explicitly ("the flow FROM + src TO dst, not the reverse") as well as the disposition, rather + than only the disposition. + + Returns a plain-English statement naming the flow's direction and its + real disposition explicitly, or None if `detail` is not in the exact + shape routing.py's assertion loop produces -- nothing to compute from, + so the model gets no extra help and reasons from the raw finding alone. + + A PARTIAL multi-path result is stated as a split, not flattened (#399 + review, @shubhamkataria2005). routing._evaluate() appends "(n of m paths + failed/succeeded)" when more than one path was traced. If n < m, some + paths disagree with the rest, and a flat "does NOT reach"/"DOES reach" + would overclaim exactly what this function exists to prevent -- the + other three computed-fact functions already refuse to pick a side on + genuinely mixed evidence (_compute_dead_rule_outcome() on disagreeing + blocking lines), and ai/query.py's own traceroute answer states a mixed + result as a fraction rather than guessing a single yes/no. This mirrors + that rather than returning None, because the fraction itself IS the + verified fact -- withholding it would leave the model to describe the + split in its own words, the exact risk this function family exists to + remove. When n == m every path agrees, so the flat statement is still + exactly accurate and unchanged. + """ + match = _ROUTING_ASSERTION_DETAIL_PATTERN.search(detail) + if not match: + return None + + expected = match.group("expected") + src, dst = match.group("src"), match.group("dst") + n, m = match.group("n"), match.group("m") + mixed = n is not None and int(n) < int(m) + + if match.group("failed_disposition"): + disposition = match.group("failed_disposition") + if mixed: + return ( + f"{n} of {m} traced paths for the flow FROM {src} TO {dst} " + f"-- that direction, not the reverse -- do NOT reach {dst} " + f"(one ended in {disposition}); the rest do. That is a " + f"real, mixed result, not a single yes or no -- state both " + f"parts, do not round it to one. The statement requires " + f"this flow to be {expected}; whether that holds depends on " + f"which path is taken." + ) + return ( + f"the flow FROM {src} TO {dst} -- that direction, not the " + f"reverse -- currently does NOT reach {dst}; the trace ended in " + f"{disposition}. That is the real, observed outcome. The " + f"statement requires this flow to be {expected}; the network's " + f"current routing is what disagrees with that requirement, not " + f"the requirement itself." + ) + disposition = match.group("succeeded_disposition") + if mixed: + return ( + f"{n} of {m} traced paths for the flow FROM {src} TO {dst} -- " + f"that direction, not the reverse -- DO reach {dst} (one " + f"succeeded as {disposition}); the rest do not. That is a " + f"real, mixed result, not a single yes or no -- state both " + f"parts, do not round it to one. The statement requires this " + f"flow to be {expected}; whether that holds depends on which " + f"path is taken." + ) + return ( + f"the flow FROM {src} TO {dst} -- that direction, not the " + f"reverse -- currently DOES reach {dst} ({disposition}). That is " + f"the real, observed outcome. The statement requires this flow to " + f"be {expected}; the network's current routing is what disagrees " + f"with that requirement, not the requirement itself." + ) + + # ------------------------------------------------------------------------------ # 1d. Deterministic remediation text (#221) # @@ -624,17 +745,18 @@ def _build_prompt(finding: Dict[str, Any]) -> str: example's answer almost verbatim for an unrelated finding, rather than reasoning about the finding it was actually given. - 2. If one of the three deterministic outcome functions can compute the + 2. If one of the four deterministic outcome functions can compute the real answer from evidence.detail -- _compute_dead_rule_outcome() for a dead-ACL-rule finding, _compute_policy_outcome() for a - policy_compliance finding (#145), or _compute_expected_actual_outcome() - for an access_control policy-statement finding -- that computed fact, - labelled as already-verified. See each function's docstring for why - it exists and how confident it has to be before it says anything at - all. The three never more than one matches at once: each check - produces its own exact evidence.detail shape, so trying the next one - only when an earlier one returns None does not risk masking one with - another. + policy_compliance finding (#145), _compute_expected_actual_outcome() + for an access_control policy-statement finding, or + _compute_routing_outcome() for a routing-assertion finding (#395) -- + that computed fact, labelled as already-verified. See each + function's docstring for why it exists and how confident it has to + be before it says anything at all. The four never more than one + matches at once: each check produces its own exact evidence.detail + shape, so trying the next one only when an earlier one returns None + does not risk masking one with another. """ parts = [ "Explain ONLY the finding below. Do not reuse any wording from the " @@ -648,6 +770,7 @@ def _build_prompt(finding: Dict[str, Any]) -> str: _compute_dead_rule_outcome(detail) or _compute_policy_outcome(detail) or _compute_expected_actual_outcome(detail) + or _compute_routing_outcome(detail) ) if computed_outcome is not None: # Deliberately NOT a distinctive, quotable label like "IMPORTANT diff --git a/tests/test_explain_safety_net.py b/tests/test_explain_safety_net.py index bf99281..0037ce5 100644 --- a/tests/test_explain_safety_net.py +++ b/tests/test_explain_safety_net.py @@ -28,6 +28,7 @@ _compute_dead_rule_outcome, _compute_expected_actual_outcome, _compute_policy_outcome, + _compute_routing_outcome, _evidence_detail, _fallback_error_explanation, _fallback_plain_restatement, @@ -326,6 +327,153 @@ def test_expected_actual_outcome_returns_none_for_a_policy_compliance_finding(): assert _compute_policy_outcome(detail) is not None # the OTHER guard still matches +# --- _compute_routing_outcome (routing assertions, US-62, #395) -------------- +# Found live, not assumed: six runs of a real routing-violation finding with +# no computed fact. The reachability direction itself (reachable vs not) was +# never inverted -- but 3 of 6 runs inverted the flow's SOURCE and +# DESTINATION ("the HQ network cannot be reached FROM the branch network" +# instead of the real "HQ cannot reach branch"). Both IPs were real in every +# run, so _names_an_invented_fact() would not have caught it -- the addresses +# were just attributed to the wrong role in the relationship. That is why the +# computed fact below states the direction explicitly, not only the +# disposition. + + +def _real_routing_detail(expected: str, disposition: str, src="10.10.10.5", + dst="10.20.20.5"): + """The genuine violation text, from the genuine function -- same + reasoning as _real_detail() above, so a future rewording of + routing._evaluate() fails this test loudly rather than leaving it green + while testing a string that no longer exists.""" + return _real_routing_detail_multi(expected, [disposition], src=src, dst=dst) + + +def _real_routing_detail_multi(expected: str, dispositions, src="10.10.10.5", + dst="10.20.20.5"): + """Same as _real_routing_detail(), but for more than one traced path -- + exercises routing._evaluate()'s "(n of m paths failed/succeeded)" suffix + against the real function, for the mixed-result case (#399 review, + @shubhamkataria2005).""" + from analysis.checks.routing import _evaluate + + class _FakeHop: + def __init__(self, node): + self.node = node + + class _FakeTrace: + def __init__(self, disposition): + self.disposition = disposition + self.hops = [_FakeHop("rtr-a")] + + violation = _evaluate(expected, [_FakeTrace(d) for d in dispositions]) + assert violation is not None, "fixture dispositions did not violate the assertion" + return f"Expected {expected} for a flow from {src} to {dst}, but {violation}" + + +def test_a_required_reachable_flow_that_is_actually_blocked(): + """Real evidence.detail shape from routing.py's assertion loop -- a flow + required to reach its destination currently does not.""" + detail = _real_routing_detail("REACHABLE", "NO_ROUTE") + outcome = _compute_routing_outcome(detail) + assert outcome is not None + assert "REACHABLE" in outcome + assert "NO_ROUTE" in outcome + assert "NOT reach" in outcome + + +def test_a_required_unreachable_flow_that_is_actually_reaching(): + """The other direction -- a flow required to be blocked currently gets + through.""" + detail = _real_routing_detail("UNREACHABLE", "ACCEPTED") + outcome = _compute_routing_outcome(detail) + assert outcome is not None + assert "UNREACHABLE" in outcome + assert "ACCEPTED" in outcome + assert "DOES reach" in outcome + + +def test_the_flow_direction_is_stated_explicitly_not_only_the_disposition(): + """THE REASON THIS FUNCTION EXISTS. Both IPs must appear, in the real + source-then-destination order, so the model has the direction already + settled rather than free to invert it the way 3 of 6 live runs did + before this guard existed.""" + detail = _real_routing_detail("REACHABLE", "NO_ROUTE", src="10.10.10.5", dst="10.20.20.5") + outcome = _compute_routing_outcome(detail) + assert outcome is not None + assert "10.10.10.5" in outcome + assert "10.20.20.5" in outcome + assert outcome.index("10.10.10.5") < outcome.index("10.20.20.5"), ( + "source must be stated before destination, matching the real flow " + "direction, not left for the model to order" + ) + assert "not the reverse" in outcome + + +def test_a_mixed_reachable_result_states_the_split_not_a_flat_claim(): + """#399 review, @shubhamkataria2005. One of two paths fails; the flow + DOES reach its destination half the time, so a flat "does NOT reach" + would overclaim exactly what this function exists to prevent.""" + detail = _real_routing_detail_multi("REACHABLE", ["NO_ROUTE", "ACCEPTED"]) + outcome = _compute_routing_outcome(detail) + assert outcome is not None + assert "1 of 2" in outcome + assert "do NOT reach" in outcome + assert "mixed" in outcome + assert "does NOT reach" not in outcome, "must not also state the flat claim" + + +def test_a_mixed_unreachable_result_states_the_split_not_a_flat_claim(): + """The other direction: one of two paths gets through, so a flat "DOES + reach" would claim more than the evidence proves.""" + detail = _real_routing_detail_multi("UNREACHABLE", ["ACCEPTED", "NO_ROUTE"]) + outcome = _compute_routing_outcome(detail) + assert outcome is not None + assert "1 of 2" in outcome + assert "DO reach" in outcome + assert "mixed" in outcome + assert "DOES reach" not in outcome, "must not also state the flat claim" + + +def test_a_unanimous_multi_path_result_still_states_the_flat_claim(): + """Every path agrees (2 of 2 failed) -- not a mixed result, so the flat + statement is still exactly accurate and must not be watered down into + a "mixed" hedge it does not need.""" + detail = _real_routing_detail_multi("REACHABLE", ["NO_ROUTE", "DENIED_IN"]) + outcome = _compute_routing_outcome(detail) + assert outcome is not None + assert "does NOT reach" in outcome + assert "mixed" not in outcome + assert "of 2" not in outcome + + +def test_routing_outcome_returns_none_for_an_unrelated_finding(): + detail = "BatfishException: Work terminated abnormally" + assert _compute_routing_outcome(detail) is None + + +def test_routing_outcome_returns_none_for_an_expected_actual_finding(): + """The four computations must not cross-match each other's shape.""" + detail = "Expected PERMIT but got DENY, decided by: deny ip any any" + assert _compute_routing_outcome(detail) is None + assert _compute_expected_actual_outcome(detail) is not None # the OTHER guard still matches + + +@pytest.mark.parametrize("expected,disposition", [ + ("REACHABLE", "NO_ROUTE"), + ("UNREACHABLE", "ACCEPTED"), +]) +def test_the_safeguard_can_read_what_routing_actually_writes(expected, disposition): + """Against the real producer, not a copy of its wording -- same + discipline test_the_safeguard_can_read_what_policy_compliance_actually_ + writes already uses.""" + detail = _real_routing_detail(expected, disposition) + outcome = _compute_routing_outcome(detail) + assert outcome is not None, ( + f"_compute_routing_outcome() did not match routing._evaluate()'s " + f"real output for {expected}/{disposition}: {detail!r}" + ) + + # --- _fallback_plain_restatement ----------------------------------------------- @@ -703,6 +851,28 @@ def test_prompt_includes_the_computed_outcome_for_a_testfilters_finding(): assert "PERMIT" in prompt +def test_prompt_includes_the_computed_outcome_for_a_routing_finding(): + """US-62/#395 -- a routing assertion gets the same deterministic help as + the other three shapes. Caught by mutation testing: removing + _compute_routing_outcome() from _build_prompt()'s chain broke nothing + in the direct unit tests above, because those call the function + directly rather than through the chain it is supposed to be wired + into -- this is the test that actually exercises the wiring.""" + finding = { + "id": "RT-001", + "evidence": { + "detail": ( + "Expected REACHABLE for a flow from 10.10.10.5 to " + "10.20.20.5, but traceroute ended in NO_ROUTE. Path: rtr-hq" + ) + }, + } + prompt = _build_prompt(finding) + assert "already verified" in prompt.lower() + assert "10.10.10.5" in prompt + assert "10.20.20.5" in prompt + + def test_prompt_tells_the_model_not_to_echo_the_computed_fact_as_a_label(): """Regression coverage for a real failure: the model echoed the label on the injected fact verbatim as its own paragraph instead of folding