diff --git a/ai/explain.py b/ai/explain.py index fcfa8b1..417ae3a 100644 --- a/ai/explain.py +++ b/ai/explain.py @@ -78,6 +78,20 @@ testing. _compute_expected_actual_outcome() computes it the same way the other two do, and costs nothing when it does not match. + A fourth shape (#395) was added on the identical reasoning, in a + check this module had not touched before: routing.py's route + assertions use the same two-sided "Expected X ... but Y" phrasing, + asking whether a flow meant to be reachable actually is, or a flow + meant to be blocked actually got through. Live-tested six times + against a real finding (routing-missing-route's RT-001, an + Expected-REACHABLE assertion resolving to NO_ROUTE) before adding + the guard -- every run correctly stated the flow was blocked, no + inversion reproduced. Recorded the same way as the third shape: a + clean result on the one real finding tested, not proof the model + can never get this backwards. _compute_route_assertion_outcome() + computes it the same way the other three do, and costs nothing + when it does not match. + WHY status="error" ALSO GETS ITS OWN CHECK For "found"/"none", a wording slip is undesirable but not dangerous -- the underlying fact was still real, the phrasing was just off. For @@ -325,10 +339,79 @@ def _compute_expected_actual_outcome(detail: str) -> Optional[str]: # ------------------------------------------------------------------------------ -# 1d. Deterministic remediation text (#221) +# 1d. Deterministic route-assertion outcome computation (#395, US-62) +# ------------------------------------------------------------------------------ + +# Matches evidence.detail EXACTLY as produced by +# analysis.checks.routing._evaluate() (via run()'s route-statement loop): +# "Expected REACHABLE for a flow from to , but traceroute ended +# in . Path: " +# "Expected UNREACHABLE for a flow from to , but traceroute +# unexpectedly succeeded: . Path: " +# Deliberately narrow, same discipline as the three patterns above -- this +# must only match the two shapes it is confident about, never a loose +# approximation of them. +_ROUTE_ASSERTION_DETAIL_PATTERN = re.compile( + r"Expected (?PREACHABLE|UNREACHABLE) for a flow from " + r"(?P\S+) to (?P\S+), but " + r"(?:traceroute ended in (?P\w+)" + r"|traceroute unexpectedly succeeded: (?P\w+))" +) + + +def _compute_route_assertion_outcome(detail: str) -> Optional[str]: + """For a routing route-assertion finding, state -- in unambiguous + words -- what actually happened versus what was expected, rather than + asking the model to keep the two straight itself. + + WHY THIS EXISTS (#395) + The identical "which of two opposite states is which" attribution + task #52 (dead rules) and #145 (policy_compliance) already got + backwards on real findings, and access_control's own + `_compute_expected_actual_outcome()` was added on the same + structural risk before a third accident. `routing.py`'s route + assertions use the same two-sided phrasing ("Expected X ... but + Y") the other guarded shapes needed a fix for -- a route meant to + be reachable that is not, or a route meant to be blocked that + gets through -- so a model restating it could plausibly invert + which one is true. Live-tested against the local model before + adding this; see the PR this shipped in for the result, recorded + honestly whether or not an inversion reproduced on that occasion, + the same bar `_compute_expected_actual_outcome()` was added at. + + Returns a plain-English statement naming both sides explicitly, or + None if `detail` is not in one of the two exact shapes + `routing._evaluate()` produces -- nothing to compute from, so the + model gets no extra help and reasons from the raw finding alone, same + as any other finding shape. + """ + match = _ROUTE_ASSERTION_DETAIL_PATTERN.search(detail) + if not match: + return None + + expected = match.group("expected") + if expected == "REACHABLE": + disposition = match.group("failed_disposition") + return ( + f"this flow was expected to be REACHABLE, but it is not -- " + f"the actual result is {disposition}. The route or filter " + f"configuration is what is blocking it, not a requirement " + f"that says it should be." + ) + disposition = match.group("succeeded_disposition") + return ( + f"this flow was expected to be UNREACHABLE, but it got through -- " + f"the actual result is {disposition}. The route or filter " + f"configuration is what is letting it through, despite the " + f"requirement that it should not." + ) + + +# ------------------------------------------------------------------------------ +# 1e. Deterministic remediation text (#221) # -# A DIFFERENT QUESTION FROM SECTIONS 1a-1c ABOVE -# Those three compute what a finding MEANS, to correct the model's +# A DIFFERENT QUESTION FROM SECTIONS 1a-1d ABOVE +# Those four compute what a finding MEANS, to correct the model's # wording before it speaks. This computes what would FIX it, and never # goes near the model at all -- ai/Modelfile rule 6 explicitly forbids # recommending a fix "unless the evidence itself states what would @@ -624,17 +707,18 @@ def _build_prompt(finding: Dict[str, Any]) -> str: example's answer almost verbatim for an unrelated finding, rather than reasoning about the finding it was actually given. - 2. If one of the three deterministic outcome functions can compute the + 2. If one of the four deterministic outcome functions can compute the real answer from evidence.detail -- _compute_dead_rule_outcome() for a dead-ACL-rule finding, _compute_policy_outcome() for a - policy_compliance finding (#145), or _compute_expected_actual_outcome() - for an access_control policy-statement finding -- that computed fact, - labelled as already-verified. See each function's docstring for why - it exists and how confident it has to be before it says anything at - all. The three never more than one matches at once: each check - produces its own exact evidence.detail shape, so trying the next one - only when an earlier one returns None does not risk masking one with - another. + policy_compliance finding (#145), _compute_expected_actual_outcome() + for an access_control policy-statement finding, or + _compute_route_assertion_outcome() for a routing route-assertion + finding (#395) -- that computed fact, labelled as already-verified. + See each function's docstring for why it exists and how confident it + has to be before it says anything at all. The four never more than + one matches at once: each check produces its own exact + evidence.detail shape, so trying the next one only when an earlier + one returns None does not risk masking one with another. """ parts = [ "Explain ONLY the finding below. Do not reuse any wording from the " @@ -648,6 +732,7 @@ def _build_prompt(finding: Dict[str, Any]) -> str: _compute_dead_rule_outcome(detail) or _compute_policy_outcome(detail) or _compute_expected_actual_outcome(detail) + or _compute_route_assertion_outcome(detail) ) if computed_outcome is not None: # Deliberately NOT a distinctive, quotable label like "IMPORTANT diff --git a/tests/test_explain_safety_net.py b/tests/test_explain_safety_net.py index bf99281..c20b613 100644 --- a/tests/test_explain_safety_net.py +++ b/tests/test_explain_safety_net.py @@ -28,6 +28,7 @@ _compute_dead_rule_outcome, _compute_expected_actual_outcome, _compute_policy_outcome, + _compute_route_assertion_outcome, _evidence_detail, _fallback_error_explanation, _fallback_plain_restatement, @@ -326,6 +327,98 @@ def test_expected_actual_outcome_returns_none_for_a_policy_compliance_finding(): assert _compute_policy_outcome(detail) is not None # the OTHER guard still matches +# --- _compute_route_assertion_outcome (routing) ----------------------------- +# Same structural reasoning as the three guards above, a fourth producer of +# the identical "which of two opposite states is which" shape: a route +# assertion names what was expected and what actually happened, on real +# routing.py route-assertion findings. Live-tested six times against a real +# finding (routing-missing-route, RT-001) before adding the guard -- see the +# PR this shipped in for the recorded result. + + +def _real_route_detail(expected: str, trace): + """The genuine evidence string, from the genuine function -- same + discipline _real_detail() already uses for policy_compliance, so a + future rewording of routing._evaluate()'s text fails this test loudly + instead of leaving it green while it tests nothing (#145/#169).""" + from analysis.checks.routing import _evaluate + + class FakeHop: + def __init__(self, node): + self.node = node + + class FakeTrace: + def __init__(self, disposition, hops=None): + self.disposition = disposition + self.hops = hops or [FakeHop("some-node")] + + violation = _evaluate(expected, [FakeTrace(trace)]) + assert violation is not None, "test setup error: this trace should violate" + return f"Expected {expected} for a flow from 10.10.10.5 to 10.20.20.5, but {violation}" + + +def test_a_required_reachable_route_that_is_actually_blocked(): + """Real evidence.detail shape from routing._evaluate()'s REACHABLE + branch -- a flow that was supposed to get through did not.""" + detail = _real_route_detail("REACHABLE", "NO_ROUTE") + outcome = _compute_route_assertion_outcome(detail) + assert outcome is not None + assert "REACHABLE" in outcome + assert "NO_ROUTE" in outcome + + +def test_a_required_unreachable_route_that_actually_got_through(): + """The other direction -- a flow that was supposed to be blocked + succeeded instead.""" + detail = _real_route_detail("UNREACHABLE", "ACCEPTED") + outcome = _compute_route_assertion_outcome(detail) + assert outcome is not None + assert "UNREACHABLE" in outcome + assert "ACCEPTED" in outcome + + +def test_route_assertion_outcome_returns_none_for_an_unrelated_finding(): + detail = "BatfishException: Work terminated abnormally" + assert _compute_route_assertion_outcome(detail) is None + + +def test_route_assertion_outcome_returns_none_for_a_dead_rule_finding(): + """The four computations must not cross-match each other's shape.""" + detail = ( + "Unreachable line: permit icmp any any (action PERMIT). " + "Blocked by: deny ip any any. Reason: BLOCKING_LINES" + ) + assert _compute_route_assertion_outcome(detail) is None + + +def test_route_assertion_outcome_returns_none_for_an_expected_actual_finding(): + """Against the real producer, not a copy of its wording -- same + #145/#169 discipline as the cross-match test above it.""" + detail = "Expected PERMIT but got DENY, decided by: deny ip any any" + assert _compute_route_assertion_outcome(detail) is None + assert _compute_expected_actual_outcome(detail) is not None # the OTHER guard still matches + + +def test_route_assertion_outcome_reaches_the_generated_prompt(): + """The wiring, not just the function -- a computed fact that exists but + is never passed to _build_prompt() protects nothing. Matches the + "wiring, not just the function" discipline this project's other + computed-fact and discard-function tests already use.""" + detail = _real_route_detail("REACHABLE", "NO_ROUTE") + finding = { + "id": "RT-001", + "check": "routing", + "severity": "high", + "device": "rtr-hq", + "summary": "A route assertion does not hold", + "evidence": {"detail": detail, "source": "traceroute"}, + "status": "found", + } + prompt = _build_prompt(finding) + assert "NO_ROUTE" in prompt + assert "already verified by code and certain" in prompt + + # --- _fallback_plain_restatement -----------------------------------------------