From 781ba8e668104f16dc0523be9d3c8fbe3de5d008 Mon Sep 17 00:00:00 2001 From: patelankeet2 Date: Sun, 8 Nov 2026 18:48:15 +1300 Subject: [PATCH 1/3] Extract the routing-violation detail string into _violation_detail (#439) Pure refactor, no behavior change. run() built the finding's detail string inline, so tests/test_explain_safety_net.py's _real_routing_detail_multi() had to retype the same literal prefix by hand to construct a realistic detail for testing ai/explain.py against. A reword of the real prefix could silently stop matching what that hand-copy produces. Pulled the prefix into its own function so the test can call the real producer instead -- same discipline the file already applies to _evaluate() one line above, rather than testing a copy of its output. --- analysis/checks/routing.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/analysis/checks/routing.py b/analysis/checks/routing.py index 8350e18..74948a3 100644 --- a/analysis/checks/routing.py +++ b/analysis/checks/routing.py @@ -301,6 +301,15 @@ def _evaluate(expected: str, traces: Sequence[Any]) -> Optional[str]: raise ValueError(f"route 'expected' must be REACHABLE or UNREACHABLE, got {expected!r}") +def _violation_detail(expected: str, src_ip: str, dst_ip: str, violation: str) -> str: + """The exact evidence.detail shape run() has always produced for a + routing-assertion finding. Pulled out to its own function (#439) so + tests can call the real producer instead of retyping this string by + hand -- the same reason _evaluate() itself is tested directly rather + than through a copy of its output.""" + return f"Expected {expected} for a flow from {src_ip} to {dst_ip}, but {violation}" + + def run(bf: Session) -> List[Dict[str, Any]]: """Check every route statement and return F-1 findings. @@ -481,9 +490,8 @@ def run(bf: Session) -> List[Dict[str, Any]]: # actually reached and the path it took to get there. We # never paraphrase it here; the AI layer does that, grounded # in this string. - detail=( - f"Expected {route['expected']} for a flow from " - f"{route['src_ip']} to {route['dst_ip']}, but {violation}" + detail=_violation_detail( + route["expected"], route["src_ip"], route["dst_ip"], violation ), source=f"{node}:{route['dst_ip']}", status="found", From c33ce49faea8654e6db6e2c40522503ed113e174 Mon Sep 17 00:00:00 2001 From: patelankeet2 Date: Sun, 8 Nov 2026 19:00:15 +1300 Subject: [PATCH 2/3] Name the real cause for a routing-assertion violation, not always routing (#439) _compute_routing_outcome() told the model, as an already verified fact, that 'the network's current routing is what disagrees' for every failed or unexpectedly-succeeded routing assertion. Measured still present on main (d424e05): analysis/checks/routing.py's own module docstring says DENIED_IN/DENIED_OUT are filter decisions, not routing, and EXITS_NETWORK most often means the destination device is missing from the snapshot (measured there by deleting a config and re-running). Both were blamed on routing anyway. Adds two small disposition sets (filter-caused, missing-device-caused), declared locally in ai/explain.py rather than imported from routing.py -- same boundary ai/query.py's own _SUCCESS_DISPOSITIONS already holds, and for the same documented reason. Everything else in this project's known disposition vocabulary (NO_ROUTE, NULL_ROUTED, LOOP, INSUFFICIENT_INFO, NEIGHBOR_UNREACHABLE) really is a routing outcome, so that wording is unchanged for the remaining case. Also fixes the mixed-result wording '(one ended in X)', which names only the first of possibly several failing dispositions and so can read as if only one path failed when the count already says otherwise. Reworded to '(at least one of them ended in X)'. Scope limit, documented in the function's own docstring rather than left implicit: routing._evaluate() itself only ever names ONE representative disposition even in a unanimous multi-path failure, so a finding where different paths fail for genuinely different reasons is still categorized by only the first. That is a limit of what routing.py reports, not something this fix can see past. Tests: new cases for both disposition sets, a control pinning that a genuine routing disposition (NO_ROUTE) still gets the original wording, and updated assertions for the reworded mixed-result phrase. Mutation-tested: removing the disposition branching breaks all 3 new cause-attribution tests for the right reason (fall back to the old blanket 'routing' wording), while the control still passes; restored. Full suite: 2230 passed, 46 skipped, 0 failed. Live-tested against the real model: 10 generations each for a DENIED_IN, an EXITS_NETWORK, and a NO_ROUTE finding. 0/30 fallbacks, and every generation correctly followed the new disposition-specific fact -- filter language for DENIED_IN, destination-device language for EXITS_NETWORK, routing language only for NO_ROUTE. --- ai/explain.py | 88 ++++++++++++++++++++++++++------ tests/test_explain_safety_net.py | 62 +++++++++++++++++++++- 2 files changed, 132 insertions(+), 18 deletions(-) diff --git a/ai/explain.py b/ai/explain.py index 3043a96..3c34bc2 100644 --- a/ai/explain.py +++ b/ai/explain.py @@ -348,6 +348,54 @@ def _compute_expected_actual_outcome(detail: str) -> Optional[str]: ) +# Same disposition sets analysis/checks/routing.py itself works with +# (SUCCESS_DISPOSITIONS there is the complement of these two plus the +# genuinely-routing ones below), re-declared here rather than imported -- +# same reason ai/query.py's own _SUCCESS_DISPOSITIONS is re-declared +# rather than imported: ai/ does not depend on a specific check module +# for a general Batfish-interpretation fact. +# +# Measured (#439, #399 follow-up): the flat branches below said "the +# network's current routing is what disagrees" for EVERY disposition, +# including DENIED_IN/DENIED_OUT -- which routing.py's own module +# docstring says are filter decisions, not routing -- and EXITS_NETWORK, +# which routing.py's own module docstring says most often means the +# destination device is missing from the snapshot (measured there by +# deleting a config and re-running). Everything else this project's +# disposition vocabulary contains (NO_ROUTE, NULL_ROUTED, LOOP, +# INSUFFICIENT_INFO, NEIGHBOR_UNREACHABLE) really is a routing/forwarding +# outcome, so the "routing" wording stays for that remaining case -- not +# a guessed default, since these two sets are the only non-routing causes +# this project has observed or documented. +_FILTER_DISPOSITIONS = {"DENIED_IN", "DENIED_OUT"} +_MISSING_DEVICE_DISPOSITIONS = {"EXITS_NETWORK"} + + +def _routing_cause_clause(disposition: str) -> str: + """What actually disagrees with the requirement, given the real + disposition Batfish reported for this flow -- see the module-level + comment above _FILTER_DISPOSITIONS for why this is not always + "routing". Always ends mid-sentence, with no trailing period, so the + two call sites below can each finish the sentence their own way.""" + if disposition in _FILTER_DISPOSITIONS: + return ( + "a filter rule somewhere along the path is what disagrees " + "with that requirement, not the requirement itself" + ) + if disposition in _MISSING_DEVICE_DISPOSITIONS: + return ( + f"traceroute reports {disposition} partway along the path, " + "which this project has measured most often means the " + "destination device named in this statement is not in this " + "snapshot -- not that the network's routing disagrees with " + "the requirement" + ) + return ( + "the network's current routing is what disagrees with that " + "requirement, not the requirement itself" + ) + + def _compute_routing_outcome(detail: str) -> Optional[str]: """For a routing-assertion finding, state -- in unambiguous words -- which direction the flow runs and what actually happens to it, rather @@ -395,6 +443,16 @@ def _compute_routing_outcome(detail: str) -> Optional[str]: split in its own words, the exact risk this function family exists to remove. When n == m every path agrees, so the flat statement is still exactly accurate and unchanged. + + WHAT THIS STILL DOES NOT COVER (#439) + The cause clause below is only as good as the ONE disposition + routing._evaluate() names (`failures[0]`/`successes[0]`). If a + unanimous multi-path result (n == m, so not "mixed" in this + function's sense) is unanimous only in pass/fail, not in WHY -- + say, one path DENIED_IN and another NO_ROUTE -- this still + categorizes only the first one. That is a limit of what + routing.py itself reports, not something this function can fix + from the one disposition string it is given. """ match = _ROUTING_ASSERTION_DETAIL_PATTERN.search(detail) if not match: @@ -411,37 +469,35 @@ def _compute_routing_outcome(detail: str) -> Optional[str]: return ( f"{n} of {m} traced paths for the flow FROM {src} TO {dst} " f"-- that direction, not the reverse -- do NOT reach {dst} " - f"(one ended in {disposition}); the rest do. That is a " - f"real, mixed result, not a single yes or no -- state both " - f"parts, do not round it to one. The statement requires " - f"this flow to be {expected}; whether that holds depends on " - f"which path is taken." + f"(at least one of them ended in {disposition}); the rest " + f"do. That is a real, mixed result, not a single yes or no " + f"-- state both parts, do not round it to one. The " + f"statement requires this flow to be {expected}; whether " + f"that holds depends on which path is taken." ) return ( f"the flow FROM {src} TO {dst} -- that direction, not the " f"reverse -- currently does NOT reach {dst}; the trace ended in " f"{disposition}. That is the real, observed outcome. The " - f"statement requires this flow to be {expected}; the network's " - f"current routing is what disagrees with that requirement, not " - f"the requirement itself." + f"statement requires this flow to be {expected}; " + f"{_routing_cause_clause(disposition)}." ) disposition = match.group("succeeded_disposition") if mixed: return ( f"{n} of {m} traced paths for the flow FROM {src} TO {dst} -- " - f"that direction, not the reverse -- DO reach {dst} (one " - f"succeeded as {disposition}); the rest do not. That is a " - f"real, mixed result, not a single yes or no -- state both " - f"parts, do not round it to one. The statement requires this " - f"flow to be {expected}; whether that holds depends on which " - f"path is taken." + f"that direction, not the reverse -- DO reach {dst} (at least " + f"one of them succeeded as {disposition}); the rest do not. " + f"That is a real, mixed result, not a single yes or no -- " + f"state both parts, do not round it to one. The statement " + f"requires this flow to be {expected}; whether that holds " + f"depends on which path is taken." ) return ( f"the flow FROM {src} TO {dst} -- that direction, not the " f"reverse -- currently DOES reach {dst} ({disposition}). That is " f"the real, observed outcome. The statement requires this flow to " - f"be {expected}; the network's current routing is what disagrees " - f"with that requirement, not the requirement itself." + f"be {expected}; {_routing_cause_clause(disposition)}." ) diff --git a/tests/test_explain_safety_net.py b/tests/test_explain_safety_net.py index 0037ce5..5050292 100644 --- a/tests/test_explain_safety_net.py +++ b/tests/test_explain_safety_net.py @@ -354,7 +354,7 @@ def _real_routing_detail_multi(expected: str, dispositions, src="10.10.10.5", exercises routing._evaluate()'s "(n of m paths failed/succeeded)" suffix against the real function, for the mixed-result case (#399 review, @shubhamkataria2005).""" - from analysis.checks.routing import _evaluate + from analysis.checks.routing import _evaluate, _violation_detail class _FakeHop: def __init__(self, node): @@ -367,7 +367,9 @@ def __init__(self, disposition): violation = _evaluate(expected, [_FakeTrace(d) for d in dispositions]) assert violation is not None, "fixture dispositions did not violate the assertion" - return f"Expected {expected} for a flow from {src} to {dst}, but {violation}" + # Through the real producer (#439), not a hand-typed copy of its + # prefix -- same reasoning as calling _evaluate() itself above. + return _violation_detail(expected, src, dst, violation) def test_a_required_reachable_flow_that_is_actually_blocked(): @@ -420,6 +422,11 @@ def test_a_mixed_reachable_result_states_the_split_not_a_flat_claim(): assert "do NOT reach" in outcome assert "mixed" in outcome assert "does NOT reach" not in outcome, "must not also state the flat claim" + assert "at least one of them ended in NO_ROUTE" in outcome, ( + "#439: 'one ended in X' implies only one path failed, when the " + "count already says otherwise -- must read as one OF the failures, " + "not THE failure" + ) def test_a_mixed_unreachable_result_states_the_split_not_a_flat_claim(): @@ -432,6 +439,9 @@ def test_a_mixed_unreachable_result_states_the_split_not_a_flat_claim(): assert "DO reach" in outcome assert "mixed" in outcome assert "DOES reach" not in outcome, "must not also state the flat claim" + assert "at least one of them succeeded as ACCEPTED" in outcome, ( + "#439: same wording fix, mirrored for the succeeded-disposition case" + ) def test_a_unanimous_multi_path_result_still_states_the_flat_claim(): @@ -446,6 +456,54 @@ def test_a_unanimous_multi_path_result_still_states_the_flat_claim(): assert "of 2" not in outcome +# --- #439: the cause named must match the real disposition, not default to +# "routing" for every one ------------------------------------------------- +# +# Measured on main (ccb2591) and confirmed still present (d424e05): both flat +# branches said "the network's current routing is what disagrees" no matter +# what Batfish actually reported. DENIED_IN/DENIED_OUT are filter decisions +# (routing.py's own module docstring); EXITS_NETWORK most often means the +# destination device is missing from the snapshot, not a routing conflict. + +def test_a_filter_denial_is_not_blamed_on_routing(): + detail = _real_routing_detail("REACHABLE", "DENIED_IN") + outcome = _compute_routing_outcome(detail) + assert outcome is not None + assert "a filter rule" in outcome + assert "current routing is what disagrees" not in outcome + + +def test_the_other_filter_disposition_is_also_not_blamed_on_routing(): + """DENIED_OUT, like DENIED_IN, is excluded from SUCCESS_DISPOSITIONS + (routing.py:158), so it can only ever appear as a failure of a + REACHABLE assertion -- never as the "unexpectedly succeeded" + disposition of an UNREACHABLE one.""" + detail = _real_routing_detail("REACHABLE", "DENIED_OUT") + outcome = _compute_routing_outcome(detail) + assert outcome is not None + assert "a filter rule" in outcome + assert "current routing is what disagrees" not in outcome + + +def test_exits_network_names_a_missing_device_not_routing(): + detail = _real_routing_detail("REACHABLE", "EXITS_NETWORK") + outcome = _compute_routing_outcome(detail) + assert outcome is not None + assert "not in this snapshot" in outcome + assert "current routing is what disagrees" not in outcome + + +def test_a_genuine_routing_disposition_still_blames_routing(): + """The control: NO_ROUTE really is a routing outcome, so the existing + wording must still appear -- without this, a fix that removed the + causal claim entirely would make every test above pass for the wrong + reason.""" + detail = _real_routing_detail("REACHABLE", "NO_ROUTE") + outcome = _compute_routing_outcome(detail) + assert outcome is not None + assert "current routing is what disagrees" in outcome + + def test_routing_outcome_returns_none_for_an_unrelated_finding(): detail = "BatfishException: Work terminated abnormally" assert _compute_routing_outcome(detail) is None From 3f415ef9cfd07462895d299fc3bd1977b24abc83 Mon Sep 17 00:00:00 2001 From: patelankeet2 Date: Sun, 8 Nov 2026 19:01:05 +1300 Subject: [PATCH 3/3] Pin _violation_detail() against the real producer (#439) Direct unit test for the function extracted in 781ba8e, so a future reword of the detail-prefix format is caught here as well as through tests/test_explain_safety_net.py's routing fixtures, which already call the same function. --- tests/test_routing_classification.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/tests/test_routing_classification.py b/tests/test_routing_classification.py index e09b23c..0624f1b 100644 --- a/tests/test_routing_classification.py +++ b/tests/test_routing_classification.py @@ -30,7 +30,7 @@ import pytest -from analysis.checks.routing import _evaluate +from analysis.checks.routing import _evaluate, _violation_detail class FakeHop: @@ -128,6 +128,24 @@ def test_exits_network_does_not_satisfy_unreachable_either(): assert _evaluate("UNREACHABLE", [FakeTrace("EXITS_NETWORK")]) is None +# --- _violation_detail (#439) --------------------------------------------------- +# run() builds a finding's evidence.detail by calling this function with +# _evaluate()'s own output. Tested directly so tests/test_explain_safety_ +# net.py's routing fixtures, which call this same function to build a +# realistic detail string, fail loudly if this format ever changes -- +# rather than silently testing a hand-typed copy that no longer matches +# what run() actually emits. + + +def test_violation_detail_matches_what_run_actually_builds(): + violation = _evaluate("REACHABLE", [FakeTrace("NO_ROUTE")]) + detail = _violation_detail("REACHABLE", "10.10.10.5", "10.20.20.5", violation) + assert detail == ( + "Expected REACHABLE for a flow from 10.10.10.5 to 10.20.20.5, " + f"but {violation}" + ) + + # --- Unknown expected value ----------------------------------------------------