diff --git a/ai/explain.py b/ai/explain.py index 3043a96..3c34bc2 100644 --- a/ai/explain.py +++ b/ai/explain.py @@ -348,6 +348,54 @@ def _compute_expected_actual_outcome(detail: str) -> Optional[str]: ) +# Same disposition sets analysis/checks/routing.py itself works with +# (SUCCESS_DISPOSITIONS there is the complement of these two plus the +# genuinely-routing ones below), re-declared here rather than imported -- +# same reason ai/query.py's own _SUCCESS_DISPOSITIONS is re-declared +# rather than imported: ai/ does not depend on a specific check module +# for a general Batfish-interpretation fact. +# +# Measured (#439, #399 follow-up): the flat branches below said "the +# network's current routing is what disagrees" for EVERY disposition, +# including DENIED_IN/DENIED_OUT -- which routing.py's own module +# docstring says are filter decisions, not routing -- and EXITS_NETWORK, +# which routing.py's own module docstring says most often means the +# destination device is missing from the snapshot (measured there by +# deleting a config and re-running). Everything else this project's +# disposition vocabulary contains (NO_ROUTE, NULL_ROUTED, LOOP, +# INSUFFICIENT_INFO, NEIGHBOR_UNREACHABLE) really is a routing/forwarding +# outcome, so the "routing" wording stays for that remaining case -- not +# a guessed default, since these two sets are the only non-routing causes +# this project has observed or documented. +_FILTER_DISPOSITIONS = {"DENIED_IN", "DENIED_OUT"} +_MISSING_DEVICE_DISPOSITIONS = {"EXITS_NETWORK"} + + +def _routing_cause_clause(disposition: str) -> str: + """What actually disagrees with the requirement, given the real + disposition Batfish reported for this flow -- see the module-level + comment above _FILTER_DISPOSITIONS for why this is not always + "routing". Always ends mid-sentence, with no trailing period, so the + two call sites below can each finish the sentence their own way.""" + if disposition in _FILTER_DISPOSITIONS: + return ( + "a filter rule somewhere along the path is what disagrees " + "with that requirement, not the requirement itself" + ) + if disposition in _MISSING_DEVICE_DISPOSITIONS: + return ( + f"traceroute reports {disposition} partway along the path, " + "which this project has measured most often means the " + "destination device named in this statement is not in this " + "snapshot -- not that the network's routing disagrees with " + "the requirement" + ) + return ( + "the network's current routing is what disagrees with that " + "requirement, not the requirement itself" + ) + + def _compute_routing_outcome(detail: str) -> Optional[str]: """For a routing-assertion finding, state -- in unambiguous words -- which direction the flow runs and what actually happens to it, rather @@ -395,6 +443,16 @@ def _compute_routing_outcome(detail: str) -> Optional[str]: split in its own words, the exact risk this function family exists to remove. When n == m every path agrees, so the flat statement is still exactly accurate and unchanged. + + WHAT THIS STILL DOES NOT COVER (#439) + The cause clause below is only as good as the ONE disposition + routing._evaluate() names (`failures[0]`/`successes[0]`). If a + unanimous multi-path result (n == m, so not "mixed" in this + function's sense) is unanimous only in pass/fail, not in WHY -- + say, one path DENIED_IN and another NO_ROUTE -- this still + categorizes only the first one. That is a limit of what + routing.py itself reports, not something this function can fix + from the one disposition string it is given. """ match = _ROUTING_ASSERTION_DETAIL_PATTERN.search(detail) if not match: @@ -411,37 +469,35 @@ def _compute_routing_outcome(detail: str) -> Optional[str]: return ( f"{n} of {m} traced paths for the flow FROM {src} TO {dst} " f"-- that direction, not the reverse -- do NOT reach {dst} " - f"(one ended in {disposition}); the rest do. That is a " - f"real, mixed result, not a single yes or no -- state both " - f"parts, do not round it to one. The statement requires " - f"this flow to be {expected}; whether that holds depends on " - f"which path is taken." + f"(at least one of them ended in {disposition}); the rest " + f"do. That is a real, mixed result, not a single yes or no " + f"-- state both parts, do not round it to one. The " + f"statement requires this flow to be {expected}; whether " + f"that holds depends on which path is taken." ) return ( f"the flow FROM {src} TO {dst} -- that direction, not the " f"reverse -- currently does NOT reach {dst}; the trace ended in " f"{disposition}. That is the real, observed outcome. The " - f"statement requires this flow to be {expected}; the network's " - f"current routing is what disagrees with that requirement, not " - f"the requirement itself." + f"statement requires this flow to be {expected}; " + f"{_routing_cause_clause(disposition)}." ) disposition = match.group("succeeded_disposition") if mixed: return ( f"{n} of {m} traced paths for the flow FROM {src} TO {dst} -- " - f"that direction, not the reverse -- DO reach {dst} (one " - f"succeeded as {disposition}); the rest do not. That is a " - f"real, mixed result, not a single yes or no -- state both " - f"parts, do not round it to one. The statement requires this " - f"flow to be {expected}; whether that holds depends on which " - f"path is taken." + f"that direction, not the reverse -- DO reach {dst} (at least " + f"one of them succeeded as {disposition}); the rest do not. " + f"That is a real, mixed result, not a single yes or no -- " + f"state both parts, do not round it to one. The statement " + f"requires this flow to be {expected}; whether that holds " + f"depends on which path is taken." ) return ( f"the flow FROM {src} TO {dst} -- that direction, not the " f"reverse -- currently DOES reach {dst} ({disposition}). That is " f"the real, observed outcome. The statement requires this flow to " - f"be {expected}; the network's current routing is what disagrees " - f"with that requirement, not the requirement itself." + f"be {expected}; {_routing_cause_clause(disposition)}." ) diff --git a/analysis/checks/routing.py b/analysis/checks/routing.py index 8350e18..74948a3 100644 --- a/analysis/checks/routing.py +++ b/analysis/checks/routing.py @@ -301,6 +301,15 @@ def _evaluate(expected: str, traces: Sequence[Any]) -> Optional[str]: raise ValueError(f"route 'expected' must be REACHABLE or UNREACHABLE, got {expected!r}") +def _violation_detail(expected: str, src_ip: str, dst_ip: str, violation: str) -> str: + """The exact evidence.detail shape run() has always produced for a + routing-assertion finding. Pulled out to its own function (#439) so + tests can call the real producer instead of retyping this string by + hand -- the same reason _evaluate() itself is tested directly rather + than through a copy of its output.""" + return f"Expected {expected} for a flow from {src_ip} to {dst_ip}, but {violation}" + + def run(bf: Session) -> List[Dict[str, Any]]: """Check every route statement and return F-1 findings. @@ -481,9 +490,8 @@ def run(bf: Session) -> List[Dict[str, Any]]: # actually reached and the path it took to get there. We # never paraphrase it here; the AI layer does that, grounded # in this string. - detail=( - f"Expected {route['expected']} for a flow from " - f"{route['src_ip']} to {route['dst_ip']}, but {violation}" + detail=_violation_detail( + route["expected"], route["src_ip"], route["dst_ip"], violation ), source=f"{node}:{route['dst_ip']}", status="found", diff --git a/tests/test_explain_safety_net.py b/tests/test_explain_safety_net.py index 0037ce5..5050292 100644 --- a/tests/test_explain_safety_net.py +++ b/tests/test_explain_safety_net.py @@ -354,7 +354,7 @@ def _real_routing_detail_multi(expected: str, dispositions, src="10.10.10.5", exercises routing._evaluate()'s "(n of m paths failed/succeeded)" suffix against the real function, for the mixed-result case (#399 review, @shubhamkataria2005).""" - from analysis.checks.routing import _evaluate + from analysis.checks.routing import _evaluate, _violation_detail class _FakeHop: def __init__(self, node): @@ -367,7 +367,9 @@ def __init__(self, disposition): violation = _evaluate(expected, [_FakeTrace(d) for d in dispositions]) assert violation is not None, "fixture dispositions did not violate the assertion" - return f"Expected {expected} for a flow from {src} to {dst}, but {violation}" + # Through the real producer (#439), not a hand-typed copy of its + # prefix -- same reasoning as calling _evaluate() itself above. + return _violation_detail(expected, src, dst, violation) def test_a_required_reachable_flow_that_is_actually_blocked(): @@ -420,6 +422,11 @@ def test_a_mixed_reachable_result_states_the_split_not_a_flat_claim(): assert "do NOT reach" in outcome assert "mixed" in outcome assert "does NOT reach" not in outcome, "must not also state the flat claim" + assert "at least one of them ended in NO_ROUTE" in outcome, ( + "#439: 'one ended in X' implies only one path failed, when the " + "count already says otherwise -- must read as one OF the failures, " + "not THE failure" + ) def test_a_mixed_unreachable_result_states_the_split_not_a_flat_claim(): @@ -432,6 +439,9 @@ def test_a_mixed_unreachable_result_states_the_split_not_a_flat_claim(): assert "DO reach" in outcome assert "mixed" in outcome assert "DOES reach" not in outcome, "must not also state the flat claim" + assert "at least one of them succeeded as ACCEPTED" in outcome, ( + "#439: same wording fix, mirrored for the succeeded-disposition case" + ) def test_a_unanimous_multi_path_result_still_states_the_flat_claim(): @@ -446,6 +456,54 @@ def test_a_unanimous_multi_path_result_still_states_the_flat_claim(): assert "of 2" not in outcome +# --- #439: the cause named must match the real disposition, not default to +# "routing" for every one ------------------------------------------------- +# +# Measured on main (ccb2591) and confirmed still present (d424e05): both flat +# branches said "the network's current routing is what disagrees" no matter +# what Batfish actually reported. DENIED_IN/DENIED_OUT are filter decisions +# (routing.py's own module docstring); EXITS_NETWORK most often means the +# destination device is missing from the snapshot, not a routing conflict. + +def test_a_filter_denial_is_not_blamed_on_routing(): + detail = _real_routing_detail("REACHABLE", "DENIED_IN") + outcome = _compute_routing_outcome(detail) + assert outcome is not None + assert "a filter rule" in outcome + assert "current routing is what disagrees" not in outcome + + +def test_the_other_filter_disposition_is_also_not_blamed_on_routing(): + """DENIED_OUT, like DENIED_IN, is excluded from SUCCESS_DISPOSITIONS + (routing.py:158), so it can only ever appear as a failure of a + REACHABLE assertion -- never as the "unexpectedly succeeded" + disposition of an UNREACHABLE one.""" + detail = _real_routing_detail("REACHABLE", "DENIED_OUT") + outcome = _compute_routing_outcome(detail) + assert outcome is not None + assert "a filter rule" in outcome + assert "current routing is what disagrees" not in outcome + + +def test_exits_network_names_a_missing_device_not_routing(): + detail = _real_routing_detail("REACHABLE", "EXITS_NETWORK") + outcome = _compute_routing_outcome(detail) + assert outcome is not None + assert "not in this snapshot" in outcome + assert "current routing is what disagrees" not in outcome + + +def test_a_genuine_routing_disposition_still_blames_routing(): + """The control: NO_ROUTE really is a routing outcome, so the existing + wording must still appear -- without this, a fix that removed the + causal claim entirely would make every test above pass for the wrong + reason.""" + detail = _real_routing_detail("REACHABLE", "NO_ROUTE") + outcome = _compute_routing_outcome(detail) + assert outcome is not None + assert "current routing is what disagrees" in outcome + + def test_routing_outcome_returns_none_for_an_unrelated_finding(): detail = "BatfishException: Work terminated abnormally" assert _compute_routing_outcome(detail) is None diff --git a/tests/test_routing_classification.py b/tests/test_routing_classification.py index e09b23c..0624f1b 100644 --- a/tests/test_routing_classification.py +++ b/tests/test_routing_classification.py @@ -30,7 +30,7 @@ import pytest -from analysis.checks.routing import _evaluate +from analysis.checks.routing import _evaluate, _violation_detail class FakeHop: @@ -128,6 +128,24 @@ def test_exits_network_does_not_satisfy_unreachable_either(): assert _evaluate("UNREACHABLE", [FakeTrace("EXITS_NETWORK")]) is None +# --- _violation_detail (#439) --------------------------------------------------- +# run() builds a finding's evidence.detail by calling this function with +# _evaluate()'s own output. Tested directly so tests/test_explain_safety_ +# net.py's routing fixtures, which call this same function to build a +# realistic detail string, fail loudly if this format ever changes -- +# rather than silently testing a hand-typed copy that no longer matches +# what run() actually emits. + + +def test_violation_detail_matches_what_run_actually_builds(): + violation = _evaluate("REACHABLE", [FakeTrace("NO_ROUTE")]) + detail = _violation_detail("REACHABLE", "10.10.10.5", "10.20.20.5", violation) + assert detail == ( + "Expected REACHABLE for a flow from 10.10.10.5 to 10.20.20.5, " + f"but {violation}" + ) + + # --- Unknown expected value ----------------------------------------------------