Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
88 changes: 72 additions & 16 deletions ai/explain.py
Original file line number Diff line number Diff line change
Expand Up @@ -348,6 +348,54 @@ def _compute_expected_actual_outcome(detail: str) -> Optional[str]:
)


# Same disposition sets analysis/checks/routing.py itself works with
# (SUCCESS_DISPOSITIONS there is the complement of these two plus the
# genuinely-routing ones below), re-declared here rather than imported --
# same reason ai/query.py's own _SUCCESS_DISPOSITIONS is re-declared
# rather than imported: ai/ does not depend on a specific check module
# for a general Batfish-interpretation fact.
#
# Measured (#439, #399 follow-up): the flat branches below said "the
# network's current routing is what disagrees" for EVERY disposition,
# including DENIED_IN/DENIED_OUT -- which routing.py's own module
# docstring says are filter decisions, not routing -- and EXITS_NETWORK,
# which routing.py's own module docstring says most often means the
# destination device is missing from the snapshot (measured there by
# deleting a config and re-running). Everything else this project's
# disposition vocabulary contains (NO_ROUTE, NULL_ROUTED, LOOP,
# INSUFFICIENT_INFO, NEIGHBOR_UNREACHABLE) really is a routing/forwarding
# outcome, so the "routing" wording stays for that remaining case -- not
# a guessed default, since these two sets are the only non-routing causes
# this project has observed or documented.
_FILTER_DISPOSITIONS = {"DENIED_IN", "DENIED_OUT"}
_MISSING_DEVICE_DISPOSITIONS = {"EXITS_NETWORK"}


def _routing_cause_clause(disposition: str) -> str:
"""What actually disagrees with the requirement, given the real
disposition Batfish reported for this flow -- see the module-level
comment above _FILTER_DISPOSITIONS for why this is not always
"routing". Always ends mid-sentence, with no trailing period, so the
two call sites below can each finish the sentence their own way."""
if disposition in _FILTER_DISPOSITIONS:
return (
"a filter rule somewhere along the path is what disagrees "
"with that requirement, not the requirement itself"
)
if disposition in _MISSING_DEVICE_DISPOSITIONS:
return (
f"traceroute reports {disposition} partway along the path, "
"which this project has measured most often means the "
"destination device named in this statement is not in this "
"snapshot -- not that the network's routing disagrees with "
"the requirement"
)
return (
"the network's current routing is what disagrees with that "
"requirement, not the requirement itself"
)


def _compute_routing_outcome(detail: str) -> Optional[str]:
"""For a routing-assertion finding, state -- in unambiguous words --
which direction the flow runs and what actually happens to it, rather
Expand Down Expand Up @@ -395,6 +443,16 @@ def _compute_routing_outcome(detail: str) -> Optional[str]:
split in its own words, the exact risk this function family exists to
remove. When n == m every path agrees, so the flat statement is still
exactly accurate and unchanged.

WHAT THIS STILL DOES NOT COVER (#439)
The cause clause below is only as good as the ONE disposition
routing._evaluate() names (`failures[0]`/`successes[0]`). If a
unanimous multi-path result (n == m, so not "mixed" in this
function's sense) is unanimous only in pass/fail, not in WHY --
say, one path DENIED_IN and another NO_ROUTE -- this still
categorizes only the first one. That is a limit of what
routing.py itself reports, not something this function can fix
from the one disposition string it is given.
"""
match = _ROUTING_ASSERTION_DETAIL_PATTERN.search(detail)
if not match:
Expand All @@ -411,37 +469,35 @@ def _compute_routing_outcome(detail: str) -> Optional[str]:
return (
f"{n} of {m} traced paths for the flow FROM {src} TO {dst} "
f"-- that direction, not the reverse -- do NOT reach {dst} "
f"(one ended in {disposition}); the rest do. That is a "
f"real, mixed result, not a single yes or no -- state both "
f"parts, do not round it to one. The statement requires "
f"this flow to be {expected}; whether that holds depends on "
f"which path is taken."
f"(at least one of them ended in {disposition}); the rest "
f"do. That is a real, mixed result, not a single yes or no "
f"-- state both parts, do not round it to one. The "
f"statement requires this flow to be {expected}; whether "
f"that holds depends on which path is taken."
)
return (
f"the flow FROM {src} TO {dst} -- that direction, not the "
f"reverse -- currently does NOT reach {dst}; the trace ended in "
f"{disposition}. That is the real, observed outcome. The "
f"statement requires this flow to be {expected}; the network's "
f"current routing is what disagrees with that requirement, not "
f"the requirement itself."
f"statement requires this flow to be {expected}; "
f"{_routing_cause_clause(disposition)}."
)
disposition = match.group("succeeded_disposition")
if mixed:
return (
f"{n} of {m} traced paths for the flow FROM {src} TO {dst} -- "
f"that direction, not the reverse -- DO reach {dst} (one "
f"succeeded as {disposition}); the rest do not. That is a "
f"real, mixed result, not a single yes or no -- state both "
f"parts, do not round it to one. The statement requires this "
f"flow to be {expected}; whether that holds depends on which "
f"path is taken."
f"that direction, not the reverse -- DO reach {dst} (at least "
f"one of them succeeded as {disposition}); the rest do not. "
f"That is a real, mixed result, not a single yes or no -- "
f"state both parts, do not round it to one. The statement "
f"requires this flow to be {expected}; whether that holds "
f"depends on which path is taken."
)
return (
f"the flow FROM {src} TO {dst} -- that direction, not the "
f"reverse -- currently DOES reach {dst} ({disposition}). That is "
f"the real, observed outcome. The statement requires this flow to "
f"be {expected}; the network's current routing is what disagrees "
f"with that requirement, not the requirement itself."
f"be {expected}; {_routing_cause_clause(disposition)}."
)


Expand Down
14 changes: 11 additions & 3 deletions analysis/checks/routing.py
Original file line number Diff line number Diff line change
Expand Up @@ -301,6 +301,15 @@ def _evaluate(expected: str, traces: Sequence[Any]) -> Optional[str]:
raise ValueError(f"route 'expected' must be REACHABLE or UNREACHABLE, got {expected!r}")


def _violation_detail(expected: str, src_ip: str, dst_ip: str, violation: str) -> str:
"""The exact evidence.detail shape run() has always produced for a
routing-assertion finding. Pulled out to its own function (#439) so
tests can call the real producer instead of retyping this string by
hand -- the same reason _evaluate() itself is tested directly rather
than through a copy of its output."""
return f"Expected {expected} for a flow from {src_ip} to {dst_ip}, but {violation}"


def run(bf: Session) -> List[Dict[str, Any]]:
"""Check every route statement and return F-1 findings.

Expand Down Expand Up @@ -481,9 +490,8 @@ def run(bf: Session) -> List[Dict[str, Any]]:
# actually reached and the path it took to get there. We
# never paraphrase it here; the AI layer does that, grounded
# in this string.
detail=(
f"Expected {route['expected']} for a flow from "
f"{route['src_ip']} to {route['dst_ip']}, but {violation}"
detail=_violation_detail(
route["expected"], route["src_ip"], route["dst_ip"], violation
),
source=f"{node}:{route['dst_ip']}",
status="found",
Expand Down
62 changes: 60 additions & 2 deletions tests/test_explain_safety_net.py
Original file line number Diff line number Diff line change
Expand Up @@ -354,7 +354,7 @@ def _real_routing_detail_multi(expected: str, dispositions, src="10.10.10.5",
exercises routing._evaluate()'s "(n of m paths failed/succeeded)" suffix
against the real function, for the mixed-result case (#399 review,
@shubhamkataria2005)."""
from analysis.checks.routing import _evaluate
from analysis.checks.routing import _evaluate, _violation_detail

class _FakeHop:
def __init__(self, node):
Expand All @@ -367,7 +367,9 @@ def __init__(self, disposition):

violation = _evaluate(expected, [_FakeTrace(d) for d in dispositions])
assert violation is not None, "fixture dispositions did not violate the assertion"
return f"Expected {expected} for a flow from {src} to {dst}, but {violation}"
# Through the real producer (#439), not a hand-typed copy of its
# prefix -- same reasoning as calling _evaluate() itself above.
return _violation_detail(expected, src, dst, violation)


def test_a_required_reachable_flow_that_is_actually_blocked():
Expand Down Expand Up @@ -420,6 +422,11 @@ def test_a_mixed_reachable_result_states_the_split_not_a_flat_claim():
assert "do NOT reach" in outcome
assert "mixed" in outcome
assert "does NOT reach" not in outcome, "must not also state the flat claim"
assert "at least one of them ended in NO_ROUTE" in outcome, (
"#439: 'one ended in X' implies only one path failed, when the "
"count already says otherwise -- must read as one OF the failures, "
"not THE failure"
)


def test_a_mixed_unreachable_result_states_the_split_not_a_flat_claim():
Expand All @@ -432,6 +439,9 @@ def test_a_mixed_unreachable_result_states_the_split_not_a_flat_claim():
assert "DO reach" in outcome
assert "mixed" in outcome
assert "DOES reach" not in outcome, "must not also state the flat claim"
assert "at least one of them succeeded as ACCEPTED" in outcome, (
"#439: same wording fix, mirrored for the succeeded-disposition case"
)


def test_a_unanimous_multi_path_result_still_states_the_flat_claim():
Expand All @@ -446,6 +456,54 @@ def test_a_unanimous_multi_path_result_still_states_the_flat_claim():
assert "of 2" not in outcome


# --- #439: the cause named must match the real disposition, not default to
# "routing" for every one -------------------------------------------------
#
# Measured on main (ccb2591) and confirmed still present (d424e05): both flat
# branches said "the network's current routing is what disagrees" no matter
# what Batfish actually reported. DENIED_IN/DENIED_OUT are filter decisions
# (routing.py's own module docstring); EXITS_NETWORK most often means the
# destination device is missing from the snapshot, not a routing conflict.

def test_a_filter_denial_is_not_blamed_on_routing():
detail = _real_routing_detail("REACHABLE", "DENIED_IN")
outcome = _compute_routing_outcome(detail)
assert outcome is not None
assert "a filter rule" in outcome
assert "current routing is what disagrees" not in outcome


def test_the_other_filter_disposition_is_also_not_blamed_on_routing():
"""DENIED_OUT, like DENIED_IN, is excluded from SUCCESS_DISPOSITIONS
(routing.py:158), so it can only ever appear as a failure of a
REACHABLE assertion -- never as the "unexpectedly succeeded"
disposition of an UNREACHABLE one."""
detail = _real_routing_detail("REACHABLE", "DENIED_OUT")
outcome = _compute_routing_outcome(detail)
assert outcome is not None
assert "a filter rule" in outcome
assert "current routing is what disagrees" not in outcome


def test_exits_network_names_a_missing_device_not_routing():
detail = _real_routing_detail("REACHABLE", "EXITS_NETWORK")
outcome = _compute_routing_outcome(detail)
assert outcome is not None
assert "not in this snapshot" in outcome
assert "current routing is what disagrees" not in outcome


def test_a_genuine_routing_disposition_still_blames_routing():
"""The control: NO_ROUTE really is a routing outcome, so the existing
wording must still appear -- without this, a fix that removed the
causal claim entirely would make every test above pass for the wrong
reason."""
detail = _real_routing_detail("REACHABLE", "NO_ROUTE")
outcome = _compute_routing_outcome(detail)
assert outcome is not None
assert "current routing is what disagrees" in outcome


def test_routing_outcome_returns_none_for_an_unrelated_finding():
detail = "BatfishException: Work terminated abnormally"
assert _compute_routing_outcome(detail) is None
Expand Down
20 changes: 19 additions & 1 deletion tests/test_routing_classification.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@

import pytest

from analysis.checks.routing import _evaluate
from analysis.checks.routing import _evaluate, _violation_detail


class FakeHop:
Expand Down Expand Up @@ -128,6 +128,24 @@ def test_exits_network_does_not_satisfy_unreachable_either():
assert _evaluate("UNREACHABLE", [FakeTrace("EXITS_NETWORK")]) is None


# --- _violation_detail (#439) ---------------------------------------------------
# run() builds a finding's evidence.detail by calling this function with
# _evaluate()'s own output. Tested directly so tests/test_explain_safety_
# net.py's routing fixtures, which call this same function to build a
# realistic detail string, fail loudly if this format ever changes --
# rather than silently testing a hand-typed copy that no longer matches
# what run() actually emits.


def test_violation_detail_matches_what_run_actually_builds():
violation = _evaluate("REACHABLE", [FakeTrace("NO_ROUTE")])
detail = _violation_detail("REACHABLE", "10.10.10.5", "10.20.20.5", violation)
assert detail == (
"Expected REACHABLE for a flow from 10.10.10.5 to 10.20.20.5, "
f"but {violation}"
)


# --- Unknown expected value ----------------------------------------------------


Expand Down
Loading