Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 75 additions & 4 deletions ai/query.py
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,27 @@
interface description text to match a name against). A real,
incremental step, not the whole client ask.

A NEGATED reachability question is answered as the negation, not as
the plain question (#435). "Can X NOT reach Y" used to be answered
identically to "Can X reach Y" -- the word "NOT" was simply inside
`source_text`, where `_find_device()` ignores anything that is not a
known device name, so it was silently dropped, not refused. Measured
live: this came back `grounded: true` with full confidence, a full
inversion of what was asked rather than a narrowed or missing detail --
the same shape of danger #70's silent CIDR substitution above is. Fixed
the same way everything else here is: a closed set of negation words
(`_NEGATION_BEFORE_REACH`), anchored to the end of `source_text` so an
unrelated earlier "not" in the sentence never triggers it, exactly the
discipline `_REACH_KEYWORDS` and the back-reference sets already use.
`question_understood` states the negation back explicitly rather than
rendering the always-affirmative phrasing it used to, and the answer
text flips Yes/No to match what was actually asked, over the same
Batfish evidence. Port and protocol constraints named in the same
question (e.g. "on port 443", "over udp") are a SEPARATE, still-open
gap -- #435 lists three, and this fixes only the negation one, on
purpose, matching this project's one-shape-at-a-time discipline for an
inversion fix (#52, #145, #384 before it).

FOLLOW-UPS: ENTITIES CARRIED FORWARD, NEVER INTENT (#318)
#240 named the unscoped version of this as "multi-step deterministic
reasoning" -- a planner decomposing a compound question into an ordered
Expand Down Expand Up @@ -209,6 +230,18 @@
_REACH_KEYWORDS = re.compile(
r"\b(?:reach|access|connect to|get to|talk to)\b", re.IGNORECASE
)

# #435: "Can rtr-us5 NOT reach X?" was answered as if it were "Can rtr-us5
# reach X?" -- a full inversion, not a dropped detail, and it came back
# grounded: True with full confidence. A closed set, anchored to the END of
# the text before the reach keyword, exactly the same discipline
# _REACH_KEYWORDS and _SOURCE_BACK_REFERENCES already use -- so a question
# naming something unrelated earlier ("the router that is not in
# maintenance") never falsely triggers this. See _is_negated_reachability().
_NEGATION_BEFORE_REACH = re.compile(
r"\b(?:not|never|cannot|can't|doesn't|does\s+not|won't|will\s+not)\s*$",
re.IGNORECASE,
)
_DEAD_RULE_KEYWORDS = re.compile(
r"\b(?:dead|unreachable|shadow(?:ed)?|never\s+(?:take|takes)\s+effect)\b"
r".*\b(?:rules?|lines?|acls?)\b"
Expand Down Expand Up @@ -302,6 +335,15 @@ def _split_on_reach_keyword(question: str) -> Optional[tuple]:
return question[: match.start()], question[match.end() :]


def _is_negated_reachability(source_text: str) -> bool:
"""True if `source_text` -- everything before the matched reach keyword
-- ends in a negation word from the closed set above. Anchored to the
end on purpose: "the router that is not in maintenance, can it reach
X" must not trigger this just because "not" appears somewhere earlier
in the sentence (#435)."""
return bool(_NEGATION_BEFORE_REACH.search(source_text))


# ------------------------------------------------------------------------------
# 1b. Back-references -- a closed set, exactly like everything else here
# ------------------------------------------------------------------------------
Expand Down Expand Up @@ -531,6 +573,7 @@ def _answer_reachability_question(
split: tuple, bf: Session, previous: Optional[Dict[str, Any]] = None
) -> Dict[str, Any]:
source_text, destination_text = split
negated = _is_negated_reachability(source_text)

present = snapshot.device_names(bf)
if present is None:
Expand Down Expand Up @@ -591,8 +634,13 @@ def _answer_reachability_question(
"destination_display": destination_display,
}

# Negated phrasing is stated back explicitly (#435) -- shape C only does
# its job of surfacing a mistranslation if the rendered question still
# says what was actually asked, not a silently-un-negated version of it.
question_understood = (
f"Can {source_device} reach {destination_display}?"
f"Does {source_device} NOT reach {destination_display}?"
if negated
else f"Can {source_device} reach {destination_display}?"
)

refusal, caveat = _what_was_not_read(bf, question_understood)
Expand Down Expand Up @@ -640,7 +688,10 @@ def _answer_reachability_question(
traces = frame.iloc[0]["Traces"]
return {
"question_understood": question_understood,
"answer": _describe_traces(source_device, destination_display, traces) + caveat,
"answer": _describe_traces(
source_device, destination_display, traces, negated=negated
)
+ caveat,
"grounded": True,
"resolved_entities": resolved_entities,
}
Expand Down Expand Up @@ -710,7 +761,9 @@ def _what_was_not_read(bf: Session, question_understood: str) -> "tuple[Optional
_SUCCESS_DISPOSITIONS = {"ACCEPTED", "DELIVERED_TO_SUBNET"}


def _describe_traces(source_device: str, destination_display: str, traces: Any) -> str:
def _describe_traces(
source_device: str, destination_display: str, traces: Any, negated: bool = False
) -> str:
"""Plain English, built directly from Batfish's own disposition and
path -- never paraphrased by a model. States the observed effect, not
an assumed cause, same discipline routing.py's own summaries hold.
Expand All @@ -719,16 +772,34 @@ def _describe_traces(source_device: str, destination_display: str, traces: Any)
call the destination -- a literal address, or (#70) "a host in
<network> (checked <address>)" -- so the answer names the same thing
question_understood does, rather than the answer being silently more
specific than what the person was told was checked."""
specific than what the person was told was checked.

`negated` answers the question actually asked (#435): "Can X NOT reach
Y" means Yes when it does NOT reach, No when it DOES -- the opposite
framing of the plain question, over the exact same Batfish evidence.
Only the clean-cut cases flip; the mixed case below already states
plainly which paths succeed and which do not, with no yes/no word to
get backwards, so it is unchanged either way."""
successes = [t for t in traces if t.disposition in _SUCCESS_DISPOSITIONS]
failures = [t for t in traces if t.disposition not in _SUCCESS_DISPOSITIONS]

if successes and not failures:
if negated:
return (
f"No -- traffic from {source_device} DOES reach "
f"{destination_display}, which is the opposite of what you "
f"asked ({source_device} NOT reaching it)."
)
return f"Yes. Traffic from {source_device} reaches {destination_display}."

if failures and not successes:
bad = failures[0]
hops = " -> ".join(hop.node for hop in bad.hops) or source_device
if negated:
return (
f"Yes -- traffic from {source_device} to {destination_display} "
f"does NOT reach it; it ends in {bad.disposition}. Path: {hops}."
)
return (
f"No. Traffic from {source_device} to {destination_display} ends in "
f"{bad.disposition}. Path: {hops}."
Expand Down
116 changes: 116 additions & 0 deletions tests/test_query_translation.py
Original file line number Diff line number Diff line change
Expand Up @@ -341,6 +341,122 @@ def test_reachability_mixed_result_is_reported_not_picked():
assert "disagree" in result["answer"]


# --- #435: a negated reachability question answers the negation, not the
# plain question -------------------------------------------------------------
#
# "Can rtr-us5 NOT reach X" used to be answered identically to "Can rtr-us5
# reach X" -- the word "NOT" sat inside source_text, where _find_device()
# ignores anything that is not a known device name, so it was dropped
# silently rather than refused. Measured live: grounded: true, full
# confidence, the answer to a question nobody asked.


def test_a_negated_question_that_does_reach_answers_no():
frame = _FakeFrame([{"Traces": [FakeTrace("ACCEPTED")]}])
session = _session_with_devices({"rtr-us5"}, traceroute_frame=frame)
result = answer_question("Can rtr-us5 NOT reach 218.8.104.58?", session)
assert result["grounded"] is True
assert result["answer"].startswith("No --")
assert "DOES reach" in result["answer"]


def test_a_negated_question_that_does_not_reach_answers_yes():
frame = _FakeFrame(
[{"Traces": [FakeTrace("DENIED_IN", hops=[FakeHop("rtr-us5")])]}]
)
session = _session_with_devices({"rtr-us5"}, traceroute_frame=frame)
result = answer_question("Can rtr-us5 NOT reach 218.8.104.58?", session)
assert result["grounded"] is True
assert result["answer"].startswith("Yes --")
assert "does NOT reach" in result["answer"]
assert "DENIED_IN" in result["answer"]


def test_negation_is_stated_back_in_question_understood():
"""Shape C only does its job if the rendered question still says what
was actually asked -- the bug this fixes made it say the opposite."""
frame = _FakeFrame([{"Traces": [FakeTrace("ACCEPTED")]}])
session = _session_with_devices({"rtr-us5"}, traceroute_frame=frame)
result = answer_question("Can rtr-us5 NOT reach 218.8.104.58?", session)
assert "NOT reach" in result["question_understood"]


def test_a_non_negated_question_is_unaffected():
"""Regression: the plain-question path must render exactly as before."""
frame = _FakeFrame([{"Traces": [FakeTrace("ACCEPTED")]}])
session = _session_with_devices({"rtr-us5"}, traceroute_frame=frame)
result = answer_question("Can rtr-us5 reach 218.8.104.58?", session)
assert result["answer"] == "Yes. Traffic from rtr-us5 reaches 218.8.104.58."
assert result["question_understood"] == "Can rtr-us5 reach 218.8.104.58?"


@pytest.mark.parametrize("phrasing", [
"Can rtr-us5 never reach 218.8.104.58?",
"Can rtr-us5 cannot reach 218.8.104.58?",
"Can rtr-us5 can't reach 218.8.104.58?",
"Does rtr-us5 doesn't reach 218.8.104.58?",
"Does rtr-us5 does not reach 218.8.104.58?",
"Will rtr-us5 won't reach 218.8.104.58?",
"Will rtr-us5 will not reach 218.8.104.58?",
])
def test_the_closed_set_of_negation_phrasings_are_all_recognised(phrasing):
"""Deliberately ungrammatical combinations (e.g. "Can ... cannot
reach") -- the point is that each individual negation word in the
closed set is recognised on its own, not that the whole sentence
reads naturally."""
frame = _FakeFrame([{"Traces": [FakeTrace("ACCEPTED")]}])
session = _session_with_devices({"rtr-us5"}, traceroute_frame=frame)
result = answer_question(phrasing, session)
assert result["answer"].startswith("No --")


def test_an_unrelated_earlier_not_does_not_falsely_trigger_negation():
"""Anchored to the END of source_text on purpose: "not" appearing
earlier in the question, unrelated to the reach keyword, must not be
read as negating the question."""
frame = _FakeFrame([{"Traces": [FakeTrace("ACCEPTED")]}])
session = _session_with_devices({"rtr-us5"}, traceroute_frame=frame)
result = answer_question(
"Can the router that is not in maintenance, rtr-us5, reach 218.8.104.58?",
session,
)
assert result["answer"].startswith("Yes.")
assert "question_understood" in result
assert result["question_understood"] == "Can rtr-us5 reach 218.8.104.58?"


def test_a_negated_mixed_result_still_reports_plainly():
"""The mixed case has no yes/no word to get backwards -- confirm
negation leaves it untouched rather than prefixing something
misleading onto already-neutral language."""
frame = _FakeFrame(
[{"Traces": [FakeTrace("ACCEPTED"), FakeTrace("DENIED_IN")]}]
)
session = _session_with_devices({"rtr-us5"}, traceroute_frame=frame)
result = answer_question("Can rtr-us5 NOT reach 218.8.104.58?", session)
assert "Mixed result" in result["answer"]
assert "disagree" in result["answer"]


def test_a_negated_back_reference_still_refuses_rather_than_guessing():
"""Combining negation with a back-reference follow-up ("does it NOT
reach X") is not something this fix teaches the back-reference
matcher to recognise -- _is_source_back_reference()'s closed set only
contains the bare reference word, not "it NOT". Confirms this falls
through to the existing refusal rather than silently resolving to the
previous turn's device, which is the safe degradation, not a gap this
issue needs to close."""
frame = _FakeFrame([{"Traces": [FakeTrace("ACCEPTED")]}])
session = _session_with_devices({"rtr-us5"}, traceroute_frame=frame)
previous = {
"source_device": "rtr-us5",
"destination_ip": "218.8.104.58",
"destination_display": "218.8.104.58",
}
result = answer_question("Does it NOT reach 218.8.104.58?", session, previous)
assert result["grounded"] is False


def test_reachability_empty_frame_is_a_refusal():
frame = _FakeFrame([])
session = _session_with_devices({"rtr-us5"}, traceroute_frame=frame)
Expand Down
Loading