From 6fcd8045166b1eeb4142bef9cf62a5b8e87ecd0f Mon Sep 17 00:00:00 2001 From: patelankeet2 Date: Sun, 15 Nov 2026 13:58:31 +1300 Subject: [PATCH] A negated reachability question answers the negation, not the plain question (#435) "Can X NOT reach Y" was answered identically to "Can X reach Y" -- the word NOT sat inside source_text, where _find_device() ignores anything that is not a known device name, so it was dropped silently rather than refused. Measured in the issue: grounded true, full confidence, a full inversion of what was asked. Added a closed set of negation words anchored to the end of source_text (_NEGATION_BEFORE_REACH / _is_negated_reachability), matching the same discipline _REACH_KEYWORDS and the back-reference sets already use. question_understood now states the negation back explicitly, and the answer text flips Yes/No to match what was actually asked, over the same Batfish evidence -- the mixed-result case is unaffected since it already has no yes/no word to get backwards. Port and protocol constraints are a separate, still-open part of #435, not touched here on purpose -- one shape at a time, same as #52/#145/#384. --- ai/query.py | 79 ++++++++++++++++++++-- tests/test_query_translation.py | 116 ++++++++++++++++++++++++++++++++ 2 files changed, 191 insertions(+), 4 deletions(-) diff --git a/ai/query.py b/ai/query.py index d220b10..934aa38 100644 --- a/ai/query.py +++ b/ai/query.py @@ -111,6 +111,27 @@ interface description text to match a name against). A real, incremental step, not the whole client ask. + A NEGATED reachability question is answered as the negation, not as + the plain question (#435). "Can X NOT reach Y" used to be answered + identically to "Can X reach Y" -- the word "NOT" was simply inside + `source_text`, where `_find_device()` ignores anything that is not a + known device name, so it was silently dropped, not refused. Measured + live: this came back `grounded: true` with full confidence, a full + inversion of what was asked rather than a narrowed or missing detail -- + the same shape of danger #70's silent CIDR substitution above is. Fixed + the same way everything else here is: a closed set of negation words + (`_NEGATION_BEFORE_REACH`), anchored to the end of `source_text` so an + unrelated earlier "not" in the sentence never triggers it, exactly the + discipline `_REACH_KEYWORDS` and the back-reference sets already use. + `question_understood` states the negation back explicitly rather than + rendering the always-affirmative phrasing it used to, and the answer + text flips Yes/No to match what was actually asked, over the same + Batfish evidence. Port and protocol constraints named in the same + question (e.g. "on port 443", "over udp") are a SEPARATE, still-open + gap -- #435 lists three, and this fixes only the negation one, on + purpose, matching this project's one-shape-at-a-time discipline for an + inversion fix (#52, #145, #384 before it). + FOLLOW-UPS: ENTITIES CARRIED FORWARD, NEVER INTENT (#318) #240 named the unscoped version of this as "multi-step deterministic reasoning" -- a planner decomposing a compound question into an ordered @@ -209,6 +230,18 @@ _REACH_KEYWORDS = re.compile( r"\b(?:reach|access|connect to|get to|talk to)\b", re.IGNORECASE ) + +# #435: "Can rtr-us5 NOT reach X?" was answered as if it were "Can rtr-us5 +# reach X?" -- a full inversion, not a dropped detail, and it came back +# grounded: True with full confidence. A closed set, anchored to the END of +# the text before the reach keyword, exactly the same discipline +# _REACH_KEYWORDS and _SOURCE_BACK_REFERENCES already use -- so a question +# naming something unrelated earlier ("the router that is not in +# maintenance") never falsely triggers this. See _is_negated_reachability(). +_NEGATION_BEFORE_REACH = re.compile( + r"\b(?:not|never|cannot|can't|doesn't|does\s+not|won't|will\s+not)\s*$", + re.IGNORECASE, +) _DEAD_RULE_KEYWORDS = re.compile( r"\b(?:dead|unreachable|shadow(?:ed)?|never\s+(?:take|takes)\s+effect)\b" r".*\b(?:rules?|lines?|acls?)\b" @@ -302,6 +335,15 @@ def _split_on_reach_keyword(question: str) -> Optional[tuple]: return question[: match.start()], question[match.end() :] +def _is_negated_reachability(source_text: str) -> bool: + """True if `source_text` -- everything before the matched reach keyword + -- ends in a negation word from the closed set above. Anchored to the + end on purpose: "the router that is not in maintenance, can it reach + X" must not trigger this just because "not" appears somewhere earlier + in the sentence (#435).""" + return bool(_NEGATION_BEFORE_REACH.search(source_text)) + + # ------------------------------------------------------------------------------ # 1b. Back-references -- a closed set, exactly like everything else here # ------------------------------------------------------------------------------ @@ -531,6 +573,7 @@ def _answer_reachability_question( split: tuple, bf: Session, previous: Optional[Dict[str, Any]] = None ) -> Dict[str, Any]: source_text, destination_text = split + negated = _is_negated_reachability(source_text) present = snapshot.device_names(bf) if present is None: @@ -591,8 +634,13 @@ def _answer_reachability_question( "destination_display": destination_display, } + # Negated phrasing is stated back explicitly (#435) -- shape C only does + # its job of surfacing a mistranslation if the rendered question still + # says what was actually asked, not a silently-un-negated version of it. question_understood = ( - f"Can {source_device} reach {destination_display}?" + f"Does {source_device} NOT reach {destination_display}?" + if negated + else f"Can {source_device} reach {destination_display}?" ) refusal, caveat = _what_was_not_read(bf, question_understood) @@ -640,7 +688,10 @@ def _answer_reachability_question( traces = frame.iloc[0]["Traces"] return { "question_understood": question_understood, - "answer": _describe_traces(source_device, destination_display, traces) + caveat, + "answer": _describe_traces( + source_device, destination_display, traces, negated=negated + ) + + caveat, "grounded": True, "resolved_entities": resolved_entities, } @@ -710,7 +761,9 @@ def _what_was_not_read(bf: Session, question_understood: str) -> "tuple[Optional _SUCCESS_DISPOSITIONS = {"ACCEPTED", "DELIVERED_TO_SUBNET"} -def _describe_traces(source_device: str, destination_display: str, traces: Any) -> str: +def _describe_traces( + source_device: str, destination_display: str, traces: Any, negated: bool = False +) -> str: """Plain English, built directly from Batfish's own disposition and path -- never paraphrased by a model. States the observed effect, not an assumed cause, same discipline routing.py's own summaries hold. @@ -719,16 +772,34 @@ def _describe_traces(source_device: str, destination_display: str, traces: Any) call the destination -- a literal address, or (#70) "a host in (checked
)" -- so the answer names the same thing question_understood does, rather than the answer being silently more - specific than what the person was told was checked.""" + specific than what the person was told was checked. + + `negated` answers the question actually asked (#435): "Can X NOT reach + Y" means Yes when it does NOT reach, No when it DOES -- the opposite + framing of the plain question, over the exact same Batfish evidence. + Only the clean-cut cases flip; the mixed case below already states + plainly which paths succeed and which do not, with no yes/no word to + get backwards, so it is unchanged either way.""" successes = [t for t in traces if t.disposition in _SUCCESS_DISPOSITIONS] failures = [t for t in traces if t.disposition not in _SUCCESS_DISPOSITIONS] if successes and not failures: + if negated: + return ( + f"No -- traffic from {source_device} DOES reach " + f"{destination_display}, which is the opposite of what you " + f"asked ({source_device} NOT reaching it)." + ) return f"Yes. Traffic from {source_device} reaches {destination_display}." if failures and not successes: bad = failures[0] hops = " -> ".join(hop.node for hop in bad.hops) or source_device + if negated: + return ( + f"Yes -- traffic from {source_device} to {destination_display} " + f"does NOT reach it; it ends in {bad.disposition}. Path: {hops}." + ) return ( f"No. Traffic from {source_device} to {destination_display} ends in " f"{bad.disposition}. Path: {hops}." diff --git a/tests/test_query_translation.py b/tests/test_query_translation.py index 8cba4d8..905bba1 100644 --- a/tests/test_query_translation.py +++ b/tests/test_query_translation.py @@ -341,6 +341,122 @@ def test_reachability_mixed_result_is_reported_not_picked(): assert "disagree" in result["answer"] +# --- #435: a negated reachability question answers the negation, not the +# plain question ------------------------------------------------------------- +# +# "Can rtr-us5 NOT reach X" used to be answered identically to "Can rtr-us5 +# reach X" -- the word "NOT" sat inside source_text, where _find_device() +# ignores anything that is not a known device name, so it was dropped +# silently rather than refused. Measured live: grounded: true, full +# confidence, the answer to a question nobody asked. + + +def test_a_negated_question_that_does_reach_answers_no(): + frame = _FakeFrame([{"Traces": [FakeTrace("ACCEPTED")]}]) + session = _session_with_devices({"rtr-us5"}, traceroute_frame=frame) + result = answer_question("Can rtr-us5 NOT reach 218.8.104.58?", session) + assert result["grounded"] is True + assert result["answer"].startswith("No --") + assert "DOES reach" in result["answer"] + + +def test_a_negated_question_that_does_not_reach_answers_yes(): + frame = _FakeFrame( + [{"Traces": [FakeTrace("DENIED_IN", hops=[FakeHop("rtr-us5")])]}] + ) + session = _session_with_devices({"rtr-us5"}, traceroute_frame=frame) + result = answer_question("Can rtr-us5 NOT reach 218.8.104.58?", session) + assert result["grounded"] is True + assert result["answer"].startswith("Yes --") + assert "does NOT reach" in result["answer"] + assert "DENIED_IN" in result["answer"] + + +def test_negation_is_stated_back_in_question_understood(): + """Shape C only does its job if the rendered question still says what + was actually asked -- the bug this fixes made it say the opposite.""" + frame = _FakeFrame([{"Traces": [FakeTrace("ACCEPTED")]}]) + session = _session_with_devices({"rtr-us5"}, traceroute_frame=frame) + result = answer_question("Can rtr-us5 NOT reach 218.8.104.58?", session) + assert "NOT reach" in result["question_understood"] + + +def test_a_non_negated_question_is_unaffected(): + """Regression: the plain-question path must render exactly as before.""" + frame = _FakeFrame([{"Traces": [FakeTrace("ACCEPTED")]}]) + session = _session_with_devices({"rtr-us5"}, traceroute_frame=frame) + result = answer_question("Can rtr-us5 reach 218.8.104.58?", session) + assert result["answer"] == "Yes. Traffic from rtr-us5 reaches 218.8.104.58." + assert result["question_understood"] == "Can rtr-us5 reach 218.8.104.58?" + + +@pytest.mark.parametrize("phrasing", [ + "Can rtr-us5 never reach 218.8.104.58?", + "Can rtr-us5 cannot reach 218.8.104.58?", + "Can rtr-us5 can't reach 218.8.104.58?", + "Does rtr-us5 doesn't reach 218.8.104.58?", + "Does rtr-us5 does not reach 218.8.104.58?", + "Will rtr-us5 won't reach 218.8.104.58?", + "Will rtr-us5 will not reach 218.8.104.58?", +]) +def test_the_closed_set_of_negation_phrasings_are_all_recognised(phrasing): + """Deliberately ungrammatical combinations (e.g. "Can ... cannot + reach") -- the point is that each individual negation word in the + closed set is recognised on its own, not that the whole sentence + reads naturally.""" + frame = _FakeFrame([{"Traces": [FakeTrace("ACCEPTED")]}]) + session = _session_with_devices({"rtr-us5"}, traceroute_frame=frame) + result = answer_question(phrasing, session) + assert result["answer"].startswith("No --") + + +def test_an_unrelated_earlier_not_does_not_falsely_trigger_negation(): + """Anchored to the END of source_text on purpose: "not" appearing + earlier in the question, unrelated to the reach keyword, must not be + read as negating the question.""" + frame = _FakeFrame([{"Traces": [FakeTrace("ACCEPTED")]}]) + session = _session_with_devices({"rtr-us5"}, traceroute_frame=frame) + result = answer_question( + "Can the router that is not in maintenance, rtr-us5, reach 218.8.104.58?", + session, + ) + assert result["answer"].startswith("Yes.") + assert "question_understood" in result + assert result["question_understood"] == "Can rtr-us5 reach 218.8.104.58?" + + +def test_a_negated_mixed_result_still_reports_plainly(): + """The mixed case has no yes/no word to get backwards -- confirm + negation leaves it untouched rather than prefixing something + misleading onto already-neutral language.""" + frame = _FakeFrame( + [{"Traces": [FakeTrace("ACCEPTED"), FakeTrace("DENIED_IN")]}] + ) + session = _session_with_devices({"rtr-us5"}, traceroute_frame=frame) + result = answer_question("Can rtr-us5 NOT reach 218.8.104.58?", session) + assert "Mixed result" in result["answer"] + assert "disagree" in result["answer"] + + +def test_a_negated_back_reference_still_refuses_rather_than_guessing(): + """Combining negation with a back-reference follow-up ("does it NOT + reach X") is not something this fix teaches the back-reference + matcher to recognise -- _is_source_back_reference()'s closed set only + contains the bare reference word, not "it NOT". Confirms this falls + through to the existing refusal rather than silently resolving to the + previous turn's device, which is the safe degradation, not a gap this + issue needs to close.""" + frame = _FakeFrame([{"Traces": [FakeTrace("ACCEPTED")]}]) + session = _session_with_devices({"rtr-us5"}, traceroute_frame=frame) + previous = { + "source_device": "rtr-us5", + "destination_ip": "218.8.104.58", + "destination_display": "218.8.104.58", + } + result = answer_question("Does it NOT reach 218.8.104.58?", session, previous) + assert result["grounded"] is False + + def test_reachability_empty_frame_is_a_refusal(): frame = _FakeFrame([]) session = _session_with_devices({"rtr-us5"}, traceroute_frame=frame)