diff --git a/ai/query.py b/ai/query.py index a92026b..d220b10 100644 --- a/ai/query.py +++ b/ai/query.py @@ -198,6 +198,7 @@ from pybatfish.datamodel.flow import HeaderConstraints from analysis import findings, snapshot +from analysis.pipeline import classify_parse_status # ------------------------------------------------------------------------------ # 1. Intent classification -- a closed set of three, nothing else @@ -594,6 +595,10 @@ def _answer_reachability_question( f"Can {source_device} reach {destination_display}?" ) + refusal, caveat = _what_was_not_read(bf, question_understood) + if refusal is not None: + return refusal + try: frame = ( bf.q.traceroute( @@ -635,12 +640,65 @@ def _answer_reachability_question( traces = frame.iloc[0]["Traces"] return { "question_understood": question_understood, - "answer": _describe_traces(source_device, destination_display, traces), + "answer": _describe_traces(source_device, destination_display, traces) + caveat, "grounded": True, "resolved_entities": resolved_entities, } +def _what_was_not_read(bf: Session, question_understood: str) -> "tuple[Optional[Dict[str, Any]], str]": + """Refuse when Batfish could not read the config; caveat when it read part. + + Returns (refusal, caveat). `refusal` is the whole answer to give INSTEAD + of asking Batfish anything, or None. `caveat` is a sentence to add to the + answer, or "". + + WHY THIS EXISTS + An empty answer from Batfish means "none" only if Batfish read the + config. Measured on main dc93ccb, a file it could not read at all + (status UNKNOWN, zero devices) answered "are there any dead rules?" + with "No, checked the whole snapshot and found none.", grounded. That + is "we found nothing" standing in for "we could not look" -- F-4. + + The rule is analyse()'s, and so is the classification + (classify_parse_status, #217): an unreadable file means no answer at + all, a partly read one means the answer AND the caveat. + """ + try: + fatal, partial = classify_parse_status(bf) + except Exception as error: + return { + "question_understood": question_understood, + "answer": ( + "I could not tell what Batfish was able to read in this " + "config, so I will not answer. The underlying reason: " + + findings.describe_error(error) + ), + "grounded": False, + # Nothing was resolved, and a follow-up must not reuse anything + # from a config we could not read (see the module docstring). + "resolved_entities": None, + }, "" + + if fatal: + return { + "question_understood": question_understood, + "answer": ( + "Batfish could not read: " + "; ".join(fatal) + ". I cannot " + "answer about rules it did not read, so I will not say yes or no." + ), + "grounded": False, + "resolved_entities": None, + }, "" + + if partial: + return None, ( + " Batfish could not read all of: " + "; ".join(partial) + + ", so nothing is claimed about the lines it skipped." + ) + return None, "" + + # Same set analysis/checks/routing.py uses, and for the same reason, # repeated here rather than imported so ai/ does not depend on a specific # check module for a general Batfish-interpretation fact. pybatfish's own @@ -695,6 +753,10 @@ def _describe_traces(source_device: str, destination_display: str, traces: Any) def _answer_whole_snapshot_question( bf: Session, *, batfish_question: str, question_understood: str ) -> Dict[str, Any]: + refusal, caveat = _what_was_not_read(bf, question_understood) + if refusal is not None: + return refusal + try: frame = getattr(bf.q, batfish_question)().answer().frame() except Exception as error: @@ -709,9 +771,13 @@ def _answer_whole_snapshot_question( } if frame.empty: + # "The whole snapshot" only when Batfish read all of it. return { "question_understood": question_understood, - "answer": "No, checked the whole snapshot and found none.", + "answer": ( + "No, found none in the part of the config Batfish could read." + caveat + if caveat else "No, checked the whole snapshot and found none." + ), "grounded": True, "resolved_entities": None, } @@ -721,6 +787,7 @@ def _answer_whole_snapshot_question( "answer": ( f"Yes, found {len(frame)}. See the dashboard's findings list for " "the specific lines, this question only confirms whether any exist." + + caveat ), "grounded": True, "resolved_entities": None, diff --git a/analysis/checks/policy_compliance.py b/analysis/checks/policy_compliance.py index a6734ce..1a1a79f 100644 --- a/analysis/checks/policy_compliance.py +++ b/analysis/checks/policy_compliance.py @@ -694,6 +694,10 @@ def run(bf: Session) -> List[Dict[str, Any]]: # note in the module docstring. hits: List[Any] = [] failures: List[Exception] = [] + # A rule with no queries asks Batfish nothing, and "nothing found" + # would then read as "the rule holds". It is a rule we could not check. + if not rule["queries"]: + failures.append(ValueError("this rule has no queries, so nothing was asked")) for headers in rule["queries"]: try: hits.extend(_search(bf, node, filter_name, action, headers)) diff --git a/analysis/checks/routing.py b/analysis/checks/routing.py index 8350e18..6f6ef69 100644 --- a/analysis/checks/routing.py +++ b/analysis/checks/routing.py @@ -269,6 +269,12 @@ def _evaluate(expected: str, traces: Sequence[Any]) -> Optional[str]: exercises both branches and specifically regression-tests the EXITS_NETWORK case described in the module docstring. """ + # No trace is no evidence either way. With nothing traced, both branches + # below find nothing wrong and report "holds" -- a false all-clear, so it + # is refused the same way as a bad "expected" at the bottom. + if len(traces) == 0: + raise ValueError("traceroute returned no traces, so nothing was checked") + successes = [t for t in traces if t.disposition in SUCCESS_DISPOSITIONS] failures = [t for t in traces if t.disposition not in SUCCESS_DISPOSITIONS] multi = len(traces) > 1 @@ -294,10 +300,9 @@ def _evaluate(expected: str, traces: Sequence[Any]) -> Optional[str]: return detail # A statement with anything else in "expected" is a bug in ROUTES, not a - # runtime condition -- raising here means run() surfaces it through the - # pipeline's own crash-isolation as a status="error" finding (see - # run_check() in analysis/pipeline.py) rather than silently doing nothing, - # which is exactly the failure this function exists to replace. + # runtime condition -- raising here means run() reports that one route as + # status="error" rather than silently doing nothing, which is exactly the + # failure this function exists to replace. raise ValueError(f"route 'expected' must be REACHABLE or UNREACHABLE, got {expected!r}") @@ -465,7 +470,24 @@ def run(bf: Session) -> List[Dict[str, Any]]: continue row = frame.iloc[0] - violation = _evaluate(route["expected"], row["Traces"]) + try: + violation = _evaluate(route["expected"], row["Traces"]) + except ValueError as error: + # This route cannot be judged: nothing was traced, or its "expected" + # is not a value we know. Same handling as the two cases above, so + # one bad route does not end the check for the routes after it or + # for the policy-free analyses below. + results.append( + findings.error_finding( + check=CHECK_NAME, + device=node, + summary=f"Could not check: {route['description'].lower()}", + detail=findings.describe_error(error), + source=f"{node}:{route['dst_ip']}", + number=route["number"], + ) + ) + continue if violation is None: # The statement holds. Nothing to report. diff --git a/analysis/pipeline.py b/analysis/pipeline.py index db196af..b713222 100644 --- a/analysis/pipeline.py +++ b/analysis/pipeline.py @@ -699,6 +699,29 @@ def _restore_protected_findings( ) continue + # Status is what a CHECK concluded, not a post-processor's to change + # (docs/severity-rules.md section 2, which says this is enforced + # here). Until it was, a stage setting status="none" turned "could + # not check" into a green tick and nothing said so. + if current.get("status") != original.get("status"): + number += 1 + complaints.append( + findings.error_finding( + check=name, + device=original.get("device", "unknown"), + summary=f"The {name.replace('_', ' ')} stage changed a finding's status", + detail=( + f"{fid} was {original.get('status')} and was set to " + f"{current.get('status')}. Restored. A post-processor " + "may re-rate and re-order, never change what a check " + "concluded." + ), + source="analysis/pipeline.py", + number=number, + ) + ) + current["status"] = original.get("status") + if original.get("status") != "error": continue diff --git a/tests/test_error_status_is_reported.py b/tests/test_error_status_is_reported.py new file mode 100644 index 0000000..f97c642 --- /dev/null +++ b/tests/test_error_status_is_reported.py @@ -0,0 +1,243 @@ +"""A check that COULD NOT RUN must say `status="error"` -- never go quiet. + +WHY THIS FILE EXISTS + tests/test_found_status_is_reported.py pins one half of F-4: a problem a + check finds is reported as `found`. This pins the other half: a Batfish + question that fails is reported as `error`, and is never allowed to + become silence that the all-clear then reads as "checked, nothing wrong". + + Measured by mutation on main dc93ccb. Each line below replaced one guard + with `continue` or `return []`, so the failure was swallowed instead of + reported, and the whole suite still passed with Batfish stopped, as on + CI. The first four were also run with Batfish up: still nothing failed. + + access_control testFilters raises -> swallowed SURVIVED + access_control testFilters empty -> swallowed SURVIVED + access_control searchFilters raises -> swallowed SURVIVED + access_control dead-rule query raises -> swallowed SURVIVED + access_control undefined-refs raises -> swallowed SURVIVED + routing traceroute raises -> swallowed SURVIVED + routing traceroute empty -> swallowed SURVIVED + explain an error card's text claims a result SURVIVED + + Every swallowed failure ends the same way: the check's all-clear names + the device -- "1 guarantee(s) proven" after searchFilters raised -- a + green tick about a question Batfish never answered. + +NO BATFISH, NO OLLAMA + Each test drives a whole check's run() with a fake session. A control + test proves the fake reaches the all-clear when nothing fails, so the + tests below it cannot pass by never getting that far. +""" + +from __future__ import annotations + +import pandas as pd +import pytest + +from ai import explain as explain_module +from analysis.checks import access_control, routing + +_EMPTY = pd.DataFrame() + + +class _Question: + def __init__(self, outcome): self._outcome = outcome + + def answer(self, **_kwargs): return self + + def frame(self): + if isinstance(self._outcome, Exception): + raise self._outcome + return self._outcome + + +class _Q: + """One preset outcome per question: a frame, or an exception to raise. + + An unprepared question fails the test, so a check asking something new + cannot get an empty frame by accident and read it as "nothing found". + """ + + def __init__(self, outcomes): self._outcomes = outcomes + + def __getattr__(self, name): + if name not in self._outcomes: + raise AssertionError(f"the check asked an unprepared question: {name}()") + return lambda *args, **kwargs: _Question(self._outcomes[name]) + + +class _FakeSession: + def __init__(self, outcomes): self.q = _Q(outcomes) + + +def _batfish_failed(): + return RuntimeError("BatfishException: Work terminated abnormally") + + +def _statuses(results): + return [f["status"] for f in results] + + +# --------------------------------------------------------------------------- +# access_control +# --------------------------------------------------------------------------- + +_STATEMENT = { + "description": "DNS must be allowed", + "node": "rtr-us5", + "filter": "acl_in", + "headers": {"dstIps": "10.10.10.5"}, + "expected": "PERMIT", + "violation_summary": "DNS is blocked", + "violation_severity": "medium", +} +_GUARANTEE = { + "description": "No plaintext web out of the subnet", + "node": "rtr-us5", + "filter": "acl_in", + "headers": {"srcIps": "10.10.10.0/24", "applications": ["http"]}, + "violation_summary": "Plaintext web is allowed out", + "violation_severity": "high", +} + + +def _access_control_outcomes(**overrides): + """Every question access_control asks, answered as a clean rtr-us5.""" + outcomes = { + "fileParseStatus": pd.DataFrame([{ + "File_Name": "configs/rtr-us5.cfg", "Status": "PASSED", + "Nodes": ["rtr-us5"]}]), + "testFilters": pd.DataFrame([{"Action": "PERMIT", + "Line_Content": "permit udp any any eq 53"}]), + "searchFilters": _EMPTY, # empty = the guarantee is proven + "filterLineReachability": _EMPTY, + "undefinedReferences": _EMPTY, + "unusedStructures": _EMPTY, + } + outcomes.update(overrides) + return outcomes + + +@pytest.fixture +def _one_statement_one_guarantee(monkeypatch): + monkeypatch.setattr(access_control, "POLICY", [_STATEMENT]) + monkeypatch.setattr(access_control, "GUARANTEES", [_GUARANTEE]) + + +def test_control_a_clean_access_control_run_reaches_the_all_clear( + _one_statement_one_guarantee): + results = access_control.run(_FakeSession(_access_control_outcomes())) + assert _statuses(results) == ["none"], results + + +@pytest.mark.parametrize("question", [ + "testFilters", "searchFilters", "filterLineReachability", + "undefinedReferences", "unusedStructures", +]) +def test_an_access_control_question_that_fails_is_an_error_not_an_all_clear( + question, _one_statement_one_guarantee): + bf = _FakeSession(_access_control_outcomes(**{question: _batfish_failed()})) + + results = access_control.run(bf) + + assert "error" in _statuses(results), ( + f"{question}() failed and access_control reported no error: {results}") + assert "none" not in _statuses(results), ( + f"{question}() failed and access_control still gave an all-clear -- " + "a green tick about a question Batfish never answered") + + +def test_an_empty_test_filters_answer_is_an_error_not_an_all_clear( + _one_statement_one_guarantee): + """No row for the filter means it was not found, not that it agreed.""" + bf = _FakeSession(_access_control_outcomes(testFilters=_EMPTY)) + + results = access_control.run(bf) + + assert "error" in _statuses(results), results + assert "none" not in _statuses(results), results + + +# --------------------------------------------------------------------------- +# routing +# --------------------------------------------------------------------------- + +_ROUTE = { + "description": "hq must reach branch", + "node": "rtr-hq", + "src_ip": "10.1.1.1", + "dst_ip": "10.2.2.2", + "expected": "REACHABLE", + "violation_summary": "hq cannot reach branch", + "violation_severity": "high", + "number": 1, +} + + +class _Trace: + def __init__(self, disposition): + self.disposition = disposition + self.hops = [] + + +def _routing_outcomes(**overrides): + outcomes = { + "fileParseStatus": pd.DataFrame([{ + "File_Name": "configs/rtr-hq.cfg", "Status": "PASSED", + "Nodes": ["rtr-hq"]}]), + "traceroute": pd.DataFrame([{"Traces": [_Trace("ACCEPTED")]}]), + "ipOwners": _EMPTY, + "detectLoops": _EMPTY, + "bgpSessionCompatibility": _EMPTY, + "ospfSessionCompatibility": _EMPTY, + } + outcomes.update(overrides) + return outcomes + + +def test_control_a_clean_routing_run_reaches_the_all_clear(monkeypatch): + monkeypatch.setattr(routing, "ROUTES", [_ROUTE]) + results = routing.run(_FakeSession(_routing_outcomes())) + assert _statuses(results) == ["none"], results + + +@pytest.mark.parametrize("outcome", [_batfish_failed(), _EMPTY], + ids=["traceroute raises", "traceroute empty"]) +def test_a_traceroute_that_fails_is_an_error_not_an_all_clear(outcome, monkeypatch): + monkeypatch.setattr(routing, "ROUTES", [_ROUTE]) + bf = _FakeSession(_routing_outcomes(traceroute=outcome)) + + results = routing.run(bf) + + assert "error" in _statuses(results), results + assert "none" not in _statuses(results), ( + "the route was never traced and routing still said every assertion holds") + + +# --------------------------------------------------------------------------- +# explain -- an error card's prose must not claim a result +# --------------------------------------------------------------------------- + + +def test_a_model_claiming_a_result_for_an_error_card_is_not_shown(monkeypatch): + """_looks_like_a_result_claim() is unit-tested; this pins that it is USED. + + A confident sentence about the network, written for a check that never + ran, is the F-4 failure arriving through prose instead of status. + """ + monkeypatch.setattr(explain_module, "_ollama_is_reachable", lambda: True) + monkeypatch.setattr(explain_module, "_generate", + lambda finding: "The HQ LAN cannot reach the branch LAN.") + finding = { + "id": "RT-000", "check": "routing", "severity": "high", + "device": "unknown", "status": "error", + "summary": "Could not check: hq must reach branch", + "evidence": {"detail": "BatfishException: Work terminated abnormally", + "source": "traceroute"}, + } + + text, source = explain_module.explain_with_source(finding) + + assert source == "fallback", text + assert "nothing is confirmed" in text diff --git a/tests/test_finding_ids.py b/tests/test_finding_ids.py index ef031ff..aaee9a4 100644 --- a/tests/test_finding_ids.py +++ b/tests/test_finding_ids.py @@ -364,6 +364,29 @@ def bury_the_error(results): assert "RT-000" in complaints[0]["evidence"]["detail"] +def test_changing_a_finding_status_is_undone_and_reported(monkeypatch): + """docs/severity-rules.md section 2: a post-processor must not change + `status`, and that document says the rule is enforced here. Measured on + main dc93ccb, it was not: a stage setting every status to "none" turned + "could not check" into a green tick, and nothing said so.""" + from analysis import pipeline + + def relabel_everything_clean(results): + for f in results: + f["status"] = "none" + return results + + monkeypatch.setattr(pipeline, "POST_PROCESSORS", {"risk": relabel_everything_clean}) + out = pipeline.run_post_processors(_mixed_results()) + + by_id = {f["id"]: f for f in out} + assert by_id["RT-000"]["status"] == "error", "a blind spot must not become a green tick" + assert by_id["AC-001"]["status"] == "found", "a problem must not become a green tick" + complaints = [f for f in out if "changed a finding's status" in f["summary"]] + assert len(complaints) == 2 + assert all(c["status"] == "error" for c in complaints) + + def test_dropping_a_finding_is_undone_and_reported(monkeypatch): """Removing a finding is the same lie as never producing it.""" from analysis import pipeline diff --git a/tests/test_partial_parse.py b/tests/test_partial_parse.py index c5c5c96..c7a13a1 100644 --- a/tests/test_partial_parse.py +++ b/tests/test_partial_parse.py @@ -221,6 +221,25 @@ def test_a_clean_parse_attaches_no_caveat(monkeypatch, _stubbed_run): assert not [f for f in results if "may be incomplete" in f["summary"]] +def test_a_parse_status_batfish_cannot_report_refuses_to_analyse(monkeypatch, + _stubbed_run): + """Batfish answering the upload and then failing fileParseStatus. + + Nothing pinned this branch: replacing its `except Exception` with one + that catches nothing passed the whole suite on main dc93ccb, with Batfish + stopped and with it running. The exception then escaped analyse(), which + promises never to raise for an operational failure. + """ + def fails(bf): + raise RuntimeError("BatfishException: Work terminated abnormally") + + monkeypatch.setattr(pipeline, "classify_parse_status", fails) + results = pipeline.analyse("tests/fixtures/rtr-us5-secure") + + assert results and all(f["status"] == "error" for f in results), results + assert "parse status could not be read" in results[0]["summary"] + + def test_a_fatal_parse_still_refuses_to_analyse(monkeypatch, _stubbed_run): """Unchanged, and it must stay unchanged. diff --git a/tests/test_policy_partial_checks.py b/tests/test_policy_partial_checks.py index 102218e..d90d7f5 100644 --- a/tests/test_policy_partial_checks.py +++ b/tests/test_policy_partial_checks.py @@ -180,6 +180,26 @@ def test_a_rule_that_holds_on_every_arm_reports_all_clear(two_arm_rule): assert results[0]["id"] == "PC-000" +def test_a_rule_with_no_queries_is_could_not_check_not_all_clear(two_arm_rule): + """Zero arms is zero questions asked, not a rule that holds. + + analysis/policy.py accepts `"queries": []`. Measured live on main + dc93ccb with such a rule on rtr-us5-insecure: "No issues found by policy + compliance -- All 1 policy rule(s) about the device(s) listed hold", + status none, without one searchFilters call. + """ + two_arm_rule["queries"] = [] + asked = [] + + with pytest.MonkeyPatch.context() as mp: + mp.setattr(policy_compliance, "_search", lambda *a, **k: asked.append(a) or []) + results = policy_compliance.run(bf=None) + + assert asked == [] + assert [f["status"] for f in results] == ["error"], results + assert results[0]["summary"].startswith("Could not check policy rule") + + def test_the_two_bands_cannot_overlap_or_reach_the_guards_id(): """Violations and check-errors must stay disjoint, and clear of PC-999. diff --git a/tests/test_query_translation.py b/tests/test_query_translation.py index 54c385e..8cba4d8 100644 --- a/tests/test_query_translation.py +++ b/tests/test_query_translation.py @@ -87,12 +87,16 @@ class _FakeQuestions: def __init__( self, *, devices=frozenset(), traceroute_frame=None, - whole_snapshot_frame=None, raises=False, + whole_snapshot_frame=None, raises=False, parse_rows=None, ): self._devices = devices self._traceroute_frame = traceroute_frame self._whole_snapshot_frame = whole_snapshot_frame self._raises = raises + # fileParseStatus rows, for a test about what Batfish could READ. + # None means one fully parsed file holding `devices`, which is what + # every other test here assumes. + self._parse_rows = parse_rows # What traceroute() was actually called with, so a test can check # what address query.py resolved and handed to Batfish, not just # what it printed back -- the #70 fix is entirely about those two @@ -102,7 +106,13 @@ def __init__( def fileParseStatus(self, **_kwargs): if self._devices is None: raise RuntimeError("Batfish said no") - return _FakeAnswer(_FakeFrame([{"Nodes": sorted(self._devices)}])) + if self._parse_rows is not None: + return _FakeAnswer(_FakeFrame(self._parse_rows)) + # The three columns real Batfish always returns, not just Nodes. + return _FakeAnswer(_FakeFrame([{ + "File_Name": "configs/fake.cfg", "Status": "PASSED", + "Nodes": sorted(self._devices), + }])) def traceroute(self, **kwargs): self.traceroute_calls.append(kwargs) @@ -416,6 +426,78 @@ def test_undefined_reference_question_nonempty_result(): assert result["answer"].startswith("Yes, found 1") +# --- A config Batfish could not read is not a config with nothing in it ------- +# +# Measured live on main dc93ccb, a snapshot whose one file Batfish could not +# read at all (fileParseStatus UNKNOWN, zero devices): +# +# "are there any dead rules?" -> "No, checked the whole snapshot and +# found none." grounded: True +# +# analyse() refuses the same snapshot ("the config did not fully parse"). The +# question path never asked what Batfish had read, so an empty answer about +# NOTHING came back as a confident, grounded "no". + +_UNREADABLE = [{"File_Name": "configs/router.cfg", "Status": "UNKNOWN", "Nodes": []}] +_PARTLY_READ = [{"File_Name": "configs/fw.cfg", "Status": "PARTIALLY_UNRECOGNIZED", + "Nodes": ["fw-asa"]}] + + +def test_a_whole_snapshot_question_about_an_unreadable_config_is_refused(): + session = FakeSession(_FakeQuestions( + whole_snapshot_frame=_FakeFrame([]), parse_rows=_UNREADABLE)) + for question in ("Are there any dead rules?", "Anything undefined in the config?"): + result = answer_question(question, session) + assert result["grounded"] is False, result + assert not result["answer"].startswith("No"), result + assert "configs/router.cfg" in result["answer"] + # Shape C survives the refusal: the person still sees what was asked. + assert result["question_understood"] is not None + + +def test_a_reachability_question_is_refused_when_any_file_could_not_be_read(): + """One readable router and one file Batfish could not read: the router + resolves, but the unread file may sit on the path, so no yes or no.""" + frame = _FakeFrame([{"Traces": [FakeTrace("DENIED_IN")]}]) + rows = _UNREADABLE + [{"File_Name": "configs/rtr-us5.cfg", "Status": "PASSED", + "Nodes": ["rtr-us5"]}] + session = FakeSession(_FakeQuestions( + devices={"rtr-us5"}, traceroute_frame=frame, parse_rows=rows)) + result = answer_question("Can rtr-us5 reach 10.20.0.5?", session) + assert result["grounded"] is False, result + assert not result["answer"].startswith(("Yes", "No")), result + assert session.q.traceroute_calls == [] + # The refusal keeps answer_question()'s four-key shape (#318), and hands + # a follow-up nothing to reuse from a config Batfish could not read. + assert result["resolved_entities"] is None + + +def test_a_parse_status_refusal_keeps_the_four_key_shape(): + session = FakeSession(_FakeQuestions(devices=None, whole_snapshot_frame=_FakeFrame([]))) + result = answer_question("Are there any dead rules?", session) + assert set(result) == {"question_understood", "answer", "grounded", "resolved_entities"} + assert result["resolved_entities"] is None + + +def test_a_partly_read_config_answers_with_the_caveat_not_whole_snapshot(): + """PARTIALLY_UNRECOGNIZED is analysed AND caveated, as analyse() does + (#217) -- never "checked the whole snapshot" about lines it skipped.""" + session = FakeSession(_FakeQuestions( + whole_snapshot_frame=_FakeFrame([]), parse_rows=_PARTLY_READ)) + result = answer_question("Are there any dead rules?", session) + assert result["grounded"] is True + assert result["answer"].startswith("No") + assert "whole snapshot" not in result["answer"] + assert "configs/fw.cfg" in result["answer"] + + +def test_a_parse_status_that_cannot_be_read_is_a_refusal(): + session = FakeSession(_FakeQuestions(devices=None, whole_snapshot_frame=_FakeFrame([]))) + result = answer_question("Are there any dead rules?", session) + assert result["grounded"] is False + assert not result["answer"].startswith("No") + + # --- Intent classification does not cross-fire ---------------------------------- diff --git a/tests/test_routing_classification.py b/tests/test_routing_classification.py index e09b23c..3346808 100644 --- a/tests/test_routing_classification.py +++ b/tests/test_routing_classification.py @@ -139,3 +139,148 @@ def test_unknown_expected_value_raises_rather_than_silently_passing(): false "all clear" the way the original UNREACHABLE gap did.""" with pytest.raises(ValueError): _evaluate("SOMETHING_ELSE", [FakeTrace("ACCEPTED")]) + + +# --- No traces at all ---------------------------------------------------------- + + +@pytest.mark.parametrize("expected", ["REACHABLE", "UNREACHABLE"]) +def test_no_traces_raises_rather_than_silently_passing(expected): + """Zero traces is zero evidence, for either expectation. + + With an empty list there are no failures (so REACHABLE "held") and no + successes (so UNREACHABLE "held"). Measured on main dc93ccb, both + returned None, and routing.run() said "All 1 route assertion(s) ... + hold", status none, for a path nobody traced. + """ + with pytest.raises(ValueError): + _evaluate(expected, []) + + +def test_no_traces_reaches_the_user_as_could_not_check(monkeypatch): + """It reaches the user as an error, not a tick.""" + import pandas as pd + + from analysis import pipeline + from analysis.checks import routing + + class _Answer: + def answer(self): return self + def frame(self): return pd.DataFrame([{"Traces": []}]) + + class _Session: + class q: # noqa: N801 -- mirrors pybatfish's bf.q + @staticmethod + def traceroute(**_kwargs): return _Answer() + + route = { + "description": "hq must reach branch", "node": "rtr-hq", + "src_ip": "10.1.1.1", "dst_ip": "10.2.2.2", "expected": "REACHABLE", + "violation_summary": "hq cannot reach branch", + "violation_severity": "high", "number": 1, + } + monkeypatch.setattr(routing, "ROUTES", [route]) + monkeypatch.setattr(routing.snapshot, "device_names", lambda bf: {"rtr-hq"}) + monkeypatch.setattr(routing, "_routing_hygiene", lambda bf: []) + + results = pipeline.run_check(_Session(), "routing") + + assert [f["status"] for f in results] == ["error"], results + + +# --- One route that cannot be judged must not take the others with it ----------- +# +# Found by Ankeet reviewing #442. _evaluate() now raises for zero traces, and its +# call site in run() was the one place in the loop with no handling of its own. +# So one such route ended the whole check: the routes after it were never asked, +# the policy-free analyses never ran, and the user saw one "failed to run" card. + + +def _routes_and_session(monkeypatch, answers): + """Three routes on three devices. `answers` maps a device to its traces.""" + import pandas as pd + + from analysis.checks import routing + + asked = [] + + class _Answer: + def __init__(self, traces): self.traces = traces + def answer(self): return self + def frame(self): return pd.DataFrame([{"Traces": self.traces}]) + + class _Session: + class q: # noqa: N801 -- mirrors pybatfish's bf.q + @staticmethod + def traceroute(startLocation=None, **_kwargs): # noqa: N803 -- pybatfish's name + asked.append(startLocation) + return _Answer(answers[startLocation]) + + def route(number, node, expected="REACHABLE"): + return {"description": f"route {number}", "node": node, "src_ip": "10.1.1.1", + "dst_ip": "10.2.2.2", "expected": expected, + "violation_summary": f"route {number} is broken", + "violation_severity": "high", "number": number} + + monkeypatch.setattr(routing, "ROUTES", [route(1, "rtr-a"), route(2, "rtr-b"), route(3, "rtr-c")]) + monkeypatch.setattr(routing.snapshot, "device_names", lambda bf: {"rtr-a", "rtr-b", "rtr-c"}) + hygiene = [] + monkeypatch.setattr(routing, "_routing_hygiene", lambda bf: (hygiene.append(True), [])[1]) + return _Session(), asked, hygiene + + +def test_a_route_with_no_traces_does_not_stop_the_other_routes(monkeypatch): + from analysis import pipeline + + session, asked, hygiene = _routes_and_session(monkeypatch, { + "rtr-a": [], # nothing traced: cannot be judged + "rtr-b": [FakeTrace("NO_ROUTE")], # a real violation, and it must still be reported + "rtr-c": [FakeTrace("ACCEPTED")], # holds + }) + + results = pipeline.run_check(session, "routing") + by_device = {f["device"]: f for f in results} + + assert asked == ["rtr-a", "rtr-b", "rtr-c"], "every route must still be asked" + assert hygiene, "the policy-free analyses must still run" + assert by_device["rtr-a"]["status"] == "error" + assert by_device["rtr-a"]["id"] == "RT-001", "the error belongs to that route, not to the check" + assert "route 1" in by_device["rtr-a"]["summary"] + assert "no traces" in by_device["rtr-a"]["evidence"]["detail"] + assert by_device["rtr-b"]["status"] == "found" + + +def test_the_route_that_could_not_be_judged_is_never_vouched_for(monkeypatch): + """The all-clear covers rtr-c, which held. It must not include rtr-a, which + nobody checked: this PR exists to stop exactly that sentence being printed.""" + from analysis import pipeline + + session, _asked, _hygiene = _routes_and_session(monkeypatch, { + "rtr-a": [], "rtr-b": [FakeTrace("ACCEPTED")], "rtr-c": [FakeTrace("ACCEPTED")], + }) + + results = pipeline.run_check(session, "routing") + clean = [f for f in results if f["status"] == "none"] + + assert len(clean) == 1, results + assert "rtr-a" not in clean[0]["evidence"]["source"] + assert "rtr-b" in clean[0]["evidence"]["source"] and "rtr-c" in clean[0]["evidence"]["source"] + + +def test_a_bad_expected_value_is_reported_on_that_route_alone(monkeypatch): + """_evaluate() raises the same way for a bad "expected", so it gets the same + treatment: that route says it could not be checked, and the rest still run.""" + from analysis import pipeline + from analysis.checks import routing + + session, asked, hygiene = _routes_and_session(monkeypatch, { + "rtr-a": [FakeTrace("ACCEPTED")], "rtr-b": [FakeTrace("ACCEPTED")], "rtr-c": [FakeTrace("ACCEPTED")], + }) + routing.ROUTES[0]["expected"] = "SOMETIMES" + + results = pipeline.run_check(session, "routing") + by_device = {f["device"]: f for f in results} + + assert asked == ["rtr-a", "rtr-b", "rtr-c"] and hygiene + assert by_device["rtr-a"]["status"] == "error" + assert "SOMETIMES" in by_device["rtr-a"]["evidence"]["detail"] diff --git a/tools/query_quality.py b/tools/query_quality.py index 0d44612..cf7faaa 100644 --- a/tools/query_quality.py +++ b/tools/query_quality.py @@ -124,7 +124,14 @@ def __init__( def fileParseStatus(self, **_kwargs): if self._devices is None: raise RuntimeError("Batfish said no") - return _FakeAnswer(_FakeFrame([{"Nodes": sorted(self._devices)}])) + # The three columns real Batfish always returns, not just Nodes: + # answer_question() now checks Status before answering (it refuses on + # a config Batfish could not read), as tests/test_query_translation.py's + # own fake does. + return _FakeAnswer(_FakeFrame([{ + "File_Name": "configs/fake.cfg", "Status": "PASSED", + "Nodes": sorted(self._devices), + }])) def traceroute(self, **kwargs): self.traceroute_calls.append(kwargs)