From 53eaace4be91dcd39de90e9c7c41cf0bad32ed78 Mon Sep 17 00:00:00 2001 From: Samika Perera Date: Tue, 3 Nov 2026 17:49:44 +1300 Subject: [PATCH 1/5] Add a test file for clean-sounding explanations the output check accepts The module docstring says what the gap is and why it matters (F-4): the check that stops a claim about the network runs only for status="error", and it is a short word list, so "Everything looks clean." passes for a check that could not run, and "No issues found." passes for a confirmed problem. Two helpers build a real finding of a given status and report whether a stubbed model answer would reach the user as model output. No tests yet, so pytest reports that none ran. --- tests/test_explain_output_check_gap.py | 87 ++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 tests/test_explain_output_check_gap.py diff --git a/tests/test_explain_output_check_gap.py b/tests/test_explain_output_check_gap.py new file mode 100644 index 0000000..1edbeb3 --- /dev/null +++ b/tests/test_explain_output_check_gap.py @@ -0,0 +1,87 @@ +""" +Netwise -- what ai/explain.py's output check lets through when a model +says "all clear" about a finding that is not. + +WHY THIS FILE EXISTS + explain_with_source() asks the model for an explanation and then runs + _is_unacceptable() on the answer. If the answer passes, it reaches the + user labelled as model-written. If it fails twice, the user gets the + deterministic fallback instead. + + The check that stops a claim about the network runs only for + status="error" (`is_error = finding.get("status") == "error"`), and it + is a word list: reach, reachable, blocked, works, working, vulnerable, + safe, secure, clear, issues. Measured with no model on 3 November, by + stubbing the generator, this lets through: + + - for status="error", "There are no problems with this rule.", + "This is fine and nothing needs fixing.", "Everything looks + clean." and "This check passed." None of them uses a listed word. + - for status="found", every clean-sounding sentence tried, including + "No issues found." and "...the rule is safe.", because the check + does not run for "found" at all. + + The first is F-4's failure (CLAUDE.md section 7a): a check that COULD + NOT RUN described to the user as if it had run and found nothing. The + second is a confirmed problem described as no problem. web/main.py + explains only status="found" findings, so the second is on the path the + dashboard actually uses. + + The tests marked as gaps below are RED ON PURPOSE. They pin what the + check should refuse, so the gap is visible in the suite rather than only + in a note. They are not marked xfail, because an expected failure is + easy to stop seeing. + +HOW THE CHECK IS DRIVEN + Through explain_with_source(), with the generator stubbed to return a + chosen sentence and the Ollama probe stubbed to say it is reachable -- + the same arrangement tests/test_explain_safety_net.py uses. No model, + no network, no Batfish. A sentence is "accepted" when it comes back + unchanged with source "model". + + Findings are built with the real constructors in analysis/findings.py, + so each one has exactly the shape a check would produce. + +RUN + pytest tests/test_explain_output_check_gap.py -v +""" + +from ai import explain as explain_module +from analysis.findings import error_finding, make_finding, no_issues_finding + + +def _finding(status: str) -> dict: + """One finding of the given status, built the way a check builds it.""" + if status == "found": + return make_finding( + check="access_control", severity="high", device="rtr-us5", + summary="A filter permits traffic the policy requires to be denied", + detail=("Expected DENY but got PERMIT for 10.10.10.0/24 -> " + "10.20.0.5 TCP/23, decided by: permit ip any any"), + source="rtr-us5.cfg:42", status="found", number=1) + if status == "none": + return no_issues_finding( + check="access_control", device="rtr-us5", + summary="No issues found by access control", + detail="1 policy statement(s) hold, 1 guarantee(s) proven", + source="analysis/checks/access_control.py") + if status == "error": + return error_finding( + check="access_control", device="rtr-us5", + summary="Could not check the access-control policy", + detail="BatfishException: Work terminated abnormally", + source="analysis/checks/access_control.py") + raise ValueError(f"unknown status {status!r}") + + +def _accepted(monkeypatch, status: str, text: str) -> bool: + """True if the explainer would show `text` to the user as model output. + + Stubs the generator to return `text` on every attempt, so a rejection + can only come from the output check, and the probe to say the model is + reachable, so the check is actually reached. + """ + monkeypatch.setattr(explain_module, "_generate", lambda finding: text) + monkeypatch.setattr(explain_module, "_ollama_is_reachable", lambda: True) + shown, source = explain_module.explain_with_source(_finding(status)) + return source == "model" and shown == text From e7f2ec3e3c7fc01e93dc375824770cd685bed3e4 Mon Sep 17 00:00:00 2001 From: Samika Perera Date: Tue, 3 Nov 2026 17:50:03 +1300 Subject: [PATCH 2/5] Add the control: a faithful explanation is still accepted for every status One accurate sentence per status (found, none, error), each repeating only what its finding says. All three pass today. This stops a future fix from making the gap tests pass by rejecting everything. --- tests/test_explain_output_check_gap.py | 28 ++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/test_explain_output_check_gap.py b/tests/test_explain_output_check_gap.py index 1edbeb3..f964d9e 100644 --- a/tests/test_explain_output_check_gap.py +++ b/tests/test_explain_output_check_gap.py @@ -46,6 +46,8 @@ pytest tests/test_explain_output_check_gap.py -v """ +import pytest + from ai import explain as explain_module from analysis.findings import error_finding, make_finding, no_issues_finding @@ -85,3 +87,29 @@ def _accepted(monkeypatch, status: str, text: str) -> bool: monkeypatch.setattr(explain_module, "_ollama_is_reachable", lambda: True) shown, source = explain_module.explain_with_source(_finding(status)) return source == "model" and shown == text + + +# --- The control: a faithful explanation still gets through ------------------- +# +# Without this, a fix that rejected everything would make every gap test +# below pass. Each sentence repeats only what its finding says. + +_FAITHFUL = { + "found": ("The filter on rtr-us5 lets telnet from 10.10.10.0/24 to " + "10.20.0.5 through, because the line permit ip any any decides " + "it, although the policy says that traffic should be denied."), + "none": ("The access-control check ran on rtr-us5 and every policy " + "statement it was given held."), + "error": ("The access-control check did not run on rtr-us5, so nothing is " + "confirmed either way. The underlying reason: BatfishException: " + "Work terminated abnormally."), +} + + +@pytest.mark.parametrize("status", ["found", "none", "error"]) +def test_a_faithful_explanation_is_still_accepted(monkeypatch, status): + assert _accepted(monkeypatch, status, _FAITHFUL[status]), ( + f"a faithful explanation of a status={status!r} finding was rejected: " + f"{_FAITHFUL[status]!r}. The output check must still let an accurate " + f"answer through, or every test in this file passes for the wrong reason." + ) From dbe3edcd6616d2f08e6208959b87eb88db499be4 Mon Sep 17 00:00:00 2001 From: Samika Perera Date: Tue, 3 Nov 2026 17:50:19 +1300 Subject: [PATCH 3/5] Pin that an error finding can be explained as fine (red on purpose) A status="error" finding means the check could not run. Four sentences that tell the user the opposite are accepted by the output check today, because none of their words is on the error-only result-claim list: "There are no problems with this rule.", "This is fine and nothing needs fixing.", "Everything looks clean." and "This check passed." That is F-4's failure. The test asserts each is rejected, so it fails on main for that reason, and is left failing on purpose rather than marked xfail. --- tests/test_explain_output_check_gap.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/test_explain_output_check_gap.py b/tests/test_explain_output_check_gap.py index f964d9e..f29cae2 100644 --- a/tests/test_explain_output_check_gap.py +++ b/tests/test_explain_output_check_gap.py @@ -113,3 +113,26 @@ def test_a_faithful_explanation_is_still_accepted(monkeypatch, status): f"{_FAITHFUL[status]!r}. The output check must still let an accurate " f"answer through, or every test in this file passes for the wrong reason." ) + + +# --- GAP 1, RED ON PURPOSE: a check that could not run, explained as fine ------ +# +# F-4. status="error" means nobody looked. Each sentence below tells the +# user the opposite, and each is accepted today because none of its words is +# in _RESULT_CLAIM_WORDS. ("No issues found." and "...the rule is safe." are +# already refused for this status, by "issues" and "safe", so they are not +# repeated here.) + +@pytest.mark.parametrize("text", [ + "There are no problems with this rule.", + "This is fine and nothing needs fixing.", + "Everything looks clean.", + "This check passed.", +]) +def test_an_error_finding_is_not_explained_as_fine(monkeypatch, text): + assert not _accepted(monkeypatch, "error", text), ( + f"a status='error' finding -- the check COULD NOT RUN -- was explained " + f"to the user as {text!r}, labelled as model output. That is F-4's " + f"failure: 'we could not check' shown as 'we checked and it is fine'. " + f"The error-only result-claim list does not cover this wording." + ) From 2890ab779137565884003b188cf73a86ea726413 Mon Sep 17 00:00:00 2001 From: Samika Perera Date: Tue, 3 Nov 2026 17:50:35 +1300 Subject: [PATCH 4/5] Pin that a found finding can be explained as clean (red on purpose) A status="found" finding is a confirmed problem, and it is the status the dashboard sends to the model. The output check runs no result-claim test for it, so all six clean-sounding sentences tried are accepted today, including "No issues found." and "...the rule is safe.", which are refused for an error finding. The test asserts each is rejected, so it fails on main for that reason, and is left failing on purpose rather than marked xfail. --- tests/test_explain_output_check_gap.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/test_explain_output_check_gap.py b/tests/test_explain_output_check_gap.py index f29cae2..ba90f35 100644 --- a/tests/test_explain_output_check_gap.py +++ b/tests/test_explain_output_check_gap.py @@ -136,3 +136,28 @@ def test_an_error_finding_is_not_explained_as_fine(monkeypatch, text): f"failure: 'we could not check' shown as 'we checked and it is fine'. " f"The error-only result-claim list does not cover this wording." ) + + +# --- GAP 2, RED ON PURPOSE: a confirmed problem, explained as no problem ------- +# +# status="found" means the check ran and found something wrong. The +# result-claim check does not run for this status at all (it is gated on +# status == "error"), so every one of these is accepted today -- including +# the two that ARE refused for an error finding. This is the status +# web/main.py actually sends to the model. + +@pytest.mark.parametrize("text", [ + "There are no problems with this rule.", + "No issues found.", + "This is fine and nothing needs fixing.", + "Everything looks clean.", + "This check passed.", + "No problems were found, the rule is safe.", +]) +def test_a_found_finding_is_not_explained_as_clean(monkeypatch, text): + assert not _accepted(monkeypatch, "found", text), ( + f"a status='found' finding -- a confirmed problem -- was explained to " + f"the user as {text!r}, labelled as model output. The output check " + f"runs no result-claim test for 'found', so an all-clear contradicting " + f"the finding it explains gets through." + ) From 8c22f2ee735ca6d0e476f914dfc617928efa85e7 Mon Sep 17 00:00:00 2001 From: Samika Perera Date: Fri, 6 Nov 2026 16:46:35 +1300 Subject: [PATCH 5/5] Refuse an all-clear explanation of a found or error finding The output check only rejected claims about the network for status "error", and only by single words. A model answer such as "No issues found." for a confirmed problem, or "Everything looks clean." for a check that could not run, reached the user labelled as model output. Add a short list of all-clear phrases and check it for "found" and "error" findings. A match is treated like any other rejected answer: one retry, then the existing deterministic fallback. "none" findings are not checked, because there an all-clear is true. The existing error-only word list is unchanged; reusing it for "found" would have rejected 3 of the 4 rated explanations in the holdout set, which say "reach". --- ai/explain.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/ai/explain.py b/ai/explain.py index 3043a96..c5b9990 100644 --- a/ai/explain.py +++ b/ai/explain.py @@ -648,6 +648,18 @@ def remediate_with_source(finding: Dict[str, Any]) -> Optional["tuple[str, str]" ) _LEAD_TO_PATTERN = re.compile(r"\bcan\s+lead\s+to\b", re.IGNORECASE) +# Phrases that tell the user everything is fine. Checked for "found" (a +# confirmed problem) and "error" (nobody looked), never for "none", where an +# all-clear is the truth. Whole phrases rather than single words, because a +# "found" explanation legitimately says "reach" or "blocked" -- the word list +# above rejected 3 of the 4 rated "found" explanations in +# tests/fixtures/explanation_quality/holdout.json. +_ALL_CLEAR_PATTERN = re.compile( + r"\b(no problems|no issues|nothing needs fixing|looks clean|" + r"check passed|is fine|is safe)\b", + re.IGNORECASE, +) + def _looks_like_a_result_claim(text: str) -> bool: """True if `text` uses language that asserts something about the @@ -859,6 +871,9 @@ def _is_unacceptable(text: str, *, is_error: bool, failure-modes sections for why these are checked separately.""" if is_error and _looks_like_a_result_claim(text): return True + status = finding.get("status") if finding is not None else None + if (is_error or status == "found") and _ALL_CLEAR_PATTERN.search(text): + return True if finding is not None and _names_an_invented_fact(text, finding): return True return _looks_like_speculation(text)