From d31d8a43d10e913b29b7836f4b524dcd3c90777b Mon Sep 17 00:00:00 2001 From: patelankeet2 Date: Sun, 8 Nov 2026 18:29:50 +1300 Subject: [PATCH] Catch a one-word-inserted all-clear phrase (#457) _ALL_CLEAR_PATTERN (#456) refuses literal "no problems"/"no issues" for a found or error finding, but an inserted adjective still escapes it. Measured live on #456's own commit: 'There are no problems with this rule.' MATCH 'There are no security problems with this configuration.' NO MATCH 'The scan found no real issues of concern.' NO MATCH 'no security problems' is the exact sentence from the prompt-injection research that motivated this whole check in the first place. Widens exactly those two alternatives to allow one inserted word: no (\w+\s+)?problems, no (\w+\s+)?issues. Scope is deliberately narrow, per the issue: one adjective, not open-ended paraphrase detection, which is #410's separate problem. Tests: the two measured real sentences added to both existing gap-test lists, a new parametrized test pinning both statuses, and a regression test that checks the widened pattern against every captured_explanation in tests/fixtures/explanation_quality/holdout.json so a future widening that starts rejecting real answers is caught immediately. Mutation-tested: reverting to the literal pattern breaks 3 of the 4 new phrase tests for the right reason; the 4th (error status, 'no real issues') is independently caught by the pre-existing result-claim word list, which already lists 'issues' for error-only -- not a gap. Full suite: 2231 passed, 46 skipped, 0 failed. Live-tested against the real model: 15 generations each for a found and an error finding. All 15 found generations accepted as genuine model output; across all 30 runs, 0 final outputs match _ALL_CLEAR_PATTERN, so the widening does not start rejecting real explanations. --- ai/explain.py | 8 ++++- tests/test_explain_output_check_gap.py | 47 ++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/ai/explain.py b/ai/explain.py index c5b9990..9bf2d86 100644 --- a/ai/explain.py +++ b/ai/explain.py @@ -654,8 +654,14 @@ def remediate_with_source(finding: Dict[str, Any]) -> Optional["tuple[str, str]" # "found" explanation legitimately says "reach" or "blocked" -- the word list # above rejected 3 of the 4 rated "found" explanations in # tests/fixtures/explanation_quality/holdout.json. +# +# "no problems"/"no issues" allow exactly one inserted word (#457) -- measured +# live, the literal phrase alone missed "no security problems", the actual +# sentence from the prompt-injection case that motivated this whole check. +# Still a fixed list, not a paraphrase match: one adjective, not open-ended +# drift detection, which is #410's separate, bigger problem. _ALL_CLEAR_PATTERN = re.compile( - r"\b(no problems|no issues|nothing needs fixing|looks clean|" + r"\b(no (\w+\s+)?problems|no (\w+\s+)?issues|nothing needs fixing|looks clean|" r"check passed|is fine|is safe)\b", re.IGNORECASE, ) diff --git a/tests/test_explain_output_check_gap.py b/tests/test_explain_output_check_gap.py index ba90f35..e8e264e 100644 --- a/tests/test_explain_output_check_gap.py +++ b/tests/test_explain_output_check_gap.py @@ -128,6 +128,8 @@ def test_a_faithful_explanation_is_still_accepted(monkeypatch, status): "This is fine and nothing needs fixing.", "Everything looks clean.", "This check passed.", + "There are no security problems with this configuration.", + "The scan found no real issues of concern.", ]) def test_an_error_finding_is_not_explained_as_fine(monkeypatch, text): assert not _accepted(monkeypatch, "error", text), ( @@ -153,6 +155,8 @@ def test_an_error_finding_is_not_explained_as_fine(monkeypatch, text): "Everything looks clean.", "This check passed.", "No problems were found, the rule is safe.", + "There are no security problems with this configuration.", + "The scan found no real issues of concern.", ]) def test_a_found_finding_is_not_explained_as_clean(monkeypatch, text): assert not _accepted(monkeypatch, "found", text), ( @@ -161,3 +165,46 @@ def test_a_found_finding_is_not_explained_as_clean(monkeypatch, text): f"runs no result-claim test for 'found', so an all-clear contradicting " f"the finding it explains gets through." ) + + +# --- #457: the inserted word must not be allowed to roam --------------------- +# +# The widened pattern allows exactly one word between "no" and +# "problems"/"issues" (#457). It must still refuse the two real sentences +# that motivated the widening, and it must not start rejecting genuine, +# accurate explanations -- checked against every rated capture in +# tests/fixtures/explanation_quality/holdout.json, not just eyeballed. + +@pytest.mark.parametrize("status,text", [ + ("found", "There are no security problems with this configuration."), + ("error", "There are no security problems with this configuration."), + ("found", "The scan found no real issues of concern."), + ("error", "The scan found no real issues of concern."), +]) +def test_a_one_word_inserted_all_clear_phrase_is_still_refused(monkeypatch, status, text): + assert not _accepted(monkeypatch, status, text), ( + f"a status={status!r} finding was explained as {text!r}, labelled as " + f"model output. This is the exact sentence from the prompt-injection " + f"case that motivated _ALL_CLEAR_PATTERN in the first place, and it " + f"must not escape through an inserted adjective." + ) + + +def test_the_widened_pattern_does_not_reject_real_rated_explanations(): + import json + from pathlib import Path + + holdout_path = ( + Path(__file__).parent / "fixtures" / "explanation_quality" / "holdout.json" + ) + data = json.loads(holdout_path.read_text(encoding="utf-8")) + for entry in data["findings"]: + text = entry["captured_explanation"] + match = explain_module._ALL_CLEAR_PATTERN.search(text) + assert match is None, ( + f"{entry['id']}'s real, rated explanation was flagged as an " + f"all-clear phrase by the widened pattern ({match.group()!r} in " + f"{text!r}). The one-word-insertion widening (#457) must not start " + f"rejecting genuine, accurate answers -- that's exactly the " + f"false-positive risk the issue asked to re-test before landing." + )