From 9b09766b82e19e82374fb7aa6019dba6fd0d9dc2 Mon Sep 17 00:00:00 2001 From: patelankeet2 Date: Sun, 8 Nov 2026 16:41:09 +1300 Subject: [PATCH] Reject a Modelfile worked example copied into a real explanation (#455) Found by Arsh while measuring PR #401: the model occasionally abandons the real finding and returns ai/Modelfile's own Example 1 or Example 2 almost word for word, unrelated to the finding it was supposedly explaining -- measured 2/120 across two independent live runs on two real findings. _is_unacceptable() did not catch this. Covers Examples 1 and 2 only, not Example 3. Both describe one fixed, never-otherwise-correct fictional scenario, so any match is unambiguously a copy. Example 3's wording is generic all-clear language a genuine status=none finding is supposed to produce, so treating it as forbidden would reject correct answers, not just copied ones. Checked for every status, not just error, since the fictional content is equally wrong glued onto any real finding. Dropped one candidate phrase while writing the tests: 'allows all traffic through with no restriction' is in Example 1's own text, but two existing tests in this file independently use that exact wording as an ordinary stub for an unrelated, genuinely wide-open ACL finding -- it is a plausible real description, not specific to the fictional scenario. The other two Example-1 phrases are specific enough to still catch the measured copy without it. Verified: 98 tests in the safety-net file (9 new), full suite 2268 passed / 13 skipped / 0 failed, mutation tested (removing the check's one call site makes 5 of the 6 phrase tests fail for the right reason; the 6th is independently caught by the existing error-only result-claim check), and 30 fresh live generations each on AC-002 and a redundant dead-rule finding through the real model: 0/30 copies slipped through, 0/30 fallbacks for any other reason. Not done: #410 (a broader, statistical drift-detection check) remains open and unaffected -- this is the small, exact check, not a substitute. --- ai/explain.py | 48 ++++++++++++++++-- tests/test_explain_safety_net.py | 86 ++++++++++++++++++++++++++++++++ 2 files changed, 131 insertions(+), 3 deletions(-) diff --git a/ai/explain.py b/ai/explain.py index 3043a96..7de1ba7 100644 --- a/ai/explain.py +++ b/ai/explain.py @@ -648,6 +648,45 @@ def remediate_with_source(finding: Dict[str, Any]) -> Optional["tuple[str, str]" ) _LEAD_TO_PATTERN = re.compile(r"\bcan\s+lead\s+to\b", re.IGNORECASE) +# ai/Modelfile's own worked examples, copied back almost verbatim instead of +# a real explanation of the finding actually given (#455). Measured live: a +# dead DNS permit and a dead SSH deny, both status="found", came back as +# Example 1's fixed, fictional scenario -- an unrelated "internal server" +# reachable over "unencrypted web traffic" -- 2 of 120 generations across +# two independent runs. Checked for EVERY status, not just "error": this +# fictional content is wrong glued onto any real finding, regardless of its +# status. +# +# Only Examples 1 and 2 are covered, not Example 3. Examples 1 and 2 each +# describe one fixed, never-otherwise-correct scenario (an unencrypted-web- +# traffic server; a crashed HQ-to-branch check), so any match is +# unambiguously a copy. Example 3's wording is generic "everything checked +# out" language that a genuine status="none" finding is SUPPOSED to produce +# -- treating it as forbidden would reject correct answers, not just copied +# ones, so it is deliberately left out. +# +# Phrases are copied verbatim from ai/Modelfile's own text -- the single +# source of truth, so a changed Modelfile goes stale here visibly (the +# phrase stops matching) rather than silently, the same trade-off +# _ALL_CLEAR_PATTERN (#456) already accepts. +# +# "allows all traffic through with no restriction" is deliberately NOT in +# this list, even though it is in Example 1's own text: found, while +# testing this guard, to be a plausible, legitimate description of any +# genuinely wide-open ACL rule, not specific to the example's fictional +# scenario -- two existing tests in this file independently use that exact +# wording as an ordinary stub for an unrelated "permit ip any any" finding. +# The other two phrases below are specific enough to the example's own +# invented scenario to still catch the measured copy without it. +_MODELFILE_EXAMPLE_PATTERN = re.compile( + r"(unencrypted web traffic that should have been blocked|" + r"which is what let this through|" + r"nothing is confirmed about whether the hq network can reach the " + r"branch network|" + r"the underlying analysis tool crashed while running the check)", + re.IGNORECASE, +) + def _looks_like_a_result_claim(text: str) -> bool: """True if `text` uses language that asserts something about the @@ -854,11 +893,14 @@ def _is_unacceptable(text: str, *, is_error: bool, finding: Optional[Dict[str, Any]] = None) -> bool: """One check, used for every status. status="error" additionally rejects any claimed result about the network; every status rejects - hedged speculation, and -- given the finding -- any address, port or - device the finding does not contain. See the module docstring's - failure-modes sections for why these are checked separately.""" + hedged speculation, a copied Modelfile worked example, and -- given + the finding -- any address, port or device the finding does not + contain. See the module docstring's failure-modes sections for why + these are checked separately.""" if is_error and _looks_like_a_result_claim(text): return True + if _MODELFILE_EXAMPLE_PATTERN.search(text): + return True if finding is not None and _names_an_invented_fact(text, finding): return True return _looks_like_speculation(text) diff --git a/tests/test_explain_safety_net.py b/tests/test_explain_safety_net.py index 0037ce5..5f643f2 100644 --- a/tests/test_explain_safety_net.py +++ b/tests/test_explain_safety_net.py @@ -1106,3 +1106,89 @@ def test_a_grounded_generation_still_comes_from_the_model(monkeypatch): monkeypatch.setattr(explain_module, "_generate", lambda finding: "Devices on 10.10.10.0/24 can reach 10.20.0.5 on port 443.") assert explain_with_source(_POLICY_FINDING)[1] == "model" + + +# --- a copied Modelfile worked example, rather than a real explanation (#455) - +# +# Found by @ARSH871-bot while measuring PR #401: the model occasionally +# abandons the real finding and returns ai/Modelfile's own "Example 1" or +# "Example 2" almost word for word, unrelated to what was actually being +# explained. Measured: 2/120 across two independent live runs on two real +# findings (a dead DNS permit, a dead SSH deny) -- not status-specific, so +# checked for every status, not just "error". +# +# Example 3 is deliberately NOT covered here -- see _MODELFILE_EXAMPLE_PATTERN's +# own comment in ai/explain.py for why: its "everything checked out" wording +# is what a genuine status="none" finding is SUPPOSED to produce. + +# The exact two sentences measured live, pinned word for word so a change to +# the phrase list cannot silently stop catching the actual failure this was +# filed for. +_MEASURED_EXAMPLE_1_COPY = ( + "The internal server can be reached over unencrypted web traffic that " + "should have been blocked. The device's rule set currently allows all " + "traffic through with no restriction, which is what let this through." +) +_MEASURED_EXAMPLE_2_COPY = ( + "This check could not be completed, so nothing is confirmed about " + "whether the HQ network can reach the branch network. The underlying " + "analysis tool crashed while running the check." +) + + +@pytest.mark.parametrize("text", [ + _MEASURED_EXAMPLE_1_COPY, + _MEASURED_EXAMPLE_2_COPY, + "unencrypted web traffic that should have been blocked, apparently.", + "a sentence ending in which is what let this through, nothing else.", + "Nothing is confirmed about whether the HQ network can reach the branch network today.", + "the underlying analysis tool crashed while running the check, again.", +]) +def test_a_copied_modelfile_example_is_rejected_for_every_status(text): + for status_is_error in (True, False): + assert explain_module._is_unacceptable( + text, is_error=status_is_error, finding=_POLICY_FINDING + ), ( + f"a Modelfile worked example was copied into the explanation, " + f"unrelated to the real finding, and not rejected: {text!r}" + ) + + +def test_the_measured_example_1_copy_is_rejected_end_to_end(monkeypatch): + """End to end, through explain_with_source(): the real failure measured + live on PR #401 -- a dead DNS permit (status="found") explained with + Example 1's fixed, fictional scenario -- now falls back instead of + reaching the user labelled as model output.""" + monkeypatch.setattr(explain_module, "_generate", + lambda finding: _MEASURED_EXAMPLE_1_COPY) + text, source = explain_with_source(_POLICY_FINDING) + assert source == "fallback" + assert "unencrypted web traffic" not in text + + +def test_a_faithful_found_explanation_is_not_caught_by_the_new_check(): + """The control: an accurate explanation that merely happens to describe + a device and some traffic must not be rejected just because it is in + the same subject area as the Modelfile's own examples.""" + faithful = ( + "The LAN can reach the internal server on port 443, because an " + "earlier rule permits that exact traffic before the required deny " + "rule is ever reached." + ) + assert not explain_module._is_unacceptable( + faithful, is_error=False, finding=_POLICY_FINDING + ) + + +def test_a_faithful_error_explanation_is_not_caught_by_the_new_check(): + """The control's error-status twin -- a legitimate status="error" + explanation never claims a result either way, so it cannot share any + wording with the earlier test's "found" sentence.""" + faithful = ( + "This check did not complete, so nothing is confirmed about the " + "internal server either way. The underlying analysis tool " + "crashed before finishing." + ) + assert not explain_module._is_unacceptable( + faithful, is_error=True, finding=_POLICY_FINDING + )