Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 45 additions & 3 deletions ai/explain.py
Original file line number Diff line number Diff line change
Expand Up @@ -648,6 +648,45 @@ def remediate_with_source(finding: Dict[str, Any]) -> Optional["tuple[str, str]"
)
_LEAD_TO_PATTERN = re.compile(r"\bcan\s+lead\s+to\b", re.IGNORECASE)

# ai/Modelfile's own worked examples, copied back almost verbatim instead of
# a real explanation of the finding actually given (#455). Measured live: a
# dead DNS permit and a dead SSH deny, both status="found", came back as
# Example 1's fixed, fictional scenario -- an unrelated "internal server"
# reachable over "unencrypted web traffic" -- 2 of 120 generations across
# two independent runs. Checked for EVERY status, not just "error": this
# fictional content is wrong glued onto any real finding, regardless of its
# status.
#
# Only Examples 1 and 2 are covered, not Example 3. Examples 1 and 2 each
# describe one fixed, never-otherwise-correct scenario (an unencrypted-web-
# traffic server; a crashed HQ-to-branch check), so any match is
# unambiguously a copy. Example 3's wording is generic "everything checked
# out" language that a genuine status="none" finding is SUPPOSED to produce
# -- treating it as forbidden would reject correct answers, not just copied
# ones, so it is deliberately left out.
#
# Phrases are copied verbatim from ai/Modelfile's own text -- the single
# source of truth, so a changed Modelfile goes stale here visibly (the
# phrase stops matching) rather than silently, the same trade-off
# _ALL_CLEAR_PATTERN (#456) already accepts.
#
# "allows all traffic through with no restriction" is deliberately NOT in
# this list, even though it is in Example 1's own text: found, while
# testing this guard, to be a plausible, legitimate description of any
# genuinely wide-open ACL rule, not specific to the example's fictional
# scenario -- two existing tests in this file independently use that exact
# wording as an ordinary stub for an unrelated "permit ip any any" finding.
# The other two phrases below are specific enough to the example's own
# invented scenario to still catch the measured copy without it.
_MODELFILE_EXAMPLE_PATTERN = re.compile(
r"(unencrypted web traffic that should have been blocked|"
r"which is what let this through|"
r"nothing is confirmed about whether the hq network can reach the "
r"branch network|"
r"the underlying analysis tool crashed while running the check)",
re.IGNORECASE,
)


def _looks_like_a_result_claim(text: str) -> bool:
"""True if `text` uses language that asserts something about the
Expand Down Expand Up @@ -854,11 +893,14 @@ def _is_unacceptable(text: str, *, is_error: bool,
finding: Optional[Dict[str, Any]] = None) -> bool:
"""One check, used for every status. status="error" additionally
rejects any claimed result about the network; every status rejects
hedged speculation, and -- given the finding -- any address, port or
device the finding does not contain. See the module docstring's
failure-modes sections for why these are checked separately."""
hedged speculation, a copied Modelfile worked example, and -- given
the finding -- any address, port or device the finding does not
contain. See the module docstring's failure-modes sections for why
these are checked separately."""
if is_error and _looks_like_a_result_claim(text):
return True
if _MODELFILE_EXAMPLE_PATTERN.search(text):
return True
if finding is not None and _names_an_invented_fact(text, finding):
return True
return _looks_like_speculation(text)
Expand Down
86 changes: 86 additions & 0 deletions tests/test_explain_safety_net.py
Original file line number Diff line number Diff line change
Expand Up @@ -1106,3 +1106,89 @@ def test_a_grounded_generation_still_comes_from_the_model(monkeypatch):
monkeypatch.setattr(explain_module, "_generate",
lambda finding: "Devices on 10.10.10.0/24 can reach 10.20.0.5 on port 443.")
assert explain_with_source(_POLICY_FINDING)[1] == "model"


# --- a copied Modelfile worked example, rather than a real explanation (#455) -
#
# Found by @ARSH871-bot while measuring PR #401: the model occasionally
# abandons the real finding and returns ai/Modelfile's own "Example 1" or
# "Example 2" almost word for word, unrelated to what was actually being
# explained. Measured: 2/120 across two independent live runs on two real
# findings (a dead DNS permit, a dead SSH deny) -- not status-specific, so
# checked for every status, not just "error".
#
# Example 3 is deliberately NOT covered here -- see _MODELFILE_EXAMPLE_PATTERN's
# own comment in ai/explain.py for why: its "everything checked out" wording
# is what a genuine status="none" finding is SUPPOSED to produce.

# The exact two sentences measured live, pinned word for word so a change to
# the phrase list cannot silently stop catching the actual failure this was
# filed for.
_MEASURED_EXAMPLE_1_COPY = (
"The internal server can be reached over unencrypted web traffic that "
"should have been blocked. The device's rule set currently allows all "
"traffic through with no restriction, which is what let this through."
)
_MEASURED_EXAMPLE_2_COPY = (
"This check could not be completed, so nothing is confirmed about "
"whether the HQ network can reach the branch network. The underlying "
"analysis tool crashed while running the check."
)


@pytest.mark.parametrize("text", [
_MEASURED_EXAMPLE_1_COPY,
_MEASURED_EXAMPLE_2_COPY,
"unencrypted web traffic that should have been blocked, apparently.",
"a sentence ending in which is what let this through, nothing else.",
"Nothing is confirmed about whether the HQ network can reach the branch network today.",
"the underlying analysis tool crashed while running the check, again.",
])
def test_a_copied_modelfile_example_is_rejected_for_every_status(text):
for status_is_error in (True, False):
assert explain_module._is_unacceptable(
text, is_error=status_is_error, finding=_POLICY_FINDING
), (
f"a Modelfile worked example was copied into the explanation, "
f"unrelated to the real finding, and not rejected: {text!r}"
)


def test_the_measured_example_1_copy_is_rejected_end_to_end(monkeypatch):
"""End to end, through explain_with_source(): the real failure measured
live on PR #401 -- a dead DNS permit (status="found") explained with
Example 1's fixed, fictional scenario -- now falls back instead of
reaching the user labelled as model output."""
monkeypatch.setattr(explain_module, "_generate",
lambda finding: _MEASURED_EXAMPLE_1_COPY)
text, source = explain_with_source(_POLICY_FINDING)
assert source == "fallback"
assert "unencrypted web traffic" not in text


def test_a_faithful_found_explanation_is_not_caught_by_the_new_check():
"""The control: an accurate explanation that merely happens to describe
a device and some traffic must not be rejected just because it is in
the same subject area as the Modelfile's own examples."""
faithful = (
"The LAN can reach the internal server on port 443, because an "
"earlier rule permits that exact traffic before the required deny "
"rule is ever reached."
)
assert not explain_module._is_unacceptable(
faithful, is_error=False, finding=_POLICY_FINDING
)


def test_a_faithful_error_explanation_is_not_caught_by_the_new_check():
"""The control's error-status twin -- a legitimate status="error"
explanation never claims a result either way, so it cannot share any
wording with the earlier test's "found" sentence."""
faithful = (
"This check did not complete, so nothing is confirmed about the "
"internal server either way. The underlying analysis tool "
"crashed before finishing."
)
assert not explain_module._is_unacceptable(
faithful, is_error=True, finding=_POLICY_FINDING
)
Loading