From 6a4cbbb742a1d5a6911f7bd0b2765607dcad3206 Mon Sep 17 00:00:00 2001 From: patelankeet2 Date: Sat, 10 Oct 2026 22:03:57 +1300 Subject: [PATCH 01/10] US-55: capture a held-out, human-rated explanation set (#346) Five real findings from the real pipeline (rtr-us5-insecure, rtr-us5-messy), regenerated live against the real model after #145's fix, rated by hand on the same accuracy/usefulness scale docs/evaluation.md Part 3 already uses. Held out in the literal sense: none of these findings or their text appear among ai/Modelfile's own worked examples. PC-001 and PC-005 are the two findings #145 was filed against -- both regenerate correctly now, confirming the fix in practice rather than by inspection. AC-002's captured explanation has a real, different defect: it describes the unreachable line's own action backwards (calls a PERMIT line a rule that 'is supposed to block'), even though the final traffic-outcome claim is correct. Checked for reproducibility (0 of 6 immediate re-runs repeated it) and filed as its own issue (#384) rather than quietly swapped for a cleaner sample or fixed inline -- fixing it here would mean rating text that no longer exists. --- .../fixtures/explanation_quality/holdout.json | 139 ++++++++++++++++++ 1 file changed, 139 insertions(+) create mode 100644 tests/fixtures/explanation_quality/holdout.json diff --git a/tests/fixtures/explanation_quality/holdout.json b/tests/fixtures/explanation_quality/holdout.json new file mode 100644 index 0000000..5b401dd --- /dev/null +++ b/tests/fixtures/explanation_quality/holdout.json @@ -0,0 +1,139 @@ +{ + "_comment": "Held-out set for US-55 (#346). Each record is one real F-1 finding, generated live from the real pipeline against tests/fixtures/rtr-us5-insecure and tests/fixtures/rtr-us5-messy, one captured explanation (ai/explain.py, real Ollama call), and independent human ratings on the same 1-5 scale docs/evaluation.md Part 3 uses. 'Held out' means these five findings are not among ai/Modelfile's own worked examples (checked: none of their IDs or exact text appear there). Captured 2026-10-11, after #145's fix landed, specifically to measure current behaviour rather than reuse docs/evaluation.md's pre-#145 text. See docs/evaluation.md Part 4 for the full writeup, including the AC-002 defect this honestly includes rather than re-rolls away.", + "rating_scale": { + "accuracy": "1 (poor) to 5 (excellent) -- does the text claim only what the finding's own evidence actually shows", + "usefulness": "1 (poor) to 5 (excellent) -- would this sentence, alone, tell someone who has never read evidence.detail what is actually wrong" + }, + "findings": [ + { + "id": "RT-050", + "fixture": "tests/fixtures/rtr-us5-insecure", + "finding": { + "id": "RT-050", + "check": "routing", + "severity": "high", + "device": "unknown", + "summary": "2 route assertion(s) could not be checked against this config", + "evidence": { + "detail": "They are written about rtr-branch, rtr-hq, which are not in this snapshot. Nothing is claimed about them either way.", + "source": "affected devices: rtr-branch, rtr-hq" + }, + "status": "error" + }, + "captured_explanation": "This check could not be completed, so nothing is confirmed about the network either way. The underlying reason: They are written about rtr-branch, rtr-hq, which are not in this snapshot. Nothing is claimed about them either way.", + "captured_explanation_source": "fallback", + "ratings": [ + { + "rater": "Ankeet", + "accuracy": 5, + "usefulness": 4, + "notes": "Deterministic fallback, so accuracy cannot be wrong by construction. Usefulness is slightly mechanical -- it restates evidence.detail almost verbatim -- but that is exactly what a safe fallback is supposed to do, and it names the two affected devices plainly." + } + ] + }, + { + "id": "AC-001", + "fixture": "tests/fixtures/rtr-us5-insecure", + "finding": { + "id": "AC-001", + "check": "access_control", + "severity": "high", + "device": "rtr-us5", + "summary": "Unencrypted web traffic reaches the internal server", + "evidence": { + "detail": "Expected DENY but got PERMIT, decided by: permit ip any any", + "source": "rtr-us5:acl_in" + }, + "status": "found" + }, + "captured_explanation": "The device's current configuration allows unencrypted web traffic to reach the internal server. This is in conflict with the policy requirement that should block such traffic, which is currently set to DENY. The reason for this discrepancy lies in the device's own configuration, specifically its rule allowing all incoming traffic without restriction.", + "captured_explanation_source": "model", + "ratings": [ + { + "rater": "Ankeet", + "accuracy": 5, + "usefulness": 4, + "notes": "Correctly attributes the problem to the device's own permit-any-any rule, and correctly states the requirement is DENY. No inversion. Slightly repetitive across its three sentences rather than genuinely re-explained." + } + ] + }, + { + "id": "PC-001", + "fixture": "tests/fixtures/rtr-us5-insecure", + "finding": { + "id": "PC-001", + "check": "policy_compliance", + "severity": "high", + "device": "rtr-us5", + "summary": "The internal network can reach servers it should not", + "evidence": { + "detail": "Flow start=rtr-us5 [10.10.10.0:49152->8.8.8.8:80 TCP (SYN)] is permitted but policy requires it to be DENIED. Decided by: permit ip any any [Rules checked: Netwise's built-in example policy (analysis/checks/policy_compliance.py) -- no policy file was supplied.]", + "source": "rtr-us5:acl_in" + }, + "status": "found" + }, + "captured_explanation": "The internal network can reach servers it should not, specifically the server at 8.8.8.8 on port 80, due to a configuration allowing unencrypted web traffic through despite policy requirements blocking it. The device's own configuration is permitting this traffic, which contradicts its written policy that forbids it.", + "captured_explanation_source": "model", + "ratings": [ + { + "rater": "Ankeet", + "accuracy": 5, + "usefulness": 4, + "notes": "This is one of the two findings #145 was filed against (the pre-#145 text here scored 4/3, 2/3, 2/2, 2/2 across the four raters in docs/evaluation.md Part 3, because the old text said 'the policy currently permits this unauthorized access' -- backwards). Regenerated after the fix: now correctly says the device permits traffic 'despite policy requirements blocking it' and that the config 'contradicts its written policy that forbids it'. The inversion is gone. Names the exact destination (8.8.8.8:80), which the old text also did." + } + ] + }, + { + "id": "AC-002", + "fixture": "tests/fixtures/rtr-us5-messy", + "finding": { + "id": "AC-002", + "check": "access_control", + "severity": "low", + "device": "rtr-us5", + "summary": "ACL rule never takes effect in acl_in", + "evidence": { + "detail": "Unreachable line: permit udp 10.10.10.0 0.0.0.255 host 218.8.104.58 eq domain (action PERMIT). Blocked by: deny ip 10.10.10.0 0.0.0.255 any. Reason: BLOCKING_LINES", + "source": "rtr-us5: acl_in" + }, + "status": "found" + }, + "captured_explanation": "The device's ACL rule, which is supposed to block traffic from a specific IP address, never actually takes effect because it is overridden by another rule that blocks all traffic with the same destination IP address. As a result, even though the unreachable line in the ACL rule would normally deny traffic matching its pattern, the blocking rule always wins and prevents any traffic from being allowed.", + "captured_explanation_source": "model", + "ratings": [ + { + "rater": "Ankeet", + "accuracy": 2, + "usefulness": 2, + "notes": "REAL DEFECT, kept rather than re-rolled -- see issue #384 and docs/evaluation.md Part 4. evidence.detail says the unreachable line's own action is PERMIT; this text calls it a rule 'which is supposed to block traffic' and says it 'would normally deny traffic matching its pattern' -- backwards about the dead line's own action, even though the final-outcome claim (traffic ends up denied) is correct. A reader walks away with the wrong idea of what kind of rule became unreachable. Checked for reproducibility: 0 of 6 immediate re-generations of this exact finding repeated it (see #384), so this is the honest, unmodified first capture, not the typical case -- but it is real, and a measurement that quietly swapped it for a cleaner sample would defeat the point of measuring at all." + } + ] + }, + { + "id": "PC-005", + "fixture": "tests/fixtures/rtr-us5-messy", + "finding": { + "id": "PC-005", + "check": "policy_compliance", + "severity": "medium", + "device": "rtr-us5", + "summary": "HTTPS to the internal server is blocked, so the service is unreachable", + "evidence": { + "detail": "Flow start=rtr-us5 [10.10.10.0:49152->10.20.0.5:443 TCP (SYN)] is denied but policy requires it to be PERMITTED. Decided by: deny ip 10.10.10.0 0.0.0.255 any [Rules checked: Netwise's built-in example policy (analysis/checks/policy_compliance.py) -- no policy file was supplied.]", + "source": "rtr-us5:acl_in" + }, + "status": "found" + }, + "captured_explanation": "The internal server can be reached over HTTPS traffic, but the device's configuration blocks it. The written policy actually requires this traffic to be allowed, so the blockage is due to the device's own configuration rather than the policy itself. As a result, the service remains unreachable because the required traffic is being denied by the device's rules.", + "captured_explanation_source": "model", + "ratings": [ + { + "rater": "Ankeet", + "accuracy": 5, + "usefulness": 3, + "notes": "This is the other finding #145 was filed against -- the pre-#145 text scored 3/2, 1/2, 1/2, 1/2 across the four raters, because it said 'a specific policy statement that requires blocking of HTTPS traffic', the opposite of the evidence. Regenerated after the fix: now correctly says the device's configuration blocks it and the policy 'requires this traffic to be allowed'. The inversion is gone -- accuracy moves from the old mean of 1.5 to 5. Usefulness held at 3, not higher: the first sentence ('can be reached over HTTPS... but blocks it') still asks the reader to resolve an apparent contradiction before the next two sentences clarify it, the same readability risk multiple raters flagged on the old text, now just no longer paired with a false claim underneath it." + } + ] + } + ] +} From b15900dd47a46a1cd4c027da71e298568c457de0 Mon Sep 17 00:00:00 2001 From: patelankeet2 Date: Sat, 10 Oct 2026 22:05:09 +1300 Subject: [PATCH 02/10] US-55: score the held-out explanation set (#346) tools/explanation_quality.py -- pure arithmetic over the frozen held-out set, mean and minimum of each rating dimension separately (not blended into one number, same reasoning docs/evaluation.md already gives for the controls table and F-4 generally). Runs both documented ways; imports neither analysis, ai, nor web, so it needs no sys.path guard, same exemption pfsense_shape.py already has. --- tools/explanation_quality.py | 127 +++++++++++++++++++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 tools/explanation_quality.py diff --git a/tools/explanation_quality.py b/tools/explanation_quality.py new file mode 100644 index 0000000..a821e41 --- /dev/null +++ b/tools/explanation_quality.py @@ -0,0 +1,127 @@ +"""Score the held-out explanation set for US-55 (#346). + +WHAT THIS DOES + Pure arithmetic over tests/fixtures/explanation_quality/holdout.json -- + the mean and minimum of each rating dimension (accuracy, usefulness) + across every rater and finding in that file. Nothing here calls Ollama + or Batfish, so this runs the same everywhere: a laptop, CI, two years + from now. "Repeatable score" means exactly this -- the same frozen + input always produces the same number, not that a live model's output + is repeatable (it is not, see ai/explain.py's own docstring). + +WHY A SEPARATE NUMBER PER DIMENSION, NOT ONE BLENDED SCORE + docs/evaluation.md already makes this argument twice (the controls + table, F-4 itself): a single averaged number hides exactly the case + that matters. Blending accuracy and usefulness into one figure would + let a high-usefulness, low-accuracy explanation (or the reverse) read + as "fine" when it is not. Reported separately instead. + +WHY MINIMUM, NOT JUST MEAN + The mean alone would have hidden AC-002's accuracy=2 behind four other + findings at 5, printing a healthy-looking 4.4 while one finding in the + set is actually a confirmed defect (#384). The minimum is what a + threshold decision should look at -- see docs/evaluation.md Part 4 and + EXPLANATION_QUALITY_THRESHOLD in ai/explain.py for what this number is + used to decide. + +RUN IT BY HAND + python -m tools.explanation_quality + python tools/explanation_quality.py +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, Dict, List + +#: Resolved relative to this file, not the caller's working directory -- +#: so this runs correctly from the repository root regardless of which of +#: the two documented invocation forms launched it. +DEFAULT_HOLDOUT_PATH = ( + Path(__file__).resolve().parent.parent + / "tests" / "fixtures" / "explanation_quality" / "holdout.json" +) + + +def load_holdout(path: Path = DEFAULT_HOLDOUT_PATH) -> List[Dict[str, Any]]: + """The held-out set's `findings` list, read from `path`. + + Raises FileNotFoundError / json.JSONDecodeError on a missing or + malformed file rather than returning an empty result -- a silently + empty score (mean of nothing) must never be mistaken for "nothing + wrong was found", the same F-4 distinction CLAUDE.md requires of the + product itself. + """ + with open(path, encoding="utf-8") as handle: + data = json.load(handle) + return data["findings"] + + +def score_holdout(findings: List[Dict[str, Any]]) -> Dict[str, Any]: + """Aggregate accuracy/usefulness across every rating in `findings`. + + Returns a dict with the overall mean and minimum of each dimension, + the number of findings and the total number of individual ratings + counted (more than one rater per finding is expected to grow over + time -- see docs/evaluation.md Part 4), and a per-finding breakdown + (that finding's own mean of each dimension) so a caller can see WHICH + finding is dragging the minimum down, not just that one is. + + Raises ValueError on an empty `findings` list -- the same reasoning as + load_holdout()'s docstring: a mean of zero ratings is not a score of + zero, it is not a score at all, and must never print as one. + """ + if not findings: + raise ValueError("cannot score an empty held-out set") + + accuracies: List[int] = [] + usefulnesses: List[int] = [] + per_finding: List[Dict[str, Any]] = [] + + for finding in findings: + ratings = finding["ratings"] + finding_accuracies = [r["accuracy"] for r in ratings] + finding_usefulnesses = [r["usefulness"] for r in ratings] + accuracies.extend(finding_accuracies) + usefulnesses.extend(finding_usefulnesses) + per_finding.append({ + "id": finding["id"], + "mean_accuracy": sum(finding_accuracies) / len(finding_accuracies), + "mean_usefulness": sum(finding_usefulnesses) / len(finding_usefulnesses), + "n_ratings": len(ratings), + }) + + return { + "n_findings": len(findings), + "n_ratings": len(accuracies), + "mean_accuracy": sum(accuracies) / len(accuracies), + "min_accuracy": min(accuracies), + "mean_usefulness": sum(usefulnesses) / len(usefulnesses), + "min_usefulness": min(usefulnesses), + "per_finding": per_finding, + } + + +def _format_report(score: Dict[str, Any]) -> str: + """Plain-text rendering of score_holdout()'s return value, for the + command-line entry point below. Not used by the test suite, which + asserts on the dict directly rather than parsing text back out of it.""" + lines = [ + f"{score['n_findings']} findings, {score['n_ratings']} ratings", + f"accuracy: mean {score['mean_accuracy']:.2f} min {score['min_accuracy']}", + f"usefulness: mean {score['mean_usefulness']:.2f} min {score['min_usefulness']}", + "", + "per finding:", + ] + for row in score["per_finding"]: + lines.append( + f" {row['id']:<8} accuracy {row['mean_accuracy']:.1f} " + f"usefulness {row['mean_usefulness']:.1f} " + f"({row['n_ratings']} rating(s))" + ) + return "\n".join(lines) + + +if __name__ == "__main__": + print(_format_report(score_holdout(load_holdout()))) From 471c2e0d483b395b9639be76e6dc527e83f99877 Mon Sep 17 00:00:00 2001 From: patelankeet2 Date: Sat, 10 Oct 2026 22:09:34 +1300 Subject: [PATCH 03/10] US-55: document the fallback-preference threshold, add tests (#346) EXPLANATION_QUALITY_THRESHOLD (mean_accuracy 4.0, mean_usefulness 3.0), deliberately based on the mean rather than the minimum -- a single low-scoring finding is a correctness bug to fix for that shape (same as #52, #145, #384), not a reason to disable the model for every finding. No runtime auto-check exists: crossing this is detected by a human re-running the held-out measurement, since there is no honest way to score one live generation without another model judging it. tests/test_explanation_quality.py: synthetic cases for the multi-rater/multi-finding averaging the real fixture's single-rater data can't exercise, plus tests pinning the real committed holdout's published numbers against the documented threshold. python -m pytest tests/test_explanation_quality.py -v -> 8 passed. --- tests/test_explanation_quality.py | 134 ++++++++++++++++++++++++++++++ tools/explanation_quality.py | 41 ++++++++- 2 files changed, 171 insertions(+), 4 deletions(-) create mode 100644 tests/test_explanation_quality.py diff --git a/tests/test_explanation_quality.py b/tests/test_explanation_quality.py new file mode 100644 index 0000000..45433f6 --- /dev/null +++ b/tests/test_explanation_quality.py @@ -0,0 +1,134 @@ +""" +Netwise -- tests for tools/explanation_quality.py (US-55, #346). + +WHY THIS FILE EXISTS + score_holdout() is pure arithmetic, so most of what is worth testing is + the arithmetic itself (synthetic cases below) -- but the whole POINT of + #346 is a published number, and a test that only exercises synthetic + data would never notice if the committed fixture itself drifted or the + threshold stopped holding. So this file also pins the real committed + holdout.json and asserts its score against the documented threshold + (see docs/evaluation.md Part 4 and ai/explain.py's + EXPLANATION_QUALITY_THRESHOLD), the same way other "published number" + tests in this project assert on real output rather than a mock. + + Needs neither Batfish nor Ollama -- the holdout set is a frozen capture, + not a live generation. + +RUN + pytest tests/test_explanation_quality.py -v +""" + +import pytest + +from tools.explanation_quality import ( + EXPLANATION_QUALITY_THRESHOLD, + load_holdout, + score_holdout, +) + +# --------------------------------------------------------------------------- +# Synthetic cases -- the real fixture currently has exactly one rater per +# finding, which would never catch a bug in averaging MULTIPLE raters' +# scores for the same finding. These exercise that path directly. +# --------------------------------------------------------------------------- + + +def _finding(fid, accuracy_usefulness_pairs): + return { + "id": fid, + "ratings": [ + {"rater": f"r{i}", "accuracy": acc, "usefulness": use} + for i, (acc, use) in enumerate(accuracy_usefulness_pairs) + ], + } + + +def test_score_averages_multiple_raters_on_one_finding(): + findings = [_finding("X-001", [(5, 5), (3, 1)])] + score = score_holdout(findings) + assert score["mean_accuracy"] == 4.0 + assert score["mean_usefulness"] == 3.0 + assert score["min_accuracy"] == 3 + assert score["min_usefulness"] == 1 + assert score["n_findings"] == 1 + assert score["n_ratings"] == 2 + + +def test_score_averages_across_multiple_findings(): + findings = [ + _finding("X-001", [(5, 5)]), + _finding("X-002", [(1, 1)]), + ] + score = score_holdout(findings) + assert score["mean_accuracy"] == 3.0 + assert score["min_accuracy"] == 1 + assert score["n_findings"] == 2 + assert score["n_ratings"] == 2 + + +def test_per_finding_breakdown_reports_each_findings_own_mean(): + findings = [ + _finding("GOOD", [(5, 5)]), + _finding("BAD", [(1, 2)]), + ] + score = score_holdout(findings) + by_id = {row["id"]: row for row in score["per_finding"]} + assert by_id["GOOD"]["mean_accuracy"] == 5.0 + assert by_id["BAD"]["mean_accuracy"] == 1.0 + assert by_id["BAD"]["n_ratings"] == 1 + + +def test_scoring_an_empty_set_raises_rather_than_returning_a_fake_zero(): + """An empty mean is not a score of zero -- it is not a score at all. + Same F-4 distinction the product itself enforces (never/error), applied + to this measurement tool.""" + with pytest.raises(ValueError): + score_holdout([]) + + +# --------------------------------------------------------------------------- +# The real, committed held-out set -- the published number itself. +# --------------------------------------------------------------------------- + + +def test_the_committed_holdout_set_loads(): + findings = load_holdout() + assert len(findings) == 5 + assert {f["id"] for f in findings} == { + "RT-050", "AC-001", "PC-001", "AC-002", "PC-005", + } + + +def test_the_published_score_matches_the_committed_ratings(): + """Pins the current, real number. A change here means either the + holdout set or its ratings changed -- re-measure deliberately + (docs/evaluation.md Part 4), don't just update this assertion to make + it pass.""" + score = score_holdout(load_holdout()) + assert score["n_findings"] == 5 + assert score["n_ratings"] == 5 + assert score["mean_accuracy"] == pytest.approx(4.4) + assert score["min_accuracy"] == 2 + assert score["mean_usefulness"] == pytest.approx(3.4) + assert score["min_usefulness"] == 2 + + +def test_the_low_scoring_finding_is_identifiable_in_the_breakdown(): + """AC-002's accuracy=2 is a real, filed defect (#384), not noise. The + per-finding breakdown is what makes it findable rather than letting the + 4.4 mean hide it -- see tools/explanation_quality.py's own module + docstring for why mean alone was rejected as the published number.""" + score = score_holdout(load_holdout()) + by_id = {row["id"]: row for row in score["per_finding"]} + assert by_id["AC-002"]["mean_accuracy"] == 2.0 + + +def test_current_mean_quality_meets_the_documented_threshold(): + """The headline, aggregate decision point: see + EXPLANATION_QUALITY_THRESHOLD's own docstring and docs/evaluation.md + Part 4 for what crossing this means and why it is a documented, + manually-reviewed decision rather than a runtime auto-disable.""" + score = score_holdout(load_holdout()) + assert score["mean_accuracy"] >= EXPLANATION_QUALITY_THRESHOLD["mean_accuracy"] + assert score["mean_usefulness"] >= EXPLANATION_QUALITY_THRESHOLD["mean_usefulness"] diff --git a/tools/explanation_quality.py b/tools/explanation_quality.py index a821e41..ab3aeb6 100644 --- a/tools/explanation_quality.py +++ b/tools/explanation_quality.py @@ -19,10 +19,9 @@ WHY MINIMUM, NOT JUST MEAN The mean alone would have hidden AC-002's accuracy=2 behind four other findings at 5, printing a healthy-looking 4.4 while one finding in the - set is actually a confirmed defect (#384). The minimum is what a - threshold decision should look at -- see docs/evaluation.md Part 4 and - EXPLANATION_QUALITY_THRESHOLD in ai/explain.py for what this number is - used to decide. + set is actually a confirmed defect (#384). The minimum is reported for + that reason, but see EXPLANATION_QUALITY_THRESHOLD below for why the + THRESHOLD decision is deliberately based on the mean, not the minimum. RUN IT BY HAND python -m tools.explanation_quality @@ -43,6 +42,40 @@ / "tests" / "fixtures" / "explanation_quality" / "holdout.json" ) +#: Acceptance criterion 3 of #346: "a documented threshold below which the +#: deterministic fallback is preferred". Grounded in the first real +#: measurement (see docs/evaluation.md Part 4): mean_accuracy 4.4, +#: mean_usefulness 3.4 -- so these two numbers sit comfortably below the +#: current measurement rather than just above it, which would make the +#: threshold meaningless. +#: +#: DELIBERATELY ON THE MEAN, NOT THE MINIMUM +#: A single low-scoring finding (AC-002, accuracy=2, #384) is treated +#: as a correctness BUG to fix for that one evidence shape -- the same +#: way #52 and #145 were each fixed for their one shape, never by +#: disabling the model for every finding. Tying this threshold to the +#: minimum would mean one narrow, already-understood defect forces +#: every explanation onto the deterministic fallback, which is a far +#: bigger behaviour change than the finding justifies. The mean is what +#: answers the real question this threshold exists for: has overall +#: quality collapsed broadly, not did one shape have one bad day. +#: +#: WHY THIS IS A DOCUMENTED NUMBER AND NOT A RUNTIME CHECK +#: There is no way to score a single live generation's quality +#: automatically without another model judging it -- which would just +#: move the hallucination risk up one level, not remove it, the same +#: reasoning CLAUDE.md constraint 2 already applies to explanation +#: generation itself. So crossing this threshold is detected the only +#: honest way available: a human re-runs the held-out measurement (same +#: method as docs/evaluation.md Part 4), and if the new mean falls +#: below either number here, that is the trigger to seriously +#: reconsider defaulting ai/explain.py to the fallback path for every +#: finding, not an automatic flag flip. +EXPLANATION_QUALITY_THRESHOLD = { + "mean_accuracy": 4.0, + "mean_usefulness": 3.0, +} + def load_holdout(path: Path = DEFAULT_HOLDOUT_PATH) -> List[Dict[str, Any]]: """The held-out set's `findings` list, read from `path`. From ab9de5b9c55e82c117e06986d16bd372cccd8153 Mon Sep 17 00:00:00 2001 From: patelankeet2 Date: Sat, 10 Oct 2026 22:15:08 +1300 Subject: [PATCH 04/10] US-55: document the held-out evaluation and its threshold (#346) docs/evaluation.md Part 4 -- the full writeup: why this round regenerated rather than reused Part 3's pre-#145 text, confirmation that both PC-001/PC-005 inversions #145 was filed against are gone on live regeneration, the new AC-002 defect found and filed as #384 (kept in the held-out set rather than re-rolled for a cleaner sample), the published score, and why EXPLANATION_QUALITY_THRESHOLD is based on the mean rather than the minimum and is a documented decision point rather than a runtime check. Explicitly left open for the team: a second independent rater on this set, the same way Part 3 grew from one rater to four. --- docs/evaluation.md | 162 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 162 insertions(+) diff --git a/docs/evaluation.md b/docs/evaluation.md index 4c02d0f..52c6f41 100644 --- a/docs/evaluation.md +++ b/docs/evaluation.md @@ -673,3 +673,165 @@ it is the strongest thing this round produced. @patelankeet2 filed #145 from — it is not, `AC-001` is two-clause and reads correctly. Same conclusion, two routes, four findings between them. That is a better result than agreement would have been. + +--- + +## Part 4 - a held-out, repeatable score for the explanation layer (US-55, #346) + +Measured 11 October 2026. Story: US-55 (#346). Part 3 above was the first +round of human rating and found a real bug (#145); this part turns that +exercise into something that runs in CI without Ollama, the same way Part 1 +turned "does Netwise find the planted flaws" into a repeatable measurement +rather than a one-off demo. + +Not written from scratch: the same five findings Part 3 already used +(`RT-050`, `AC-001`, `PC-001`, `AC-002`, `PC-005`), because they are already +real, already span three checks and both a `found` and an `error`, and are +already outside `ai/Modelfile`'s own worked examples (checked directly - none +of the Modelfile's examples share an id or exact wording with these five), so +they qualify as held out without inventing a second set. + +### Why this round regenerated rather than reused Part 3's text + +Part 3's text predates #145's fix. Reusing it would pin a known-fixed bug's +output forever and call that "current behaviour", which is not honest. So +all five explanations were regenerated live, against the real pipeline and +the real model, on 11 October - after #145 landed, specifically to measure +what the system does today. + +### What changed since #145 + +``` +PC-001 accuracy, old mean 2.5 (4,2,2,2 across four raters) -> new 5 + "The policy currently permits this unauthorized access" (backwards) + -> "despite policy requirements blocking it... contradicts its + written policy that forbids it" (correct) + +PC-005 accuracy, old mean 1.5 (3,1,1,1 across four raters) -> new 5 + "a policy statement that requires blocking of HTTPS traffic" (backwards) + -> "the written policy actually requires this traffic to be + allowed" (correct) +``` + +Both inversions #145 was filed against are gone on this regeneration. That is +not proof they can never recur - `ai/explain.py`'s own docstring is explicit +that a 3B model's output is not perfectly repeatable - but it is a real, +measured confirmation of the fix, not an assumption that closing the issue +meant it worked. + +### A different, new defect, found by doing this rather than assumed away + +`AC-002`'s regenerated text has its own problem, unrelated to #145: + +``` +detail: Unreachable line: permit udp ... (action PERMIT). Blocked by: + deny ip ... any. Reason: BLOCKING_LINES + +explanation: "...which is supposed to block traffic from a specific IP + address... even though the unreachable line in the ACL rule + would normally deny traffic matching its pattern..." +``` + +The unreachable line's own action is `PERMIT`. The text describes it twice as +if it were a block/deny rule. The final-outcome claim (the traffic ends up +denied, because the blocking rule wins) is correct - this is not #52 +repeating. What is backwards is the dead line's own stated action, a +different fact. + +Checked for reproducibility before concluding anything: six more immediate +live regenerations of the same finding, zero repeated it. One inversion in +eight live generations total. Filed as +[#384](https://github.com/ARSH871-bot/Netwise/issues/384) with the full +repro log, a suggested fix (extend `_compute_dead_rule_outcome()` to also +name the dead line's own action, the same way the other two computed-outcome +functions already name both sides of their fact), and the same discipline +#145 itself set: **not fixed here.** Fixing it now would mean rating text +that no longer exists by the time anyone reads this table. + +### The held-out set and its ratings + +One rater so far (Ankeet) - see +[`tests/fixtures/explanation_quality/holdout.json`](../tests/fixtures/explanation_quality/holdout.json) +for the full captured findings, explanations and reasoning behind each +score. Same 1-5 scale as Part 3 (accuracy: does the text claim only what the +evidence shows; usefulness: would this sentence alone tell someone who never +read `evidence.detail` what is wrong). + +| id | accuracy | usefulness | note | +|---|---|---|---| +| RT-050 | 5 | 4 | Deterministic fallback, cannot be wrong by construction. | +| AC-001 | 5 | 4 | Correct, no inversion. | +| PC-001 | 5 | 4 | Regenerated post-#145: inversion gone. | +| AC-002 | **2** | **2** | **Real defect, kept rather than re-rolled - see #384.** | +| PC-005 | 5 | 3 | Regenerated post-#145: inversion gone. First-sentence phrasing still asks the reader to resolve an apparent contradiction before it resolves itself - a readability risk, not a false claim this time. | + +@ARSH871-bot @shubhamkataria2005 @SamikaPerera - a second independent rating +on this set, the same way Part 3 grew from one rater to four, would turn this +from a first round into the same strength of evidence Part 3 reached. Not +required to use the tool below; the fixture format is plain JSON. + +### The repeatable score + +```bash +python -m tools.explanation_quality +``` + +``` +5 findings, 5 ratings +accuracy: mean 4.40 min 2 +usefulness: mean 3.40 min 2 +``` + +**Published, and pinned in `tests/test_explanation_quality.py`** so CI +catches it if the committed ratings ever drift from what this document +claims - the same "generated artefact can go stale between the commit and +the merge" risk `docs/traceability.md`'s own history already names in +`CLAUDE.md` §11. This needs neither Batfish nor Ollama: the held-out set is a +frozen capture, not a live call, so "repeatable" means exactly what it says - +the same input always produces the same number. + +**Reported as two numbers, never blended into one**, and as a mean +alongside a minimum rather than the mean alone - for the same reason the +controls table earlier in this document separates "checked clean" from +"could not check" instead of reporting one combined pass rate: a single +figure here would have hidden `AC-002`'s accuracy=2 behind four findings at +5, printing a reassuring 4.4 while one finding in the set is a confirmed, +filed defect. + +### The documented threshold (acceptance criterion 3) + +`tools/explanation_quality.py` defines `EXPLANATION_QUALITY_THRESHOLD`: +mean accuracy 4.0, mean usefulness 3.0 - both below the measurement above, +so the threshold is a real tripwire rather than a number chosen to always +pass. + +**Deliberately based on the mean, not the minimum.** `AC-002`'s accuracy=2 is +a correctness bug in one evidence shape, the same class of thing #52 and +#145 already were, and both of those were fixed for their one shape rather +than by turning the model off for every finding. Tying this threshold to the +minimum would mean one narrow, already-understood, 1-in-8 defect forces every +explanation in the product onto the deterministic fallback - a far bigger +change than the finding justifies. The mean is what actually answers the +question this threshold exists for: has quality collapsed broadly, not did +one shape have one bad day. + +**Why this is a documented number, not a runtime check.** There is no honest +way to score one live generation's quality automatically without another +model judging it, which would move the hallucination risk up a level rather +than remove it - the same reasoning `CLAUDE.md` constraint 2 already applies +to explanation generation itself. So crossing this threshold is something a +human detects, by re-running this measurement (same method as this section), +not something `ai/explain.py` checks on every call. If a future re-run ever +reports a mean below either number, that is the trigger to seriously +reconsider defaulting the whole explanation layer to the deterministic +fallback - a team decision, not an automatic flag flip. + +### What this does not show + +Same discipline as Part 1's own honesty section. One rater, five findings, +is a first round - Part 3 only reached four independent raters because +people kept adding rows after the first. A single bad draw from the model +(`AC-002`) was caught here only because this round happened to regenerate +rather than reuse old text; the held-out set's SIZE is still small enough +that a rarer defect could sit undetected for a while. Growing the set and +adding raters is explicitly left open above rather than claimed as finished. From 4be3c266b2de9065e08a878f1d8d2f93b43f05f8 Mon Sep 17 00:00:00 2001 From: patelankeet2 Date: Sat, 10 Oct 2026 22:25:01 +1300 Subject: [PATCH 05/10] US-55: fix a wrong date (10 October, not 11) in the new evaluation docs --- docs/evaluation.md | 4 ++-- tests/fixtures/explanation_quality/holdout.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/evaluation.md b/docs/evaluation.md index 52c6f41..160f85c 100644 --- a/docs/evaluation.md +++ b/docs/evaluation.md @@ -678,7 +678,7 @@ would have been. ## Part 4 - a held-out, repeatable score for the explanation layer (US-55, #346) -Measured 11 October 2026. Story: US-55 (#346). Part 3 above was the first +Measured 10 October 2026. Story: US-55 (#346). Part 3 above was the first round of human rating and found a real bug (#145); this part turns that exercise into something that runs in CI without Ollama, the same way Part 1 turned "does Netwise find the planted flaws" into a repeatable measurement @@ -696,7 +696,7 @@ they qualify as held out without inventing a second set. Part 3's text predates #145's fix. Reusing it would pin a known-fixed bug's output forever and call that "current behaviour", which is not honest. So all five explanations were regenerated live, against the real pipeline and -the real model, on 11 October - after #145 landed, specifically to measure +the real model, on 10 October - after #145 landed, specifically to measure what the system does today. ### What changed since #145 diff --git a/tests/fixtures/explanation_quality/holdout.json b/tests/fixtures/explanation_quality/holdout.json index 5b401dd..bf53acf 100644 --- a/tests/fixtures/explanation_quality/holdout.json +++ b/tests/fixtures/explanation_quality/holdout.json @@ -1,5 +1,5 @@ { - "_comment": "Held-out set for US-55 (#346). Each record is one real F-1 finding, generated live from the real pipeline against tests/fixtures/rtr-us5-insecure and tests/fixtures/rtr-us5-messy, one captured explanation (ai/explain.py, real Ollama call), and independent human ratings on the same 1-5 scale docs/evaluation.md Part 3 uses. 'Held out' means these five findings are not among ai/Modelfile's own worked examples (checked: none of their IDs or exact text appear there). Captured 2026-10-11, after #145's fix landed, specifically to measure current behaviour rather than reuse docs/evaluation.md's pre-#145 text. See docs/evaluation.md Part 4 for the full writeup, including the AC-002 defect this honestly includes rather than re-rolls away.", + "_comment": "Held-out set for US-55 (#346). Each record is one real F-1 finding, generated live from the real pipeline against tests/fixtures/rtr-us5-insecure and tests/fixtures/rtr-us5-messy, one captured explanation (ai/explain.py, real Ollama call), and independent human ratings on the same 1-5 scale docs/evaluation.md Part 3 uses. 'Held out' means these five findings are not among ai/Modelfile's own worked examples (checked: none of their IDs or exact text appear there). Captured 2026-10-10, after #145's fix landed, specifically to measure current behaviour rather than reuse docs/evaluation.md's pre-#145 text. See docs/evaluation.md Part 4 for the full writeup, including the AC-002 defect this honestly includes rather than re-rolls away.", "rating_scale": { "accuracy": "1 (poor) to 5 (excellent) -- does the text claim only what the finding's own evidence actually shows", "usefulness": "1 (poor) to 5 (excellent) -- would this sentence, alone, tell someone who has never read evidence.detail what is actually wrong" From 1ea50c07f7880aab83b8253b31b3063b44a6e637 Mon Sep 17 00:00:00 2001 From: patelankeet2 Date: Mon, 12 Oct 2026 13:38:44 +1300 Subject: [PATCH 06/10] US-55: score model-written explanations only, not fallback (review) Found by @ARSH871-bot: RT-050's fallback-sourced capture sat in the same aggregate as model-written text, so the published score could be met by a held-out set with no model output at all (five fallback copies alone would score 5.00/4.00, comfortably above the threshold). EXPLANATION_QUALITY_THRESHOLD exists to judge the model, and a fallback explanation cannot be wrong by construction, so it says nothing about that question. RT-050 is also status="error", which web/main.py never attaches an explanation to, so no user sees it regardless of source. score_holdout() now filters to captured_explanation_source == "model" for the headline numbers, raises ValueError if none remain (a fallback-only set included), and reports any fallback-sourced entries under their own "fallback" key rather than dropping them. Published numbers move from 5 findings/4.40/3.40 to 4 model-written findings/4.25/3.25 -- the conclusion (clears the 4.0/3.0 threshold) does not change, but the number it is measured from now actually measures what the threshold exists to judge. python -m pytest tests/test_explanation_quality.py -v -> 13 passed (was 8). --- tests/test_explanation_quality.py | 87 +++++++++++++++++++++++++--- tools/explanation_quality.py | 94 +++++++++++++++++++++++++------ 2 files changed, 154 insertions(+), 27 deletions(-) diff --git a/tests/test_explanation_quality.py b/tests/test_explanation_quality.py index 45433f6..c6bba81 100644 --- a/tests/test_explanation_quality.py +++ b/tests/test_explanation_quality.py @@ -34,9 +34,10 @@ # --------------------------------------------------------------------------- -def _finding(fid, accuracy_usefulness_pairs): +def _finding(fid, accuracy_usefulness_pairs, source="model"): return { "id": fid, + "captured_explanation_source": source, "ratings": [ {"rater": f"r{i}", "accuracy": acc, "usefulness": use} for i, (acc, use) in enumerate(accuracy_usefulness_pairs) @@ -87,6 +88,58 @@ def test_scoring_an_empty_set_raises_rather_than_returning_a_fake_zero(): score_holdout([]) +# --------------------------------------------------------------------------- +# Fallback captures must never count toward the model score (found in +# review, @ARSH871-bot). A fallback-sourced explanation cannot be wrong by +# construction, so a set with no model output at all must not be able to +# pass the threshold that exists to judge the MODEL. +# --------------------------------------------------------------------------- + + +def test_a_holdout_set_of_only_fallback_entries_cannot_be_scored(): + """THE CASE THE REVIEW NAMED DIRECTLY: five perfect fallback ratings + must not produce a passing model score -- there is no model score to + produce, and this must say so loudly rather than quietly report 5.0.""" + findings = [ + _finding("X-001", [(5, 5)], source="fallback"), + _finding("X-002", [(5, 5)], source="fallback"), + ] + with pytest.raises(ValueError): + score_holdout(findings) + + +def test_fallback_entries_are_reported_separately_not_blended_in(): + findings = [ + _finding("MODEL-ONE", [(3, 3)], source="model"), + _finding("FALLBACK-ONE", [(5, 5)], source="fallback"), + ] + score = score_holdout(findings) + # The headline numbers are the model entry alone. + assert score["n_findings"] == 1 + assert score["mean_accuracy"] == 3.0 + assert {row["id"] for row in score["per_finding"]} == {"MODEL-ONE"} + # The fallback entry exists in its own section, not folded in. + assert score["fallback"]["n_findings"] == 1 + assert score["fallback"]["mean_accuracy"] == 5.0 + assert {row["id"] for row in score["fallback"]["per_finding"]} == {"FALLBACK-ONE"} + + +def test_a_set_with_no_fallback_entries_has_no_fallback_key(): + findings = [_finding("MODEL-ONE", [(5, 5)], source="model")] + score = score_holdout(findings) + assert "fallback" not in score + + +def test_a_missing_source_field_is_treated_as_not_model(): + """Defensive: an older or malformed record with no + captured_explanation_source at all must not be silently scored as if + it were model-written -- it lands in the fallback bucket, same as any + other non-"model" value.""" + findings = [{"id": "X-001", "ratings": [{"rater": "r", "accuracy": 5, "usefulness": 5}]}] + with pytest.raises(ValueError): + score_holdout(findings) + + # --------------------------------------------------------------------------- # The real, committed held-out set -- the published number itself. # --------------------------------------------------------------------------- @@ -101,17 +154,33 @@ def test_the_committed_holdout_set_loads(): def test_the_published_score_matches_the_committed_ratings(): - """Pins the current, real number. A change here means either the - holdout set or its ratings changed -- re-measure deliberately - (docs/evaluation.md Part 4), don't just update this assertion to make - it pass.""" + """Pins the current, real number -- MODEL-WRITTEN entries only (RT-050 + is fallback-sourced and reported separately, see + tools/explanation_quality.py's score_holdout() docstring). A change + here means either the holdout set or its ratings changed -- re-measure + deliberately (docs/evaluation.md Part 4), don't just update this + assertion to make it pass.""" score = score_holdout(load_holdout()) - assert score["n_findings"] == 5 - assert score["n_ratings"] == 5 - assert score["mean_accuracy"] == pytest.approx(4.4) + assert score["n_findings"] == 4 + assert score["n_ratings"] == 4 + assert score["mean_accuracy"] == pytest.approx(4.25) assert score["min_accuracy"] == 2 - assert score["mean_usefulness"] == pytest.approx(3.4) + assert score["mean_usefulness"] == pytest.approx(3.25) assert score["min_usefulness"] == 2 + assert {row["id"] for row in score["per_finding"]} == { + "AC-001", "PC-001", "AC-002", "PC-005", + } + + +def test_rt_050_is_reported_as_fallback_not_scored_against_the_threshold(): + """RT-050 is status="error" (never shown an explanation in the real + app -- see web/main.py's attachment loop) AND fallback-sourced (cannot + be wrong by construction). Both are reasons it must not count toward + whether the MODEL clears the threshold.""" + score = score_holdout(load_holdout()) + assert "RT-050" not in {row["id"] for row in score["per_finding"]} + assert score["fallback"]["n_findings"] == 1 + assert score["fallback"]["per_finding"][0]["id"] == "RT-050" def test_the_low_scoring_finding_is_identifiable_in_the_breakdown(): diff --git a/tools/explanation_quality.py b/tools/explanation_quality.py index ab3aeb6..ba30202 100644 --- a/tools/explanation_quality.py +++ b/tools/explanation_quality.py @@ -91,23 +91,11 @@ def load_holdout(path: Path = DEFAULT_HOLDOUT_PATH) -> List[Dict[str, Any]]: return data["findings"] -def score_holdout(findings: List[Dict[str, Any]]) -> Dict[str, Any]: - """Aggregate accuracy/usefulness across every rating in `findings`. - - Returns a dict with the overall mean and minimum of each dimension, - the number of findings and the total number of individual ratings - counted (more than one rater per finding is expected to grow over - time -- see docs/evaluation.md Part 4), and a per-finding breakdown - (that finding's own mean of each dimension) so a caller can see WHICH - finding is dragging the minimum down, not just that one is. - - Raises ValueError on an empty `findings` list -- the same reasoning as - load_holdout()'s docstring: a mean of zero ratings is not a score of - zero, it is not a score at all, and must never print as one. - """ - if not findings: - raise ValueError("cannot score an empty held-out set") - +def _aggregate(findings: List[Dict[str, Any]]) -> Dict[str, Any]: + """The mean/min/per-finding arithmetic, shared by the model score and + the fallback report below. Does not raise on an empty list -- callers + decide what emptiness means for them (score_holdout() raises for the + model group; the fallback group is simply absent when there isn't one).""" accuracies: List[int] = [] usefulnesses: List[int] = [] per_finding: List[Dict[str, Any]] = [] @@ -136,12 +124,70 @@ def score_holdout(findings: List[Dict[str, Any]]) -> Dict[str, Any]: } +def score_holdout(findings: List[Dict[str, Any]]) -> Dict[str, Any]: + """Aggregate accuracy/usefulness across every MODEL-WRITTEN rating in + `findings`. Fallback-sourced captures (`captured_explanation_source` + other than `"model"`) are reported separately under the `"fallback"` + key, never blended into the headline numbers. + + WHY THE SPLIT (found in review, @ARSH871-bot, on the first version of + this set) + `EXPLANATION_QUALITY_THRESHOLD` exists to answer one question: is + the MODEL good enough to prefer over the deterministic fallback. + `_fallback_error_explanation()` and `_fallback_plain_restatement()` + cannot be wrong in the way this threshold cares about -- they are + not generated, so "accuracy cannot be wrong by construction" (the + held-out set's own note on RT-050). Averaging a fallback capture's + score into the model's score lets a set with NO model output at + all pass the threshold -- measured directly: five copies of + RT-050's rating alone would score 5.00/4.00, comfortably above + 4.0/3.0, having said nothing whatsoever about model quality. That + is a weaker claim standing in for a stronger one, the same failure + shape this module's own docstring already names for blending + accuracy into usefulness. + + RT-050 has a second, independent reason to stay out of the model + score: it is `status="error"`, and `web/main.py` only attaches an + explanation to a `status="found"` finding (confirmed directly, + `web/main.py`'s explanation-attachment loop: `if finding.get( + "status") != "found": continue`) -- so no user is ever shown an + explanation for it regardless of what source produced the capture. + + Raises ValueError if there are no model-written entries to score -- + same reasoning as the empty-set case this function always refused: a + score with nothing behind it must never print as a number. A held-out + set consisting ENTIRELY of fallback captures hits this path too, which + is deliberate -- see the test for the shape of that case. + """ + model_findings = [ + f for f in findings + if f.get("captured_explanation_source") == "model" + ] + if not model_findings: + raise ValueError( + "cannot score: no model-written entries in this held-out set " + "(fallback-sourced captures cannot be wrong by construction, " + "so they say nothing about whether the model is good enough)" + ) + + result = _aggregate(model_findings) + + fallback_findings = [ + f for f in findings + if f.get("captured_explanation_source") != "model" + ] + if fallback_findings: + result["fallback"] = _aggregate(fallback_findings) + + return result + + def _format_report(score: Dict[str, Any]) -> str: """Plain-text rendering of score_holdout()'s return value, for the command-line entry point below. Not used by the test suite, which asserts on the dict directly rather than parsing text back out of it.""" lines = [ - f"{score['n_findings']} findings, {score['n_ratings']} ratings", + f"{score['n_findings']} model-written findings, {score['n_ratings']} ratings", f"accuracy: mean {score['mean_accuracy']:.2f} min {score['min_accuracy']}", f"usefulness: mean {score['mean_usefulness']:.2f} min {score['min_usefulness']}", "", @@ -153,6 +199,18 @@ def _format_report(score: Dict[str, Any]) -> str: f"usefulness {row['mean_usefulness']:.1f} " f"({row['n_ratings']} rating(s))" ) + if "fallback" in score: + lines.append("") + lines.append( + f"fallback-sourced (not scored against the threshold -- " + f"cannot be wrong by construction):" + ) + for row in score["fallback"]["per_finding"]: + lines.append( + f" {row['id']:<8} accuracy {row['mean_accuracy']:.1f} " + f"usefulness {row['mean_usefulness']:.1f} " + f"({row['n_ratings']} rating(s))" + ) return "\n".join(lines) From e308c3af26695e0d5c74e9f551a72439794186c5 Mon Sep 17 00:00:00 2001 From: patelankeet2 Date: Mon, 12 Oct 2026 13:43:26 +1300 Subject: [PATCH 07/10] US-55: update the threshold comment and doc to the model-only numbers EXPLANATION_QUALITY_THRESHOLD's own comment still cited the old blended 4.4/3.4 measurement. docs/evaluation.md Part 4 now shows the corrected model-only score (4.25/3.25), reports RT-050 on its own fallback line instead of folded into the table, and carries a dated correction note explaining what changed and why, per the project's own convention of recording a correction rather than silently editing a past measurement. --- docs/evaluation.md | 48 ++++++++++++++++++++++++++---------- tools/explanation_quality.py | 8 +++--- 2 files changed, 39 insertions(+), 17 deletions(-) diff --git a/docs/evaluation.md b/docs/evaluation.md index 160f85c..3e8d700 100644 --- a/docs/evaluation.md +++ b/docs/evaluation.md @@ -757,13 +757,13 @@ score. Same 1-5 scale as Part 3 (accuracy: does the text claim only what the evidence shows; usefulness: would this sentence alone tell someone who never read `evidence.detail` what is wrong). -| id | accuracy | usefulness | note | -|---|---|---|---| -| RT-050 | 5 | 4 | Deterministic fallback, cannot be wrong by construction. | -| AC-001 | 5 | 4 | Correct, no inversion. | -| PC-001 | 5 | 4 | Regenerated post-#145: inversion gone. | -| AC-002 | **2** | **2** | **Real defect, kept rather than re-rolled - see #384.** | -| PC-005 | 5 | 3 | Regenerated post-#145: inversion gone. First-sentence phrasing still asks the reader to resolve an apparent contradiction before it resolves itself - a readability risk, not a false claim this time. | +| id | source | accuracy | usefulness | note | +|---|---|---|---|---| +| RT-050 | fallback | 5 | 4 | Deterministic, cannot be wrong by construction - **reported separately below, not part of the model score.** | +| AC-001 | model | 5 | 4 | Correct, no inversion. | +| PC-001 | model | 5 | 4 | Regenerated post-#145: inversion gone. | +| AC-002 | model | **2** | **2** | **Real defect, kept rather than re-rolled - see #384.** | +| PC-005 | model | 5 | 3 | Regenerated post-#145: inversion gone. First-sentence phrasing still asks the reader to resolve an apparent contradiction before it resolves itself - a readability risk, not a false claim this time. | @ARSH871-bot @shubhamkataria2005 @SamikaPerera - a second independent rating on this set, the same way Part 3 grew from one rater to four, would turn this @@ -777,9 +777,13 @@ python -m tools.explanation_quality ``` ``` -5 findings, 5 ratings -accuracy: mean 4.40 min 2 -usefulness: mean 3.40 min 2 +4 model-written findings, 4 ratings +accuracy: mean 4.25 min 2 +usefulness: mean 3.25 min 2 + +fallback-sourced (not scored against the threshold - cannot be wrong by +construction): + RT-050 accuracy 5.0 usefulness 4.0 (1 rating(s)) ``` **Published, and pinned in `tests/test_explanation_quality.py`** so CI @@ -794,9 +798,27 @@ the same input always produces the same number. alongside a minimum rather than the mean alone - for the same reason the controls table earlier in this document separates "checked clean" from "could not check" instead of reporting one combined pass rate: a single -figure here would have hidden `AC-002`'s accuracy=2 behind four findings at -5, printing a reassuring 4.4 while one finding in the set is a confirmed, -filed defect. +figure here would have hidden `AC-002`'s accuracy=2 behind three other +model-written findings at 5, printing a reassuring 4.25 while one finding +in the set is a confirmed, filed defect. + +> **CORRECTED IN REVIEW, 11-12 October (@ARSH871-bot).** This section +> originally scored all 5 findings together - 4.40/3.40, RT-050 folded in +> with the four model-written ones. That let a held-out set with **no +> model output at all** pass the threshold: five copies of RT-050's rating +> alone score 5.00/4.00, comfortably above 4.0/3.0, having said nothing +> about whether the model is any good. `EXPLANATION_QUALITY_THRESHOLD` +> exists to judge the model specifically, and a fallback-sourced capture +> cannot be wrong by construction, so it cannot speak to that question - +> the same reasoning this section already applies to keeping accuracy and +> usefulness separate, just not carried far enough the first time. RT-050 +> also carries a second, independent reason to stay out: it is +> `status="error"`, and `web/main.py` never attaches an explanation to +> anything but `status="found"`, so no user is ever shown it regardless of +> source. The conclusion is unchanged - 4.25/3.25 still clears the +> threshold - but the number it is measured from now actually measures +> what the threshold exists to judge. RT-050 stays in the committed set and +> is still reported, just on its own line rather than blended in. ### The documented threshold (acceptance criterion 3) diff --git a/tools/explanation_quality.py b/tools/explanation_quality.py index ba30202..48f15a6 100644 --- a/tools/explanation_quality.py +++ b/tools/explanation_quality.py @@ -44,10 +44,10 @@ #: Acceptance criterion 3 of #346: "a documented threshold below which the #: deterministic fallback is preferred". Grounded in the first real -#: measurement (see docs/evaluation.md Part 4): mean_accuracy 4.4, -#: mean_usefulness 3.4 -- so these two numbers sit comfortably below the -#: current measurement rather than just above it, which would make the -#: threshold meaningless. +#: measurement (see docs/evaluation.md Part 4), model-written entries only: +#: mean_accuracy 4.25, mean_usefulness 3.25 -- so these two numbers sit +#: comfortably below the current measurement rather than just above it, +#: which would make the threshold meaningless. #: #: DELIBERATELY ON THE MEAN, NOT THE MINIMUM #: A single low-scoring finding (AC-002, accuracy=2, #384) is treated From 3c110ecbc58421439642d32a8dd1858ac956a6a5 Mon Sep 17 00:00:00 2001 From: patelankeet2 Date: Mon, 12 Oct 2026 13:45:58 +1300 Subject: [PATCH 08/10] US-55: detect when the held-out set goes stale against ai/Modelfile Non-blocking note from the review (@ARSH871-bot): nothing made the published score go stale when the model that produced it changes -- appending one comment line to ai/Modelfile left the whole suite green, because every test replays frozen captured text rather than calling the model again. holdout.json now records what it was captured against (captured_against.modelfile_sha256, base_model). A new test compares that recorded hash to ai/Modelfile's live one and fails loudly, pointing at re-capturing and re-rating rather than at silently updating the recorded hash. python -m pytest tests/test_explanation_quality.py -v -> 14 passed (was 13). --- .../fixtures/explanation_quality/holdout.json | 5 ++ tests/test_explanation_quality.py | 24 +++++++++ tools/explanation_quality.py | 54 +++++++++++++++++-- 3 files changed, 79 insertions(+), 4 deletions(-) diff --git a/tests/fixtures/explanation_quality/holdout.json b/tests/fixtures/explanation_quality/holdout.json index bf53acf..3250e8e 100644 --- a/tests/fixtures/explanation_quality/holdout.json +++ b/tests/fixtures/explanation_quality/holdout.json @@ -4,6 +4,11 @@ "accuracy": "1 (poor) to 5 (excellent) -- does the text claim only what the finding's own evidence actually shows", "usefulness": "1 (poor) to 5 (excellent) -- would this sentence, alone, tell someone who has never read evidence.detail what is actually wrong" }, + "captured_against": { + "_comment": "What produced every model-written explanation below. If ai/Modelfile's sha256 no longer matches this, the set is stale: re-capture and re-rate rather than editing the ratings in place (#346 review, @ARSH871-bot).", + "modelfile_sha256": "728bf3194f7a5f5d672c36480f3f93126753a83cda191bfc375b42eb81480e68", + "base_model": "llama3.2:3b" + }, "findings": [ { "id": "RT-050", diff --git a/tests/test_explanation_quality.py b/tests/test_explanation_quality.py index c6bba81..808adb5 100644 --- a/tests/test_explanation_quality.py +++ b/tests/test_explanation_quality.py @@ -23,7 +23,9 @@ from tools.explanation_quality import ( EXPLANATION_QUALITY_THRESHOLD, + current_modelfile_sha256, load_holdout, + load_holdout_metadata, score_holdout, ) @@ -201,3 +203,25 @@ def test_current_mean_quality_meets_the_documented_threshold(): score = score_holdout(load_holdout()) assert score["mean_accuracy"] >= EXPLANATION_QUALITY_THRESHOLD["mean_accuracy"] assert score["mean_usefulness"] >= EXPLANATION_QUALITY_THRESHOLD["mean_usefulness"] + + +# --------------------------------------------------------------------------- +# Staleness (#346 review, @ARSH871-bot, non-blocking but addressed here). +# Nothing made the published score go stale when the model that produced +# it changes -- appending one line to ai/Modelfile left this whole file +# green, 8 of 8, with no live model call involved anywhere. This is the +# guard: the held-out set records what it was captured against, and this +# test fails loudly, not silently, the moment that stops matching. +# --------------------------------------------------------------------------- + + +def test_the_holdout_set_is_captured_against_the_current_modelfile(): + recorded = load_holdout_metadata()["modelfile_sha256"] + live = current_modelfile_sha256() + assert recorded == live, ( + "ai/Modelfile has changed since this held-out set was captured " + "(recorded sha256 no longer matches). The published score no " + "longer reflects what the model currently produces -- re-capture " + "and re-rate the held-out set (see docs/evaluation.md Part 4's " + "method), do not just update the recorded hash to silence this." + ) diff --git a/tools/explanation_quality.py b/tools/explanation_quality.py index 48f15a6..5db9530 100644 --- a/tools/explanation_quality.py +++ b/tools/explanation_quality.py @@ -30,6 +30,7 @@ from __future__ import annotations +import hashlib import json from pathlib import Path from typing import Any, Dict, List @@ -77,8 +78,42 @@ } -def load_holdout(path: Path = DEFAULT_HOLDOUT_PATH) -> List[Dict[str, Any]]: - """The held-out set's `findings` list, read from `path`. +#: The file whose content shapes every captured explanation in the held-out +#: set. Hashed so a test can tell when the set has gone stale -- see +#: current_modelfile_sha256() and holdout.json's own "captured_against". +MODELFILE_PATH = Path(__file__).resolve().parent.parent / "ai" / "Modelfile" + + +def current_modelfile_sha256(path: Path = MODELFILE_PATH) -> str: + """The live `ai/Modelfile`'s sha256, for comparison against what the + held-out set says it was captured against (#346 review, + @ARSH871-bot's non-blocking note). + + WHY THIS EXISTS + Nothing made the published score go stale when the model that + produced it changes. Checked directly: appending one comment line + to `ai/Modelfile` left the whole suite green, 8 of 8, because + every existing test replays FROZEN captured text rather than + calling the model again. So `test_current_mean_quality_meets_the_ + documented_threshold` keeps reading as "the explanation layer + meets its threshold" after a prompt or base-model change this set + never measured -- exactly the shape #267's traceability-staleness + problem already names, but reached from the model side rather + than the generated-document side. + + Deliberately file-content hashing rather than a version string living + in two places -- the same reasoning this project already applies to + duplicated constants (`analysis/draft_policy.py` reading + `policy_compliance.HIGHEST_POLICY_NUMBER` rather than keeping its own + copy, #376): a number a human has to remember to bump is a number that + goes stale silently. A hash cannot be "forgotten to update" in a way + that still matches. + """ + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _load_raw(path: Path) -> Dict[str, Any]: + """The held-out set's full parsed JSON, metadata included. Raises FileNotFoundError / json.JSONDecodeError on a missing or malformed file rather than returning an empty result -- a silently @@ -87,8 +122,19 @@ def load_holdout(path: Path = DEFAULT_HOLDOUT_PATH) -> List[Dict[str, Any]]: product itself. """ with open(path, encoding="utf-8") as handle: - data = json.load(handle) - return data["findings"] + return json.load(handle) + + +def load_holdout(path: Path = DEFAULT_HOLDOUT_PATH) -> List[Dict[str, Any]]: + """The held-out set's `findings` list, read from `path`. See + _load_raw() for what happens on a missing or malformed file.""" + return _load_raw(path)["findings"] + + +def load_holdout_metadata(path: Path = DEFAULT_HOLDOUT_PATH) -> Dict[str, Any]: + """The held-out set's `captured_against` block -- what `ai/Modelfile` + and base model this set's explanations were actually generated by.""" + return _load_raw(path)["captured_against"] def _aggregate(findings: List[Dict[str, Any]]) -> Dict[str, Any]: From 59d368cfa11be9fa5490318773939f49d6bfc7d8 Mon Sep 17 00:00:00 2001 From: patelankeet2 Date: Sun, 1 Nov 2026 15:59:47 +1300 Subject: [PATCH 09/10] US-55: remove extraneous f prefix from two plain strings (ruff F541) Both strings in _format_report() have no placeholders; ruff flagged them as the two failures blocking CI on this PR. --- tools/explanation_quality.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/explanation_quality.py b/tools/explanation_quality.py index 5db9530..c9b30d6 100644 --- a/tools/explanation_quality.py +++ b/tools/explanation_quality.py @@ -248,8 +248,8 @@ def _format_report(score: Dict[str, Any]) -> str: if "fallback" in score: lines.append("") lines.append( - f"fallback-sourced (not scored against the threshold -- " - f"cannot be wrong by construction):" + "fallback-sourced (not scored against the threshold -- " + "cannot be wrong by construction):" ) for row in score["fallback"]["per_finding"]: lines.append( From 2f84b0bc599bbfb134227d01539bdf04e1a05ebb Mon Sep 17 00:00:00 2001 From: patelankeet2 Date: Sun, 1 Nov 2026 23:43:31 +1300 Subject: [PATCH 10/10] US-55: hash the built prompt per finding, not just ai/Modelfile The Modelfile hash cannot see a change to _build_prompt() itself. Measured: with #401 merged on top of this held-out set, every test still passed, because #401 rewrites the computed fact for AC-002 (this set's own #384 capture, rated 2/2) without touching ai/Modelfile at all. The published score would stay green while describing a prompt the product no longer sends. Added current_prompt_sha256(finding), recorded one hash per model-written entry in holdout.json, and a test that rebuilds each prompt from the stored finding dict and compares. One hash per finding rather than one for the whole set, so a change to one computed-fact function marks only the findings whose own prompt actually moved. Also closes the nit that current_modelfile_sha256() was never shown to hash file content rather than return a constant, and adds the __package__ guard the new ai.explain import requires for the script invocation form (tests/test_tools_invocation.py caught this immediately). Addresses the change requested on PR #385 (review by Shubham). --- .../fixtures/explanation_quality/holdout.json | 4 ++ tests/test_explanation_quality.py | 44 +++++++++++++++++++ tools/explanation_quality.py | 38 ++++++++++++++++ 3 files changed, 86 insertions(+) diff --git a/tests/fixtures/explanation_quality/holdout.json b/tests/fixtures/explanation_quality/holdout.json index 3250e8e..ac3392f 100644 --- a/tests/fixtures/explanation_quality/holdout.json +++ b/tests/fixtures/explanation_quality/holdout.json @@ -53,6 +53,7 @@ }, "captured_explanation": "The device's current configuration allows unencrypted web traffic to reach the internal server. This is in conflict with the policy requirement that should block such traffic, which is currently set to DENY. The reason for this discrepancy lies in the device's own configuration, specifically its rule allowing all incoming traffic without restriction.", "captured_explanation_source": "model", + "captured_prompt_sha256": "544575345b55b033543e550f12437347c34a8b78ab90354a6fe5ef57aa91eef6", "ratings": [ { "rater": "Ankeet", @@ -79,6 +80,7 @@ }, "captured_explanation": "The internal network can reach servers it should not, specifically the server at 8.8.8.8 on port 80, due to a configuration allowing unencrypted web traffic through despite policy requirements blocking it. The device's own configuration is permitting this traffic, which contradicts its written policy that forbids it.", "captured_explanation_source": "model", + "captured_prompt_sha256": "8860f90d1c6da32f54e516d2b49fb8a3a0467624c89e6ddf504697d83359b51b", "ratings": [ { "rater": "Ankeet", @@ -105,6 +107,7 @@ }, "captured_explanation": "The device's ACL rule, which is supposed to block traffic from a specific IP address, never actually takes effect because it is overridden by another rule that blocks all traffic with the same destination IP address. As a result, even though the unreachable line in the ACL rule would normally deny traffic matching its pattern, the blocking rule always wins and prevents any traffic from being allowed.", "captured_explanation_source": "model", + "captured_prompt_sha256": "6906691c3cc74d26649b8c1a2a4877658ec7585dd767dd41f56c07d2b2e9eaf5", "ratings": [ { "rater": "Ankeet", @@ -131,6 +134,7 @@ }, "captured_explanation": "The internal server can be reached over HTTPS traffic, but the device's configuration blocks it. The written policy actually requires this traffic to be allowed, so the blockage is due to the device's own configuration rather than the policy itself. As a result, the service remains unreachable because the required traffic is being denied by the device's rules.", "captured_explanation_source": "model", + "captured_prompt_sha256": "a68cfcf30533b96ae24ceb376d17720d659d42abb391d55d491bee8bde5bbcd5", "ratings": [ { "rater": "Ankeet", diff --git a/tests/test_explanation_quality.py b/tests/test_explanation_quality.py index 808adb5..3383543 100644 --- a/tests/test_explanation_quality.py +++ b/tests/test_explanation_quality.py @@ -24,6 +24,7 @@ from tools.explanation_quality import ( EXPLANATION_QUALITY_THRESHOLD, current_modelfile_sha256, + current_prompt_sha256, load_holdout, load_holdout_metadata, score_holdout, @@ -225,3 +226,46 @@ def test_the_holdout_set_is_captured_against_the_current_modelfile(): "and re-rate the held-out set (see docs/evaluation.md Part 4's " "method), do not just update the recorded hash to silence this." ) + + +def test_current_modelfile_sha256_actually_hashes_the_file_not_a_constant(tmp_path): + """#385 review nit, @shubhamkataria2005: nothing proved this function + reads the file rather than returning a hard-coded value, since every + other test only compares it against the one real Modelfile it was + already going to match.""" + import hashlib + + probe = tmp_path / "Modelfile" + probe.write_text("a probe Modelfile that is not the real one\n", encoding="utf-8") + expected = hashlib.sha256(probe.read_bytes()).hexdigest() + assert current_modelfile_sha256(probe) == expected + + probe.write_text("a DIFFERENT probe Modelfile\n", encoding="utf-8") + assert current_modelfile_sha256(probe) != expected, ( + "changing the file's content must change the hash -- a constant " + "would pass the first assertion and fail to change here" + ) + + +def test_each_model_written_entry_is_captured_against_the_current_prompt(): + """#385 review, @shubhamkataria2005: the Modelfile hash above cannot + see a change to _build_prompt() itself -- measured, with #401 merged + on top of this set, all tests still passed even though #401 rewrites + the computed fact for AC-002, this set's own #384 capture. One prompt + hash per finding, so a change to one computed-fact function marks only + the findings whose own prompt actually moved, not the whole set.""" + stale = [] + for finding in load_holdout(): + if finding["captured_explanation_source"] != "model": + continue + live = current_prompt_sha256(finding["finding"]) + if live != finding["captured_prompt_sha256"]: + stale.append(finding["id"]) + assert not stale, ( + f"these held-out entries were captured against a prompt " + f"_build_prompt() no longer builds: {stale}. The published rating " + f"for each one now describes text the product would not actually " + f"send -- re-capture and re-rate those entries (see " + f"docs/evaluation.md Part 4's method), do not just update the " + f"recorded hash to silence this." + ) diff --git a/tools/explanation_quality.py b/tools/explanation_quality.py index c9b30d6..f4defdd 100644 --- a/tools/explanation_quality.py +++ b/tools/explanation_quality.py @@ -32,9 +32,21 @@ import hashlib import json +import sys from pathlib import Path from typing import Any, Dict, List +# RUN AS A SCRIPT, NOT ONLY AS A MODULE (same fix as tools/preflight.py, #172). +# `python -m tools.explanation_quality` puts the repository root on +# sys.path; `python tools/explanation_quality.py` puts tools/ there +# instead, so `from ai.explain import ...` below (current_prompt_sha256, +# #385 review) would fail with ModuleNotFoundError. This file had no +# cross-package import before that function existed, so it never needed +# this -- tests/test_tools_invocation.py enforces it the moment one is +# added, which is why it caught this immediately rather than later. +if __package__ in (None, ""): + sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + #: Resolved relative to this file, not the caller's working directory -- #: so this runs correctly from the repository root regardless of which of #: the two documented invocation forms launched it. @@ -112,6 +124,32 @@ def current_modelfile_sha256(path: Path = MODELFILE_PATH) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() +def current_prompt_sha256(finding: Dict[str, Any]) -> str: + """The sha256 of what `ai/explain.py` would actually send the model for + `finding` right now (#385 review, @shubhamkataria2005). + + WHY THIS EXISTS, SEPARATELY FROM current_modelfile_sha256() + What the model sees is `ai/Modelfile` PLUS `_build_prompt(finding)` + in `ai/explain.py`, which includes the computed-fact functions + (#395/#399/#401). The Modelfile hash above cannot see a change to + those -- measured: with #401 merged on top of this held-out set, + all tests still passed, because #401 rewrites the computed fact for + AC-002 (this set's own #384 capture, rated 2/2) without touching + `ai/Modelfile` at all. So the set would stay "fresh" by the + Modelfile check while describing a prompt the product no longer + sends. + + One hash per finding, not one hash for the whole set, because a change + to one computed-fact function (say, the routing one) must not mark + EVERY finding stale -- only the ones whose own prompt actually moved. + That keeps this from becoming #267's always-red case, where any new + test turns the whole document stale regardless of what it covers. + """ + from ai.explain import _build_prompt + + return hashlib.sha256(_build_prompt(finding).encode("utf-8")).hexdigest() + + def _load_raw(path: Path) -> Dict[str, Any]: """The held-out set's full parsed JSON, metadata included.