diff --git a/scripts/asvs/scorecard.py b/scripts/asvs/scorecard.py index 663ad95d..55b3276a 100644 --- a/scripts/asvs/scorecard.py +++ b/scripts/asvs/scorecard.py @@ -49,6 +49,17 @@ #: the first was never examined, the second was examined and left open on purpose. DECIDED_VERDICTS: Final[frozenset[str]] = frozenset({"pass", "partial", "fail", "na"}) +#: A verdict whose cell has been READ against the requirement text. Not the same set as +#: :data:`DECIDED_VERDICTS`, and the difference is `needs-review`: that cell was examined and then +#: left open on purpose, so it belongs in survey PROGRESS while staying out of the verdict counts. +#: +#: Survey progress used `DECIDED_VERDICTS` until the scorecard acquired its first `needs-review` cell +#: (11.4.4, the V7/V11 baseline sweep). The page then reported "123 of 345 read … 222 have not" +#: directly above its own table saying 221 unverified — two numbers for one quantity, on the page +#: that IS the record. The bug had been latent since the renderer was written: with zero +#: `needs-review` cells the two sets are identical, so no test and no CI run could tell them apart. +EXAMINED_VERDICTS: Final[frozenset[str]] = DECIDED_VERDICTS | {"needs-review"} + #: How far from the recorded line the expected token may drift before the anchor is considered broken. #: Anchors name a TOKEN rather than a bare line number precisely so that ordinary edits above a cell's #: evidence do not thrash every anchor in the file; the window keeps the line number meaningful without @@ -458,8 +469,10 @@ def render_current(cells: list[Cell], *, anchor_sha: str) -> str: """ n = count(cells) total = sum(n.values()) - decided = [c for c in cells if c.verdict in DECIDED_VERDICTS] - examined = [c for c in decided if c.last_verified] + # EXAMINED, not DECIDED: a `needs-review` cell was read and then parked, so it is survey progress + # even though it carries no verdict. Counting it as unread made this line contradict the table + # below it (see EXAMINED_VERDICTS). + examined = [c for c in cells if c.verdict in EXAMINED_VERDICTS and c.last_verified] unexamined = total - len(examined) pct = (100.0 * len(examined) / total) if total else 0.0 inherited = sum(1 for c in cells if c.verdict in DECIDED_VERDICTS and not c.last_verified) diff --git a/tests/test_asvs_scorecard.py b/tests/test_asvs_scorecard.py index ebd6502a..ffdc13c6 100644 --- a/tests/test_asvs_scorecard.py +++ b/tests/test_asvs_scorecard.py @@ -366,6 +366,33 @@ def test_render_leads_with_survey_progress_not_a_headline_score() -> None: assert "deadbeef" in out +def test_render_counts_a_needs_review_cell_as_read_not_as_unexamined() -> None: + """Survey progress counts what was READ; `needs-review` was read and then parked on purpose. + + Latent from the day the renderer was written and invisible until the scorecard acquired its first + `needs-review` cell: with none present, "decided" and "examined" are the same set, so nothing + could distinguish them. The symptom was two numbers for one quantity on the page that IS the + record — a survey line saying N have *not* been read, directly above a table whose `unverified` + row said N-1. + + The assertion that matters is the LAST one: the two figures the page prints must agree, because a + reader comparing them is the only thing that ever noticed. + """ + cells = [ + Cell(id="1.1.1", level=1, verdict="pass", last_verified="2026-08-02"), + Cell(id="1.1.2", level=2, verdict="unverified"), + Cell(id="2.1.1", level=3, verdict="needs-review", last_verified="2026-08-02"), + ] + out = render_current(cells, anchor_sha="x") + + assert ( + "**2 of 3 requirements have been read against the ASVS text (66.7%).** 1 have not." in out + ) + # ...and it stays OUT of the verdict counts, which is why the two sets differ at all. + assert "| Needs review | 1 |" in out + assert "| **Unverified** | **1** |" in out + + def test_render_flags_a_decided_verdict_carrying_no_verified_date_as_inherited() -> None: cells = [Cell(id="1.1.1", level=1, verdict="pass")] # decided, but never dated out = render_current(cells, anchor_sha="x")