Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 15 additions & 2 deletions scripts/asvs/scorecard.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,17 @@
#: the first was never examined, the second was examined and left open on purpose.
DECIDED_VERDICTS: Final[frozenset[str]] = frozenset({"pass", "partial", "fail", "na"})

#: A verdict whose cell has been READ against the requirement text. Not the same set as
#: :data:`DECIDED_VERDICTS`, and the difference is `needs-review`: that cell was examined and then
#: left open on purpose, so it belongs in survey PROGRESS while staying out of the verdict counts.
#:
#: Survey progress used `DECIDED_VERDICTS` until the scorecard acquired its first `needs-review` cell
#: (11.4.4, the V7/V11 baseline sweep). The page then reported "123 of 345 read … 222 have not"
#: directly above its own table saying 221 unverified — two numbers for one quantity, on the page
#: that IS the record. The bug had been latent since the renderer was written: with zero
#: `needs-review` cells the two sets are identical, so no test and no CI run could tell them apart.
EXAMINED_VERDICTS: Final[frozenset[str]] = DECIDED_VERDICTS | {"needs-review"}

#: How far from the recorded line the expected token may drift before the anchor is considered broken.
#: Anchors name a TOKEN rather than a bare line number precisely so that ordinary edits above a cell's
#: evidence do not thrash every anchor in the file; the window keeps the line number meaningful without
Expand Down Expand Up @@ -458,8 +469,10 @@ def render_current(cells: list[Cell], *, anchor_sha: str) -> str:
"""
n = count(cells)
total = sum(n.values())
decided = [c for c in cells if c.verdict in DECIDED_VERDICTS]
examined = [c for c in decided if c.last_verified]
# EXAMINED, not DECIDED: a `needs-review` cell was read and then parked, so it is survey progress
# even though it carries no verdict. Counting it as unread made this line contradict the table
# below it (see EXAMINED_VERDICTS).
examined = [c for c in cells if c.verdict in EXAMINED_VERDICTS and c.last_verified]
unexamined = total - len(examined)
pct = (100.0 * len(examined) / total) if total else 0.0
inherited = sum(1 for c in cells if c.verdict in DECIDED_VERDICTS and not c.last_verified)
Expand Down
27 changes: 27 additions & 0 deletions tests/test_asvs_scorecard.py
Original file line number Diff line number Diff line change
Expand Up @@ -366,6 +366,33 @@ def test_render_leads_with_survey_progress_not_a_headline_score() -> None:
assert "deadbeef" in out


def test_render_counts_a_needs_review_cell_as_read_not_as_unexamined() -> None:
"""Survey progress counts what was READ; `needs-review` was read and then parked on purpose.

Latent from the day the renderer was written and invisible until the scorecard acquired its first
`needs-review` cell: with none present, "decided" and "examined" are the same set, so nothing
could distinguish them. The symptom was two numbers for one quantity on the page that IS the
record — a survey line saying N have *not* been read, directly above a table whose `unverified`
row said N-1.

The assertion that matters is the LAST one: the two figures the page prints must agree, because a
reader comparing them is the only thing that ever noticed.
"""
cells = [
Cell(id="1.1.1", level=1, verdict="pass", last_verified="2026-08-02"),
Cell(id="1.1.2", level=2, verdict="unverified"),
Cell(id="2.1.1", level=3, verdict="needs-review", last_verified="2026-08-02"),
]
out = render_current(cells, anchor_sha="x")

assert (
"**2 of 3 requirements have been read against the ASVS text (66.7%).** 1 have not." in out
)
# ...and it stays OUT of the verdict counts, which is why the two sets differ at all.
assert "| Needs review | 1 |" in out
assert "| **Unverified** | **1** |" in out


def test_render_flags_a_decided_verdict_carrying_no_verified_date_as_inherited() -> None:
cells = [Cell(id="1.1.1", level=1, verdict="pass")] # decided, but never dated
out = render_current(cells, anchor_sha="x")
Expand Down
Loading