Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 53 additions & 0 deletions schemas/security_report_validation.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,11 @@ def diagnose_security_report_example(
for finding in findings
if isinstance(finding, dict)
]
finding_counts = {}
for finding in findings:
if isinstance(finding, dict) and isinstance(finding.get("code"), str):
code = finding["code"]
finding_counts[code] = finding_counts.get(code, 0) + 1

for expected in warning_derived_findings:
if expected not in actual_projected_findings:
Expand All @@ -83,6 +88,46 @@ def diagnose_security_report_example(
"for warning-derived findings"
)

for code in sorted(set(summary.keys()) | set(finding_counts.keys())):
expected_count = finding_counts.get(code, 0)
if summary.get(code, 0) != expected_count:
diagnostics.append(
f"{ctx}: summary.{code} must be {expected_count} for report findings"
)

inventories = report.get("inventories") if isinstance(report, dict) else {}
if not isinstance(inventories, dict):
diagnostics.append(f"{ctx}: inventories must be an object")
return diagnostics

inventory_lists = {
name: inventory_items(inventories, name, ctx, diagnostics)
for name in ("annotations", "actions", "attachments", "scripts", "links")
}
annotations = inventory_lists["annotations"]
links = inventory_lists["links"]
external_links = [
link for link in links if isinstance(link, dict) and link.get("external") is True
]

if annotations and finding_counts.get("annotations_present", 0) == 0:
diagnostics.append(
f"{ctx}: inventories.annotations requires annotations_present finding"
)
if finding_counts.get("annotations_present", 0) > 0 and not annotations:
diagnostics.append(
f"{ctx}: annotations_present finding requires inventories.annotations entry"
)

if external_links and finding_counts.get("external_links_present", 0) == 0:
diagnostics.append(
f"{ctx}: inventories.links external=true requires external_links_present finding"
)
if finding_counts.get("external_links_present", 0) > 0 and not external_links:
diagnostics.append(
f"{ctx}: external_links_present finding requires inventories.links external=true entry"
)

return diagnostics


Expand All @@ -108,3 +153,11 @@ def project_report_finding(finding):
if key in finding:
projected[key] = finding[key]
return projected


def inventory_items(inventories, name, ctx, diagnostics):
items = inventories.get(name, [])
if not isinstance(items, list):
diagnostics.append(f"{ctx}: inventories.{name} must be an array")
return []
return items
118 changes: 118 additions & 0 deletions schemas/test_security_report_validation.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,18 @@ def test_warning_derived_summary_must_match_document_warning_count(self) -> None
diagnostics,
)

def test_summary_must_match_all_report_finding_counts(self) -> None:
report = copy.deepcopy(self.report)
report["summary"]["external_links_present"] = 2

diagnostics = diagnose_security_report_example(self.document, report)

self.assertIn(
"security-report.example.json: summary.external_links_present must be 1 "
"for report findings",
diagnostics,
)

def test_document_security_warnings_must_have_matching_findings(self) -> None:
report = copy.deepcopy(self.report)
report["findings"] = [
Expand All @@ -65,6 +77,22 @@ def test_document_security_warnings_must_have_matching_findings(self) -> None:
diagnostics,
)

def test_stale_summary_without_matching_finding_fails_closed(self) -> None:
report = copy.deepcopy(self.report)
report["findings"] = [
finding
for finding in report["findings"]
if finding["code"] != "external_links_present"
]

diagnostics = diagnose_security_report_example(self.document, report)

self.assertIn(
"security-report.example.json: summary.external_links_present must be 0 "
"for report findings",
diagnostics,
)

def test_warning_refs_must_match_report_finding_projection(self) -> None:
report = copy.deepcopy(self.report)
report["findings"][0]["span_ref"] = "s999999"
Expand All @@ -87,6 +115,96 @@ def test_default_excluded_warning_codes_must_be_flagged(self) -> None:
diagnostics,
)

def test_annotations_inventory_requires_matching_finding(self) -> None:
report = copy.deepcopy(self.report)
report["findings"] = [
finding
for finding in report["findings"]
if finding["code"] != "annotations_present"
]
report["summary"].pop("annotations_present")

diagnostics = diagnose_security_report_example(self.document, report)

self.assertIn(
"security-report.example.json: inventories.annotations requires annotations_present finding",
diagnostics,
)

def test_annotations_finding_requires_inventory_entry(self) -> None:
report = copy.deepcopy(self.report)
report["inventories"]["annotations"] = []

diagnostics = diagnose_security_report_example(self.document, report)

self.assertIn(
"security-report.example.json: annotations_present finding requires "
"inventories.annotations entry",
diagnostics,
)

def test_external_link_inventory_requires_matching_finding(self) -> None:
report = copy.deepcopy(self.report)
report["findings"] = [
finding
for finding in report["findings"]
if finding["code"] != "external_links_present"
]
report["summary"].pop("external_links_present")

diagnostics = diagnose_security_report_example(self.document, report)

self.assertIn(
"security-report.example.json: inventories.links external=true requires "
"external_links_present finding",
diagnostics,
)

def test_external_link_finding_requires_external_inventory_entry(self) -> None:
report = copy.deepcopy(self.report)
report["inventories"]["links"][0]["external"] = False

diagnostics = diagnose_security_report_example(self.document, report)

self.assertIn(
"security-report.example.json: external_links_present finding requires "
"inventories.links external=true entry",
diagnostics,
)

def test_inventory_shape_must_be_deterministic_arrays(self) -> None:
report = copy.deepcopy(self.report)
report["inventories"]["links"] = {"page": "p0001"}

diagnostics = diagnose_security_report_example(self.document, report)

self.assertIn(
"security-report.example.json: inventories.links must be an array",
diagnostics,
)

def test_action_inventory_shape_is_checked_without_action_semantics(self) -> None:
report = copy.deepcopy(self.report)
report["inventories"]["actions"] = {"kind": "uri"}

diagnostics = diagnose_security_report_example(self.document, report)

self.assertIn(
"security-report.example.json: inventories.actions must be an array",
diagnostics,
)

def test_inventories_must_be_deterministic_object(self) -> None:
report = copy.deepcopy(self.report)
report["inventories"] = []

diagnostics = diagnose_security_report_example(self.document, report)

self.assertIn(
"security-report.example.json: inventories must be an object",
diagnostics,
)

def test_reportable_parser_warning_codes_are_included_when_present(self) -> None:
document = copy.deepcopy(self.document)
document["payload"]["parser_warnings"].append(
Expand Down
Loading