feat(cli): write catalog-summary.json after catalog validate

CHANGELOG.md

@@ -6,6 +6,9 @@ All notable changes to SkillEvaluator are documented in this file.
 
 ### Added
 
+- Catalog validation now writes `catalog-summary.json` at the reports root with
+  per-skill pass/fail status, optional severity rollups from child JSON reports,
+  and paths to per-skill report directories.
 - SARIF 2.1.0 reporter (`-r sarif`) for GitHub Code Scanning and other SARIF
   consumers. Findings map to rule IDs, severity levels, and file locations from
   Tier 1 validation results.

src/skillevaluator/cli.py

@@ -6,8 +6,10 @@
 from __future__ import annotations
 
 import copy
+import json
 import logging
 import math
+from datetime import UTC, datetime
 from pathlib import Path
 
 import click
@@ -692,6 +694,96 @@ def _print_catalog_summary(total: int, failures: list[tuple[str, str]], reports_
     console_.print(Text.assemble(("      reports     ", MUTED), (f"{reports_root}/<skill>/", MUTED)))
 
 
+CATALOG_SUMMARY_FILENAME = "catalog-summary.json"
+
+
+def _new_skill_json_report_name(output_dir: Path, existing_reports: set[Path]) -> str | None:
+    """Return the JSON report filename produced during this catalog skill run."""
+    if not output_dir.is_dir():
+        return None
+    new_reports = set(output_dir.glob("skillevaluator-output-*.json")) - existing_reports
+    if not new_reports:
+        return None
+    return sorted(new_reports, reverse=True)[0].name
+
+
+def _catalog_skill_entry(
+    skill_name: str,
+    skill_report_dir: Path,
+    *,
+    passed: bool,
+    reason: str,
+    json_report_name: str | None = None,
+) -> dict[str, object]:
+    entry: dict[str, object] = {
+        "name": skill_name,
+        "passed": passed,
+        "report_dir": skill_name,
+    }
+    if not passed:
+        entry["reason"] = reason
+
+    if json_report_name:
+        json_report = skill_report_dir / json_report_name
+    else:
+        return entry
+
+    if not json_report.is_file():
+        return entry
+
+    entry["json_report"] = json_report.name
+    try:
+        payload = json.loads(json_report.read_text(encoding="utf-8"))
+    except (json.JSONDecodeError, OSError):
+        return entry
+    if not isinstance(payload, dict):
+        return entry
+
+    for key in ("overall_passed", "overall_status", "incomplete_scans", "severity_counts"):
+        if key in payload:
+            entry[key] = payload[key]
+    return entry
+
+
+def _write_catalog_summary(output_dir: Path, skills: list[dict[str, object]]) -> Path:
+    """Write a machine-readable fleet rollup for catalog validation."""
+    from skillevaluator.reporting.base import _write_report_atomically
+
+    total = len(skills)
+    passed = sum(1 for skill in skills if skill.get("passed"))
+    failed = total - passed
+    severity_totals = {
+        "critical": 0,
+        "high": 0,
+        "medium": 0,
+        "low": 0,
+    }
+    for skill in skills:
+        counts = skill.get("severity_counts")
+        if not isinstance(counts, dict):
+            continue
+        for key in severity_totals:
+            value = counts.get(key)
+            if isinstance(value, int):
+                severity_totals[key] += value
+
+    summary: dict[str, object] = {
+        "total": total,
+        "passed": passed,
+        "failed": failed,
+        "overall_passed": failed == 0,
+        "reports_root": output_dir.name,
+        "summary_path": CATALOG_SUMMARY_FILENAME,
+        "severity_totals": severity_totals,
+        "skills": skills,
+        "generated_at": datetime.now(tz=UTC).isoformat(),
+    }
+    output_dir.mkdir(parents=True, exist_ok=True)
+    output_path = output_dir / CATALOG_SUMMARY_FILENAME
+    _write_report_atomically(output_path, json.dumps(summary, indent=2, default=str, allow_nan=False))
+    return output_path
+
+
 def _validate_catalog(
     ctx: click.Context,
     *,
@@ -712,20 +804,38 @@ def _validate_catalog(
 
     skill_dirs = sorted(marker.parent for marker in resolved_target.glob("*/SKILL.md"))
     failures: list[tuple[str, str]] = []
+    skill_reports: dict[str, str | None] = {}
     for index, skill_dir in enumerate(skill_dirs, start=1):
         _print_catalog_divider(index, len(skill_dirs), skill_dir.name)
+        skill_output = output_dir / skill_dir.name
+        existing_reports = (
+            set(skill_output.glob("skillevaluator-output-*.json")) if skill_output.is_dir() else set()
+        )
         overrides = {
             **ctx.params,
             "target_path": skill_dir,
             "content_type": "skill",
-            "output_dir": output_dir / skill_dir.name,
+            "output_dir": skill_output,
         }
         try:
             ctx.invoke(validate, **overrides)
         except click.ClickException as exc:
             failures.append((skill_dir.name, str(getattr(exc, "message", exc))))
         except Exception as exc:  # unexpected: keep the catalog running, report it on the scoreboard
             failures.append((skill_dir.name, f"unexpected error: {exc}"))
+        skill_reports[skill_dir.name] = _new_skill_json_report_name(skill_output, existing_reports)
+    failure_map = dict(failures)
+    skill_entries = [
+        _catalog_skill_entry(
+            skill_dir.name,
+            output_dir / skill_dir.name,
+            passed=skill_dir.name not in failure_map,
+            reason=failure_map.get(skill_dir.name, ""),
+            json_report_name=skill_reports.get(skill_dir.name),
+        )
+        for skill_dir in skill_dirs
+    ]
+    _write_catalog_summary(output_dir, skill_entries)
     _print_catalog_summary(len(skill_dirs), failures, output_dir)
     if failures:
         raise click.ClickException(

tests/test_commands.py

@@ -10,6 +10,7 @@
 import pytest
 from click.testing import CliRunner
 
+from skillevaluator import cli as cli_module
 from skillevaluator.cli import cli
 from skillevaluator.tier3.commands import parse_agent_model_overrides, parse_agents
 
@@ -326,6 +327,32 @@ def test_validate_catalog_runs_each_skill_as_separate_job() -> None:
         assert result.exit_code == 0, result.output
         assert any(Path("out/simple").glob("*.html"))
         assert any(Path("out/simple2").glob("*.html"))
+        summary_path = Path("out/catalog-summary.json")
+        assert summary_path.is_file()
+        summary = json.loads(summary_path.read_text(encoding="utf-8"))
+        assert summary["total"] == 2
+        assert summary["passed"] == 2
+        assert summary["overall_passed"] is True
+        assert len(summary["skills"]) == 2
+
+
+def test_catalog_skill_entry_skips_stale_json_without_report_name(tmp_path: Path) -> None:
+    skill_dir = tmp_path / "simple"
+    skill_dir.mkdir()
+    stale = skill_dir / "skillevaluator-output-19990101T000000.json"
+    stale.write_text(
+        json.dumps({"overall_passed": True, "severity_counts": {"high": 7}}),
+        encoding="utf-8",
+    )
+    entry = cli_module._catalog_skill_entry(
+        "simple",
+        skill_dir,
+        passed=False,
+        reason="validation failed",
+        json_report_name=None,
+    )
+    assert "overall_passed" not in entry
+    assert "severity_counts" not in entry
 
 
 def test_validate_catalog_rejects_one_previous_version_for_every_skill() -> None:
@@ -726,11 +753,15 @@ def _failing_tier1(*_args, **_kwargs) -> list[ValidationResult]:
             cli, ["validate", str(catalog.resolve()), "--no-llm", "--no-dedup", "--checks", "schema", "-o", "out"]
         )
 
-    out = _plain_text(result.output)
-    assert "skill 1/2" in out and "skill 2/2" in out
-    assert "Catalog Result" in out
-    assert "0/2 skills passed" in out
-    assert result.exit_code != 0
+        out = _plain_text(result.output)
+        assert "skill 1/2" in out and "skill 2/2" in out
+        assert "Catalog Result" in out
+        assert "0/2 skills passed" in out
+        assert result.exit_code != 0
+        summary = json.loads(Path("out/catalog-summary.json").read_text(encoding="utf-8"))
+        assert summary["failed"] == 2
+        assert summary["overall_passed"] is False
+        assert all(not skill["passed"] for skill in summary["skills"])
 
 
 def test_render_evaluation_result_invokes_findings_report(monkeypatch) -> None:
@@ -870,10 +901,13 @@ def _boom(*_args, **_kwargs):
             cli, ["validate", str(catalog.resolve()), "--no-llm", "--no-dedup", "--checks", "schema", "-o", "out"]
         )
 
-    out = _plain_text(result.output)
-    assert "Catalog Result" in out
-    assert "unexpected error: validator exploded" in out
-    assert result.exit_code != 0
+        out = _plain_text(result.output)
+        assert "Catalog Result" in out
+        assert "unexpected error: validator exploded" in out
+        assert result.exit_code != 0
+        summary = json.loads(Path("out/catalog-summary.json").read_text(encoding="utf-8"))
+        assert summary["failed"] == 2
+        assert summary["skills"][0]["reason"] == "unexpected error: validator exploded"
 
 
 def test_summarize_tier2_empty_results_name_a_reason() -> None: