diff --git a/CHANGELOG.md b/CHANGELOG.md index 57deee3..7937e1c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ All notable changes to SkillEvaluator are documented in this file. ### Added +- Catalog validation now writes `catalog-summary.json` at the reports root with + per-skill pass/fail status, optional severity rollups from child JSON reports, + and paths to per-skill report directories. - SARIF 2.1.0 reporter (`-r sarif`) for GitHub Code Scanning and other SARIF consumers. Findings map to rule IDs, severity levels, and file locations from Tier 1 validation results. diff --git a/src/skillevaluator/cli.py b/src/skillevaluator/cli.py index 8aa423a..15dedaf 100644 --- a/src/skillevaluator/cli.py +++ b/src/skillevaluator/cli.py @@ -6,8 +6,10 @@ from __future__ import annotations import copy +import json import logging import math +from datetime import UTC, datetime from pathlib import Path import click @@ -692,6 +694,96 @@ def _print_catalog_summary(total: int, failures: list[tuple[str, str]], reports_ console_.print(Text.assemble((" reports ", MUTED), (f"{reports_root}//", MUTED))) +CATALOG_SUMMARY_FILENAME = "catalog-summary.json" + + +def _new_skill_json_report_name(output_dir: Path, existing_reports: set[Path]) -> str | None: + """Return the JSON report filename produced during this catalog skill run.""" + if not output_dir.is_dir(): + return None + new_reports = set(output_dir.glob("skillevaluator-output-*.json")) - existing_reports + if not new_reports: + return None + return sorted(new_reports, reverse=True)[0].name + + +def _catalog_skill_entry( + skill_name: str, + skill_report_dir: Path, + *, + passed: bool, + reason: str, + json_report_name: str | None = None, +) -> dict[str, object]: + entry: dict[str, object] = { + "name": skill_name, + "passed": passed, + "report_dir": skill_name, + } + if not passed: + entry["reason"] = reason + + if json_report_name: + json_report = skill_report_dir / json_report_name + else: + return entry + + if not json_report.is_file(): + return entry + + entry["json_report"] = json_report.name + try: + payload = json.loads(json_report.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError): + return entry + if not isinstance(payload, dict): + return entry + + for key in ("overall_passed", "overall_status", "incomplete_scans", "severity_counts"): + if key in payload: + entry[key] = payload[key] + return entry + + +def _write_catalog_summary(output_dir: Path, skills: list[dict[str, object]]) -> Path: + """Write a machine-readable fleet rollup for catalog validation.""" + from skillevaluator.reporting.base import _write_report_atomically + + total = len(skills) + passed = sum(1 for skill in skills if skill.get("passed")) + failed = total - passed + severity_totals = { + "critical": 0, + "high": 0, + "medium": 0, + "low": 0, + } + for skill in skills: + counts = skill.get("severity_counts") + if not isinstance(counts, dict): + continue + for key in severity_totals: + value = counts.get(key) + if isinstance(value, int): + severity_totals[key] += value + + summary: dict[str, object] = { + "total": total, + "passed": passed, + "failed": failed, + "overall_passed": failed == 0, + "reports_root": output_dir.name, + "summary_path": CATALOG_SUMMARY_FILENAME, + "severity_totals": severity_totals, + "skills": skills, + "generated_at": datetime.now(tz=UTC).isoformat(), + } + output_dir.mkdir(parents=True, exist_ok=True) + output_path = output_dir / CATALOG_SUMMARY_FILENAME + _write_report_atomically(output_path, json.dumps(summary, indent=2, default=str, allow_nan=False)) + return output_path + + def _validate_catalog( ctx: click.Context, *, @@ -712,13 +804,18 @@ def _validate_catalog( skill_dirs = sorted(marker.parent for marker in resolved_target.glob("*/SKILL.md")) failures: list[tuple[str, str]] = [] + skill_reports: dict[str, str | None] = {} for index, skill_dir in enumerate(skill_dirs, start=1): _print_catalog_divider(index, len(skill_dirs), skill_dir.name) + skill_output = output_dir / skill_dir.name + existing_reports = ( + set(skill_output.glob("skillevaluator-output-*.json")) if skill_output.is_dir() else set() + ) overrides = { **ctx.params, "target_path": skill_dir, "content_type": "skill", - "output_dir": output_dir / skill_dir.name, + "output_dir": skill_output, } try: ctx.invoke(validate, **overrides) @@ -726,6 +823,19 @@ def _validate_catalog( failures.append((skill_dir.name, str(getattr(exc, "message", exc)))) except Exception as exc: # unexpected: keep the catalog running, report it on the scoreboard failures.append((skill_dir.name, f"unexpected error: {exc}")) + skill_reports[skill_dir.name] = _new_skill_json_report_name(skill_output, existing_reports) + failure_map = dict(failures) + skill_entries = [ + _catalog_skill_entry( + skill_dir.name, + output_dir / skill_dir.name, + passed=skill_dir.name not in failure_map, + reason=failure_map.get(skill_dir.name, ""), + json_report_name=skill_reports.get(skill_dir.name), + ) + for skill_dir in skill_dirs + ] + _write_catalog_summary(output_dir, skill_entries) _print_catalog_summary(len(skill_dirs), failures, output_dir) if failures: raise click.ClickException( diff --git a/tests/test_commands.py b/tests/test_commands.py index b9e28b5..2e803f6 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -10,6 +10,7 @@ import pytest from click.testing import CliRunner +from skillevaluator import cli as cli_module from skillevaluator.cli import cli from skillevaluator.tier3.commands import parse_agent_model_overrides, parse_agents @@ -326,6 +327,32 @@ def test_validate_catalog_runs_each_skill_as_separate_job() -> None: assert result.exit_code == 0, result.output assert any(Path("out/simple").glob("*.html")) assert any(Path("out/simple2").glob("*.html")) + summary_path = Path("out/catalog-summary.json") + assert summary_path.is_file() + summary = json.loads(summary_path.read_text(encoding="utf-8")) + assert summary["total"] == 2 + assert summary["passed"] == 2 + assert summary["overall_passed"] is True + assert len(summary["skills"]) == 2 + + +def test_catalog_skill_entry_skips_stale_json_without_report_name(tmp_path: Path) -> None: + skill_dir = tmp_path / "simple" + skill_dir.mkdir() + stale = skill_dir / "skillevaluator-output-19990101T000000.json" + stale.write_text( + json.dumps({"overall_passed": True, "severity_counts": {"high": 7}}), + encoding="utf-8", + ) + entry = cli_module._catalog_skill_entry( + "simple", + skill_dir, + passed=False, + reason="validation failed", + json_report_name=None, + ) + assert "overall_passed" not in entry + assert "severity_counts" not in entry def test_validate_catalog_rejects_one_previous_version_for_every_skill() -> None: @@ -726,11 +753,15 @@ def _failing_tier1(*_args, **_kwargs) -> list[ValidationResult]: cli, ["validate", str(catalog.resolve()), "--no-llm", "--no-dedup", "--checks", "schema", "-o", "out"] ) - out = _plain_text(result.output) - assert "skill 1/2" in out and "skill 2/2" in out - assert "Catalog Result" in out - assert "0/2 skills passed" in out - assert result.exit_code != 0 + out = _plain_text(result.output) + assert "skill 1/2" in out and "skill 2/2" in out + assert "Catalog Result" in out + assert "0/2 skills passed" in out + assert result.exit_code != 0 + summary = json.loads(Path("out/catalog-summary.json").read_text(encoding="utf-8")) + assert summary["failed"] == 2 + assert summary["overall_passed"] is False + assert all(not skill["passed"] for skill in summary["skills"]) def test_render_evaluation_result_invokes_findings_report(monkeypatch) -> None: @@ -870,10 +901,13 @@ def _boom(*_args, **_kwargs): cli, ["validate", str(catalog.resolve()), "--no-llm", "--no-dedup", "--checks", "schema", "-o", "out"] ) - out = _plain_text(result.output) - assert "Catalog Result" in out - assert "unexpected error: validator exploded" in out - assert result.exit_code != 0 + out = _plain_text(result.output) + assert "Catalog Result" in out + assert "unexpected error: validator exploded" in out + assert result.exit_code != 0 + summary = json.loads(Path("out/catalog-summary.json").read_text(encoding="utf-8")) + assert summary["failed"] == 2 + assert summary["skills"][0]["reason"] == "unexpected error: validator exploded" def test_summarize_tier2_empty_results_name_a_reason() -> None: