From ae749bc94e93f21b428f656dad11c2d4aeaf21ac Mon Sep 17 00:00:00 2001 From: mimran-khan Date: Sat, 29 Aug 2026 03:38:04 +0530 Subject: [PATCH 1/2] feat(cli): write catalog-summary.json after catalog validate Emit a machine-readable fleet rollup at the reports root with per-skill status, optional severity totals from child JSON reports, and report paths. Create the output directory when needed so summary writes survive early skill failures. Fixes #120 Signed-off-by: mimran-khan --- CHANGELOG.md | 6 +++ src/skillevaluator/cli.py | 96 +++++++++++++++++++++++++++++++++++++++ tests/test_commands.py | 32 +++++++++---- 3 files changed, 125 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f54c4e84..8e7f6aa8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ All notable changes to SkillEvaluator are documented in this file. ## Unreleased +### Added + +- Catalog validation now writes `catalog-summary.json` at the reports root with + per-skill pass/fail status, optional severity rollups from child JSON reports, + and paths to per-skill report directories. + ### Fixed - Tier 3 accuracy and custom goal judges now retry one malformed (including diff --git a/src/skillevaluator/cli.py b/src/skillevaluator/cli.py index 0ed29141..68de9590 100644 --- a/src/skillevaluator/cli.py +++ b/src/skillevaluator/cli.py @@ -6,8 +6,10 @@ from __future__ import annotations import copy +import json import logging import math +from datetime import UTC, datetime from pathlib import Path import click @@ -691,6 +693,89 @@ def _print_catalog_summary(total: int, failures: list[tuple[str, str]], reports_ console_.print(Text.assemble((" reports ", MUTED), (f"{reports_root}//", MUTED))) +CATALOG_SUMMARY_FILENAME = "catalog-summary.json" + + +def _latest_skill_json_report(skill_report_dir: Path) -> Path | None: + """Return the newest per-skill machine-readable report when present.""" + if not skill_report_dir.is_dir(): + return None + candidates = sorted(skill_report_dir.glob("skillevaluator-output-*.json"), reverse=True) + if candidates: + return candidates[0] + return None + + +def _catalog_skill_entry( + skill_name: str, + skill_report_dir: Path, + *, + passed: bool, + reason: str, +) -> dict[str, object]: + entry: dict[str, object] = { + "name": skill_name, + "passed": passed, + "report_dir": skill_name, + } + if not passed: + entry["reason"] = reason + + json_report = _latest_skill_json_report(skill_report_dir) + if json_report is None: + return entry + + entry["json_report"] = json_report.name + try: + payload = json.loads(json_report.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError): + return entry + if not isinstance(payload, dict): + return entry + + for key in ("overall_passed", "overall_status", "incomplete_scans", "severity_counts"): + if key in payload: + entry[key] = payload[key] + return entry + + +def _write_catalog_summary(output_dir: Path, skills: list[dict[str, object]]) -> Path: + """Write a machine-readable fleet rollup for catalog validation.""" + total = len(skills) + passed = sum(1 for skill in skills if skill.get("passed")) + failed = total - passed + severity_totals = { + "critical": 0, + "high": 0, + "medium": 0, + "low": 0, + } + for skill in skills: + counts = skill.get("severity_counts") + if not isinstance(counts, dict): + continue + for key in severity_totals: + value = counts.get(key) + if isinstance(value, int): + severity_totals[key] += value + + summary: dict[str, object] = { + "total": total, + "passed": passed, + "failed": failed, + "overall_passed": failed == 0, + "reports_root": output_dir.name, + "summary_path": CATALOG_SUMMARY_FILENAME, + "severity_totals": severity_totals, + "skills": skills, + "generated_at": datetime.now(tz=UTC).isoformat(), + } + output_dir.mkdir(parents=True, exist_ok=True) + output_path = output_dir / CATALOG_SUMMARY_FILENAME + output_path.write_text(json.dumps(summary, indent=2, default=str, allow_nan=False), encoding="utf-8") + return output_path + + def _validate_catalog( ctx: click.Context, *, @@ -725,6 +810,17 @@ def _validate_catalog( failures.append((skill_dir.name, str(getattr(exc, "message", exc)))) except Exception as exc: # unexpected: keep the catalog running, report it on the scoreboard failures.append((skill_dir.name, f"unexpected error: {exc}")) + failure_map = dict(failures) + skill_entries = [ + _catalog_skill_entry( + skill_dir.name, + output_dir / skill_dir.name, + passed=skill_dir.name not in failure_map, + reason=failure_map.get(skill_dir.name, ""), + ) + for skill_dir in skill_dirs + ] + _write_catalog_summary(output_dir, skill_entries) _print_catalog_summary(len(skill_dirs), failures, output_dir) if failures: raise click.ClickException( diff --git a/tests/test_commands.py b/tests/test_commands.py index b9e28b50..0b727bbe 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -326,6 +326,13 @@ def test_validate_catalog_runs_each_skill_as_separate_job() -> None: assert result.exit_code == 0, result.output assert any(Path("out/simple").glob("*.html")) assert any(Path("out/simple2").glob("*.html")) + summary_path = Path("out/catalog-summary.json") + assert summary_path.is_file() + summary = json.loads(summary_path.read_text(encoding="utf-8")) + assert summary["total"] == 2 + assert summary["passed"] == 2 + assert summary["overall_passed"] is True + assert len(summary["skills"]) == 2 def test_validate_catalog_rejects_one_previous_version_for_every_skill() -> None: @@ -726,11 +733,15 @@ def _failing_tier1(*_args, **_kwargs) -> list[ValidationResult]: cli, ["validate", str(catalog.resolve()), "--no-llm", "--no-dedup", "--checks", "schema", "-o", "out"] ) - out = _plain_text(result.output) - assert "skill 1/2" in out and "skill 2/2" in out - assert "Catalog Result" in out - assert "0/2 skills passed" in out - assert result.exit_code != 0 + out = _plain_text(result.output) + assert "skill 1/2" in out and "skill 2/2" in out + assert "Catalog Result" in out + assert "0/2 skills passed" in out + assert result.exit_code != 0 + summary = json.loads(Path("out/catalog-summary.json").read_text(encoding="utf-8")) + assert summary["failed"] == 2 + assert summary["overall_passed"] is False + assert all(not skill["passed"] for skill in summary["skills"]) def test_render_evaluation_result_invokes_findings_report(monkeypatch) -> None: @@ -870,10 +881,13 @@ def _boom(*_args, **_kwargs): cli, ["validate", str(catalog.resolve()), "--no-llm", "--no-dedup", "--checks", "schema", "-o", "out"] ) - out = _plain_text(result.output) - assert "Catalog Result" in out - assert "unexpected error: validator exploded" in out - assert result.exit_code != 0 + out = _plain_text(result.output) + assert "Catalog Result" in out + assert "unexpected error: validator exploded" in out + assert result.exit_code != 0 + summary = json.loads(Path("out/catalog-summary.json").read_text(encoding="utf-8")) + assert summary["failed"] == 2 + assert summary["skills"][0]["reason"] == "unexpected error: validator exploded" def test_summarize_tier2_empty_results_name_a_reason() -> None: From 2e151ae2848ba22f7bc5dad2e564350615fbe2a5 Mon Sep 17 00:00:00 2001 From: mimran-khan Date: Mon, 31 Aug 2026 22:26:19 +0530 Subject: [PATCH 2/2] fix(cli): harden catalog-summary provenance and atomic writes Track JSON reports per serial catalog run, skip stale report attribution when no new JSON was written, and write catalog-summary.json through the atomic report writer. Signed-off-by: mimran-khan --- src/skillevaluator/cli.py | 36 +++++++++++++++++++++++++----------- tests/test_commands.py | 20 ++++++++++++++++++++ 2 files changed, 45 insertions(+), 11 deletions(-) diff --git a/src/skillevaluator/cli.py b/src/skillevaluator/cli.py index 68de9590..fe95e98d 100644 --- a/src/skillevaluator/cli.py +++ b/src/skillevaluator/cli.py @@ -696,14 +696,14 @@ def _print_catalog_summary(total: int, failures: list[tuple[str, str]], reports_ CATALOG_SUMMARY_FILENAME = "catalog-summary.json" -def _latest_skill_json_report(skill_report_dir: Path) -> Path | None: - """Return the newest per-skill machine-readable report when present.""" - if not skill_report_dir.is_dir(): +def _new_skill_json_report_name(output_dir: Path, existing_reports: set[Path]) -> str | None: + """Return the JSON report filename produced during this catalog skill run.""" + if not output_dir.is_dir(): return None - candidates = sorted(skill_report_dir.glob("skillevaluator-output-*.json"), reverse=True) - if candidates: - return candidates[0] - return None + new_reports = set(output_dir.glob("skillevaluator-output-*.json")) - existing_reports + if not new_reports: + return None + return sorted(new_reports, reverse=True)[0].name def _catalog_skill_entry( @@ -712,6 +712,7 @@ def _catalog_skill_entry( *, passed: bool, reason: str, + json_report_name: str | None = None, ) -> dict[str, object]: entry: dict[str, object] = { "name": skill_name, @@ -721,8 +722,12 @@ def _catalog_skill_entry( if not passed: entry["reason"] = reason - json_report = _latest_skill_json_report(skill_report_dir) - if json_report is None: + if json_report_name: + json_report = skill_report_dir / json_report_name + else: + return entry + + if not json_report.is_file(): return entry entry["json_report"] = json_report.name @@ -741,6 +746,8 @@ def _catalog_skill_entry( def _write_catalog_summary(output_dir: Path, skills: list[dict[str, object]]) -> Path: """Write a machine-readable fleet rollup for catalog validation.""" + from skillevaluator.reporting.base import _write_report_atomically + total = len(skills) passed = sum(1 for skill in skills if skill.get("passed")) failed = total - passed @@ -772,7 +779,7 @@ def _write_catalog_summary(output_dir: Path, skills: list[dict[str, object]]) -> } output_dir.mkdir(parents=True, exist_ok=True) output_path = output_dir / CATALOG_SUMMARY_FILENAME - output_path.write_text(json.dumps(summary, indent=2, default=str, allow_nan=False), encoding="utf-8") + _write_report_atomically(output_path, json.dumps(summary, indent=2, default=str, allow_nan=False)) return output_path @@ -796,13 +803,18 @@ def _validate_catalog( skill_dirs = sorted(marker.parent for marker in resolved_target.glob("*/SKILL.md")) failures: list[tuple[str, str]] = [] + skill_reports: dict[str, str | None] = {} for index, skill_dir in enumerate(skill_dirs, start=1): _print_catalog_divider(index, len(skill_dirs), skill_dir.name) + skill_output = output_dir / skill_dir.name + existing_reports = ( + set(skill_output.glob("skillevaluator-output-*.json")) if skill_output.is_dir() else set() + ) overrides = { **ctx.params, "target_path": skill_dir, "content_type": "skill", - "output_dir": output_dir / skill_dir.name, + "output_dir": skill_output, } try: ctx.invoke(validate, **overrides) @@ -810,6 +822,7 @@ def _validate_catalog( failures.append((skill_dir.name, str(getattr(exc, "message", exc)))) except Exception as exc: # unexpected: keep the catalog running, report it on the scoreboard failures.append((skill_dir.name, f"unexpected error: {exc}")) + skill_reports[skill_dir.name] = _new_skill_json_report_name(skill_output, existing_reports) failure_map = dict(failures) skill_entries = [ _catalog_skill_entry( @@ -817,6 +830,7 @@ def _validate_catalog( output_dir / skill_dir.name, passed=skill_dir.name not in failure_map, reason=failure_map.get(skill_dir.name, ""), + json_report_name=skill_reports.get(skill_dir.name), ) for skill_dir in skill_dirs ] diff --git a/tests/test_commands.py b/tests/test_commands.py index 0b727bbe..2e803f6a 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -10,6 +10,7 @@ import pytest from click.testing import CliRunner +from skillevaluator import cli as cli_module from skillevaluator.cli import cli from skillevaluator.tier3.commands import parse_agent_model_overrides, parse_agents @@ -335,6 +336,25 @@ def test_validate_catalog_runs_each_skill_as_separate_job() -> None: assert len(summary["skills"]) == 2 +def test_catalog_skill_entry_skips_stale_json_without_report_name(tmp_path: Path) -> None: + skill_dir = tmp_path / "simple" + skill_dir.mkdir() + stale = skill_dir / "skillevaluator-output-19990101T000000.json" + stale.write_text( + json.dumps({"overall_passed": True, "severity_counts": {"high": 7}}), + encoding="utf-8", + ) + entry = cli_module._catalog_skill_entry( + "simple", + skill_dir, + passed=False, + reason="validation failed", + json_report_name=None, + ) + assert "overall_passed" not in entry + assert "severity_counts" not in entry + + def test_validate_catalog_rejects_one_previous_version_for_every_skill() -> None: runner = CliRunner() with runner.isolated_filesystem():