diff --git a/action.yml b/action.yml index 4225245..c23a78c 100644 --- a/action.yml +++ b/action.yml @@ -336,6 +336,42 @@ runs: run-id: ${{ steps.baseline-run.outputs.run-id }} github-token: ${{ github.token }} + - name: Warm the artifact cache from the baseline + # `assert-ai run` has a fingerprint-based artifact cache under each + # behavior's `artifacts_root`. `actions/checkout` starts every job with + # an empty workspace, so the cache is cold and the test_set stage + # regenerates on every dispatch even when nothing changed. The paired + # McNemar gate cannot reach PASS until the current run's cache is + # seeded with the baseline artifacts. Copy each behavior's baseline + # tree into its `artifacts_root` here; `assert-ai run` then hydrates + # its context from the baseline's `latest.json`, reuses the frozen + # test_set (identical fingerprint), and only re-runs downstream stages + # whose fingerprints actually changed (a target-code change on a real + # PR still misses the inference cache and scores fresh, as intended). + if: ${{ steps.resolve-baseline.outputs.needs-download == 'true' && steps.baseline-run.outputs.found == 'true' }} + shell: bash + run: | + python <<'PY' + import json + import shutil + from pathlib import Path + + behaviors = json.loads(Path("assert-ai-behaviors.json").read_text(encoding="utf-8")) + baseline_root = Path("assert-ai-baseline").resolve() + + for b in behaviors: + src = baseline_root / b["slug"] + dst = Path(b["artifacts_root"]) + if not src.exists(): + print(f"[warm-cache] no baseline dir for {b['slug']}, skipping") + continue + dst.mkdir(parents=True, exist_ok=True) + # dirs_exist_ok=True lets Python 3.11+ merge trees; any files the + # baseline doesn't cover fall through to the fresh run's own writes. + shutil.copytree(src, dst, dirs_exist_ok=True) + print(f"[warm-cache] seeded {b['slug']} from baseline") + PY + - name: Export provider credentials shell: bash env: diff --git a/scripts/plan_behaviors.py b/scripts/plan_behaviors.py index 602c28b..9d81937 100644 --- a/scripts/plan_behaviors.py +++ b/scripts/plan_behaviors.py @@ -92,6 +92,20 @@ def plan(args: argparse.Namespace) -> int: artifacts_root = artifacts_base / slug cfg["artifacts_root"] = str(artifacts_root) + # Pin the suite_id so every dispatch of the same behavior writes into + # the same `results//` directory. Without this, `assert-ai` + # defaults `suite_id` to `eval-` on every run, so + # the previous run's cached test_set / inference / judge artifacts + # live at a path the current run never looks at. The paired McNemar + # gate then re-generates the test_set from scratch, the drift + # detector correctly reports `TestSetChanged`, and no-code-change PRs + # can never reach PASS. + # + # Respect a user-supplied `suite:` when present so anyone already + # relying on their own suite naming keeps that layout; only default + # to the slug when the config left it unset. + suite = cfg.get("suite") or slug + cfg["suite"] = suite frozen = frozen_dir / f"{slug}.yaml" with frozen.open("w", encoding="utf-8") as fh: @@ -101,7 +115,7 @@ def plan(args: argparse.Namespace) -> int: { "name": name, "slug": slug, - "suite": cfg.get("suite"), + "suite": suite, "config": str(config_path).replace(os.sep, "/"), "frozen": str(frozen).replace(os.sep, "/"), "artifacts_root": str(artifacts_root).replace(os.sep, "/"), diff --git a/tests/test_plan_behaviors.py b/tests/test_plan_behaviors.py index ca1bc47..2d5a2ca 100644 --- a/tests/test_plan_behaviors.py +++ b/tests/test_plan_behaviors.py @@ -412,3 +412,75 @@ def test_resolve_returns_outer_run_dir_for_nested_layouts(tmp_path, monkeypatch) f"_find_run_dir picked the inner generation dir instead of the outer eval dir" ) assert (Path(entry[role]) / "suite.json").is_file() + + +def test_plan_pins_suite_to_slug_when_config_omits_it(tmp_path, monkeypatch) -> None: + """Regression: when a behavior config has no `suite:`, `assert-ai` defaults + `suite_id` to `eval-` and every dispatch writes into a fresh + `results//`. The cache lookup never crosses dispatches, so the + paired McNemar gate re-generates the test_set every run and drift always + fires. Pin `suite` to the behavior slug so consecutive runs share one + `results//` and the artifact cache can actually reuse the frozen + test_set from the baseline dispatch. + """ + monkeypatch.chdir(tmp_path) + # No `suite:` in this fixture; matches the shape of banking/foundry demos. + (tmp_path / "eval" / "behaviors").mkdir(parents=True) + (tmp_path / "eval" / "behaviors" / "leakage.yaml").write_text( + yaml.safe_dump( + { + "behavior": {"name": "leakage", "description": "..."}, + "pipeline": {"inference": {"target": {"callable": "agent:chat"}}}, + }, + sort_keys=False, + ), + encoding="utf-8", + ) + + rc = plan_behaviors.main( + [ + "plan", + "--configs", + "eval/behaviors/*.yaml", + "--artifacts-root", + str(tmp_path / "arts"), + "--out", + "manifest.json", + ] + ) + assert rc == 0 + + manifest = json.loads((tmp_path / "manifest.json").read_text(encoding="utf-8")) + assert manifest[0]["suite"] == manifest[0]["slug"] + + frozen = yaml.safe_load(Path(manifest[0]["frozen"]).read_text(encoding="utf-8")) + assert frozen["suite"] == manifest[0]["slug"], ( + "the frozen config must carry the pinned suite so `assert-ai run` picks " + "it up as its suite_id instead of defaulting to eval-" + ) + + +def test_plan_preserves_user_supplied_suite(tmp_path, monkeypatch) -> None: + """A user who already set `suite:` in their config keeps their own layout; + we only default to the slug when the field is unset. + """ + monkeypatch.chdir(tmp_path) + _write_config(tmp_path / "eval" / "behaviors" / "leakage.yaml", "custom-suite-id", "leakage") + + rc = plan_behaviors.main( + [ + "plan", + "--configs", + "eval/behaviors/*.yaml", + "--artifacts-root", + str(tmp_path / "arts"), + "--out", + "manifest.json", + ] + ) + assert rc == 0 + + manifest = json.loads((tmp_path / "manifest.json").read_text(encoding="utf-8")) + assert manifest[0]["suite"] == "custom-suite-id" + frozen = yaml.safe_load(Path(manifest[0]["frozen"]).read_text(encoding="utf-8")) + assert frozen["suite"] == "custom-suite-id"