fix(cache): warm assert-ai's artifact cache from the baseline so paired McNemar can reach PASS

action.yml

@@ -336,6 +336,42 @@ runs:
         run-id: ${{ steps.baseline-run.outputs.run-id }}
         github-token: ${{ github.token }}
 
+    - name: Warm the artifact cache from the baseline
+      # `assert-ai run` has a fingerprint-based artifact cache under each
+      # behavior's `artifacts_root`. `actions/checkout` starts every job with
+      # an empty workspace, so the cache is cold and the test_set stage
+      # regenerates on every dispatch even when nothing changed. The paired
+      # McNemar gate cannot reach PASS until the current run's cache is
+      # seeded with the baseline artifacts. Copy each behavior's baseline
+      # tree into its `artifacts_root` here; `assert-ai run` then hydrates
+      # its context from the baseline's `latest.json`, reuses the frozen
+      # test_set (identical fingerprint), and only re-runs downstream stages
+      # whose fingerprints actually changed (a target-code change on a real
+      # PR still misses the inference cache and scores fresh, as intended).
+      if: ${{ steps.resolve-baseline.outputs.needs-download == 'true' && steps.baseline-run.outputs.found == 'true' }}
+      shell: bash
+      run: |
+        python <<'PY'
+        import json
+        import shutil
+        from pathlib import Path
+
+        behaviors = json.loads(Path("assert-ai-behaviors.json").read_text(encoding="utf-8"))
+        baseline_root = Path("assert-ai-baseline").resolve()
+
+        for b in behaviors:
+            src = baseline_root / b["slug"]
+            dst = Path(b["artifacts_root"])
+            if not src.exists():
+                print(f"[warm-cache] no baseline dir for {b['slug']}, skipping")
+                continue
+            dst.mkdir(parents=True, exist_ok=True)
+            # dirs_exist_ok=True lets Python 3.11+ merge trees; any files the
+            # baseline doesn't cover fall through to the fresh run's own writes.
+            shutil.copytree(src, dst, dirs_exist_ok=True)
+            print(f"[warm-cache] seeded {b['slug']} from baseline")
+        PY
+
     - name: Export provider credentials
       shell: bash
       env:

scripts/plan_behaviors.py

@@ -92,6 +92,20 @@ def plan(args: argparse.Namespace) -> int:
 
         artifacts_root = artifacts_base / slug
         cfg["artifacts_root"] = str(artifacts_root)
+        # Pin the suite_id so every dispatch of the same behavior writes into
+        # the same `results/<suite>/` directory. Without this, `assert-ai`
+        # defaults `suite_id` to `eval-<current-timestamp>` on every run, so
+        # the previous run's cached test_set / inference / judge artifacts
+        # live at a path the current run never looks at. The paired McNemar
+        # gate then re-generates the test_set from scratch, the drift
+        # detector correctly reports `TestSetChanged`, and no-code-change PRs
+        # can never reach PASS.
+        #
+        # Respect a user-supplied `suite:` when present so anyone already
+        # relying on their own suite naming keeps that layout; only default
+        # to the slug when the config left it unset.
+        suite = cfg.get("suite") or slug
+        cfg["suite"] = suite
 
         frozen = frozen_dir / f"{slug}.yaml"
         with frozen.open("w", encoding="utf-8") as fh:
@@ -101,7 +115,7 @@ def plan(args: argparse.Namespace) -> int:
             {
                 "name": name,
                 "slug": slug,
-                "suite": cfg.get("suite"),
+                "suite": suite,
                 "config": str(config_path).replace(os.sep, "/"),
                 "frozen": str(frozen).replace(os.sep, "/"),
                 "artifacts_root": str(artifacts_root).replace(os.sep, "/"),

tests/test_plan_behaviors.py

@@ -412,3 +412,75 @@ def test_resolve_returns_outer_run_dir_for_nested_layouts(tmp_path, monkeypatch)
             f"_find_run_dir picked the inner generation dir instead of the outer eval dir"
         )
         assert (Path(entry[role]) / "suite.json").is_file()
+
+
+def test_plan_pins_suite_to_slug_when_config_omits_it(tmp_path, monkeypatch) -> None:
+    """Regression: when a behavior config has no `suite:`, `assert-ai` defaults
+    `suite_id` to `eval-<timestamp>` and every dispatch writes into a fresh
+    `results/<eval-XXX>/`. The cache lookup never crosses dispatches, so the
+    paired McNemar gate re-generates the test_set every run and drift always
+    fires. Pin `suite` to the behavior slug so consecutive runs share one
+    `results/<slug>/` and the artifact cache can actually reuse the frozen
+    test_set from the baseline dispatch.
+    """
+    monkeypatch.chdir(tmp_path)
+    # No `suite:` in this fixture; matches the shape of banking/foundry demos.
+    (tmp_path / "eval" / "behaviors").mkdir(parents=True)
+    (tmp_path / "eval" / "behaviors" / "leakage.yaml").write_text(
+        yaml.safe_dump(
+            {
+                "behavior": {"name": "leakage", "description": "..."},
+                "pipeline": {"inference": {"target": {"callable": "agent:chat"}}},
+            },
+            sort_keys=False,
+        ),
+        encoding="utf-8",
+    )
+
+    rc = plan_behaviors.main(
+        [
+            "plan",
+            "--configs",
+            "eval/behaviors/*.yaml",
+            "--artifacts-root",
+            str(tmp_path / "arts"),
+            "--out",
+            "manifest.json",
+        ]
+    )
+    assert rc == 0
+
+    manifest = json.loads((tmp_path / "manifest.json").read_text(encoding="utf-8"))
+    assert manifest[0]["suite"] == manifest[0]["slug"]
+
+    frozen = yaml.safe_load(Path(manifest[0]["frozen"]).read_text(encoding="utf-8"))
+    assert frozen["suite"] == manifest[0]["slug"], (
+        "the frozen config must carry the pinned suite so `assert-ai run` picks "
+        "it up as its suite_id instead of defaulting to eval-<timestamp>"
+    )
+
+
+def test_plan_preserves_user_supplied_suite(tmp_path, monkeypatch) -> None:
+    """A user who already set `suite:` in their config keeps their own layout;
+    we only default to the slug when the field is unset.
+    """
+    monkeypatch.chdir(tmp_path)
+    _write_config(tmp_path / "eval" / "behaviors" / "leakage.yaml", "custom-suite-id", "leakage")
+
+    rc = plan_behaviors.main(
+        [
+            "plan",
+            "--configs",
+            "eval/behaviors/*.yaml",
+            "--artifacts-root",
+            str(tmp_path / "arts"),
+            "--out",
+            "manifest.json",
+        ]
+    )
+    assert rc == 0
+
+    manifest = json.loads((tmp_path / "manifest.json").read_text(encoding="utf-8"))
+    assert manifest[0]["suite"] == "custom-suite-id"
+    frozen = yaml.safe_load(Path(manifest[0]["frozen"]).read_text(encoding="utf-8"))
+    assert frozen["suite"] == "custom-suite-id"