fix(metrics): finish permissibility metric transition

assert_ai/results.py

@@ -264,6 +264,32 @@ def summarize(permissible: bool) -> dict[str, Any]:
     }
 
 
+def has_permissibility_split_data(*metric_sets: Any) -> bool:
+    """Return whether a computed permissibility split contains usable rows.
+
+    Key presence alone is insufficient after a suite taxonomy is regenerated:
+    older runs can still receive split-shaped summaries whose two buckets both
+    have zero matching judgments. A one-sided taxonomy remains valid because
+    its populated bucket has data even though the other bucket is empty.
+    """
+    key_pairs = (
+        ("policy_violation_on_permissible", "permissible_policy_violation_rate"),
+        ("policy_violation_on_not_permissible", "not_permissible_policy_violation_rate"),
+    )
+    for metrics in metric_sets:
+        if not isinstance(metrics, dict):
+            continue
+        for summary_key, rate_key in key_pairs:
+            summary = metrics.get(summary_key)
+            count = summary.get("count") if isinstance(summary, dict) else None
+            if isinstance(count, (int, float)) and not isinstance(count, bool) and count > 0:
+                return True
+            rate = metrics.get(rate_key)
+            if isinstance(rate, (int, float)) and not isinstance(rate, bool):
+                return True
+    return False
+
+
 def _first_str(rows: Iterable[dict[str, Any]], key: str) -> str:
     for row in rows:
         value = row.get(key)

assert_ai/runner.py

@@ -514,17 +514,19 @@ def _log_run_headline(run_root: Path) -> None:
     """Log the same headline numbers a user sees on the viewer's run page.
 
     Pulls scores from ``run_root/scores.jsonl`` and prints target/judge plus the
-    headline rates (policy violation, overrefusal, judge failure). Silently
-    does nothing if the judge stage hasn't produced scores yet — that matches
-    the viewer's behavior, which only shows the headline once scores exist.
+    permissibility-split rates and judge failure. Runs without a behavior
+    taxonomy retain the legacy policy-violation/overrefusal fallback. Silently
+    does nothing if the judge stage hasn't produced scores yet, matching the
+    viewer's behavior.
     """
     # Imported lazily to avoid a hard dependency for callers that import the
     # runner without ever invoking it (e.g. test scaffolding).
     from assert_ai.results import (
         compute_prompt_metrics,
         compute_scenario_metrics,
+        has_permissibility_split_data,
     )
-    from assert_ai.core.io import load_jsonl
+    from assert_ai.core.io import load_json, load_jsonl
 
     scores_path = run_root / "scores.jsonl"
     if not scores_path.exists():
@@ -535,8 +537,15 @@ def _log_run_headline(run_root: Path) -> None:
 
     prompt_rows = [row for row in score_rows if not row.get("tester_model")]
     scenario_rows = [row for row in score_rows if row.get("tester_model")]
-    prompt_metrics = compute_prompt_metrics(prompt_rows)
-    scenario_metrics = compute_scenario_metrics(scenario_rows)
+    taxonomy = load_json(run_root.parent / "taxonomy.json")
+    raw_categories = (taxonomy or {}).get("behavior_categories")
+    behavior_categories = (
+        [entry for entry in raw_categories if isinstance(entry, dict)]
+        if isinstance(raw_categories, list)
+        else []
+    )
+    prompt_metrics = compute_prompt_metrics(prompt_rows, behavior_categories)
+    scenario_metrics = compute_scenario_metrics(scenario_rows, behavior_categories)
     primary = prompt_metrics or scenario_metrics
     if primary is None:
         return
@@ -565,16 +574,30 @@ def _emit(label: str, prompt_value: Any, scenario_value: Any) -> None:
         if parts:
             log.info(f"  {label}: {' · '.join(parts)}")
 
-    _emit(
-        label_metric("policy_violation_rate"),
-        (prompt_metrics or {}).get("policy_violation_rate"),
-        (scenario_metrics or {}).get("policy_violation_rate"),
-    )
-    _emit(
-        label_metric("overrefusal_rate"),
-        (prompt_metrics or {}).get("overrefusal_rate"),
-        (scenario_metrics or {}).get("overrefusal_rate"),
-    )
+    metric_sets = (prompt_metrics or {}, scenario_metrics or {})
+    has_permissibility_split = has_permissibility_split_data(*metric_sets)
+    if has_permissibility_split:
+        _emit(
+            label_metric("not_permissible_policy_violation_rate"),
+            metric_sets[0].get("not_permissible_policy_violation_rate"),
+            metric_sets[1].get("not_permissible_policy_violation_rate"),
+        )
+        _emit(
+            label_metric("permissible_policy_violation_rate"),
+            metric_sets[0].get("permissible_policy_violation_rate"),
+            metric_sets[1].get("permissible_policy_violation_rate"),
+        )
+    else:
+        _emit(
+            label_metric("policy_violation_rate"),
+            metric_sets[0].get("policy_violation_rate"),
+            metric_sets[1].get("policy_violation_rate"),
+        )
+        _emit(
+            label_metric("overrefusal_rate"),
+            metric_sets[0].get("overrefusal_rate"),
+            metric_sets[1].get("overrefusal_rate"),
+        )
     _emit(
         label_metric("judge_failure_rate"),
         (prompt_metrics or {}).get("judge_failure_rate"),

docs/cli/commands.md

@@ -117,7 +117,7 @@ assert-ai results compare <suite1>/<run1> <suite2>/<run2> [suite3/run3 ...] [OPT
 Options:
 
 - `--results-dir <path>` optional
-- `--metric <dimension>` optional, default `policy_violation`
+- `--metric <dimension>` optional; defaults to `policy_violation_not_permissible` when every compared run has permissibility-split data, otherwise `policy_violation`
 - `--limit <int>` optional, default `8`
 - `--json` optional flag
 - `--no-color` optional flag
@@ -133,7 +133,7 @@ assert-ai results compare-suites <suite1>/<run1> <suite2>/<run2> [OPTIONS]
 Options:
 
 - `--results-dir <path>` optional
-- `--metric <dimension>` optional
+- `--metric <dimension>` optional; defaults to `policy_violation_not_permissible` when every compared run has permissibility-split data, otherwise `policy_violation`
 - `--json` optional flag
 - `--no-color` optional flag
 

pyproject.toml

@@ -44,7 +44,10 @@ dependencies = [
 
 [project.optional-dependencies]
 otel = [
-  "arize-phoenix>=15.0.0",
+  # Phoenix 19.18.0-19.19.0 cannot import on supported Python 3.11: its
+  # dataclass uses MappingProxyType values as mutable defaults. Keep the last
+  # working release until upstream switches those fields to default_factory.
+  "arize-phoenix>=15.0.0,<19.18.0",
   "arize-phoenix-otel>=0.15.0",
   "openinference-instrumentation-langchain>=0.1.62",
 ]