From 057fafb58105fdd10f7293ca9b5eb3fcda92c6aa Mon Sep 17 00:00:00 2001 From: Jake Present Date: Thu, 6 Aug 2026 16:40:30 -0400 Subject: [PATCH 1/4] fix(cli): finish permissibility metric alignment --- assert_ai/cli.py | 185 ++++++++++++++++++++++------- tests/test_results.py | 262 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 406 insertions(+), 41 deletions(-) diff --git a/assert_ai/cli.py b/assert_ai/cli.py index b70cfb51..7914108a 100644 --- a/assert_ai/cli.py +++ b/assert_ai/cli.py @@ -42,6 +42,18 @@ DEFAULT_COMPARE_METRIC = "policy_violation" +_POLICY_VIOLATION_NOT_PERMISSIBLE = "policy_violation_not_permissible" +_POLICY_VIOLATION_PERMISSIBLE = "policy_violation_permissible" +_DERIVED_PERMISSIBILITY_RATE_KEYS = { + _POLICY_VIOLATION_NOT_PERMISSIBLE: "not_permissible_policy_violation_rate", + _POLICY_VIOLATION_PERMISSIBLE: "permissible_policy_violation_rate", +} +_DERIVED_PERMISSIBILITY_SUMMARY_KEYS = { + _POLICY_VIOLATION_NOT_PERMISSIBLE: "policy_violation_on_not_permissible", + _POLICY_VIOLATION_PERMISSIBLE: "policy_violation_on_permissible", +} +_SUPERSEDED_DISPLAY_METRICS = {"policy_violation", "overrefusal"} + _RUNNER_MODULE: Any | None = None _TEST_SET_METRICS_MODULE: Any | None = None @@ -137,10 +149,7 @@ def _fmt_percent(value: Optional[float]) -> str: return f"{value * 100:.1f}%" -_PERMISSIBILITY_SPLIT_RATE_KEYS = ( - "not_permissible_policy_violation_rate", - "permissible_policy_violation_rate", -) +_PERMISSIBILITY_SPLIT_RATE_KEYS = tuple(_DERIVED_PERMISSIBILITY_RATE_KEYS.values()) def _has_permissibility_split(*metric_sets: Any) -> bool: @@ -165,12 +174,13 @@ def _has_permissibility_split(*metric_sets: Any) -> bool: ) -def _violation_column_titles(split: bool) -> tuple[str, str, str]: +def _violation_column_titles(split: bool) -> tuple[str, ...]: if split: return ( "Prompt impermissible violations", "Prompt permissible violations", "Scenario impermissible violations", + "Scenario permissible violations", ) return ( "Prompt policy violations", @@ -183,12 +193,13 @@ def _violation_cells( prompt_metrics: dict[str, Any], scenario_metrics: dict[str, Any], split: bool, -) -> tuple[str, str, str]: +) -> tuple[str, ...]: if split: return ( _fmt_percent(prompt_metrics.get("not_permissible_policy_violation_rate")), _fmt_percent(prompt_metrics.get("permissible_policy_violation_rate")), _fmt_percent(scenario_metrics.get("not_permissible_policy_violation_rate")), + _fmt_percent(scenario_metrics.get("permissible_policy_violation_rate")), ) return ( _fmt_percent(_dimension_rate(prompt_metrics, "policy_violation")), @@ -232,6 +243,34 @@ def _fmt_dimension_summary(summary: dict[str, Any]) -> tuple[str, str, str]: ) +def _visible_dimension_summaries(metrics: dict[str, Any]) -> list[tuple[str, dict[str, Any]]]: + """Return the dimensions shown by default on CLI detail surfaces. + + The raw built-ins stay in artifacts and JSON for compatibility. When the + permissibility split is available, the text UI replaces that older pair + with the same two derived measures used by the viewer. + """ + raw_dimensions = metrics.get("dimensions") + dimensions = raw_dimensions if isinstance(raw_dimensions, dict) else {} + visible: list[tuple[str, dict[str, Any]]] = [] + + if _has_permissibility_split(metrics): + for metric in (_POLICY_VIOLATION_NOT_PERMISSIBLE, _POLICY_VIOLATION_PERMISSIBLE): + summary = metrics.get(_DERIVED_PERMISSIBILITY_SUMMARY_KEYS[metric]) + if isinstance(summary, dict): + visible.append((metric, summary)) + hidden = _SUPERSEDED_DISPLAY_METRICS + else: + hidden = set() + + visible.extend( + (name, summary) + for name, summary in sorted(dimensions.items()) + if name not in hidden and isinstance(summary, dict) + ) + return visible + + def _metric_label(metric: str) -> str: return label_metric(metric) @@ -401,7 +440,7 @@ def _load_dimensions() -> dict[str, Any]: def _complete_metric(_: click.Context, __: click.Parameter, incomplete: str) -> list[CompletionItem]: dims = _load_dimensions() - items = sorted(dims.keys()) + items = sorted(set(dims) | set(_DERIVED_PERMISSIBILITY_RATE_KEYS)) return [CompletionItem(name) for name in items if not incomplete or name.startswith(incomplete)] @@ -428,6 +467,10 @@ def _compute_dimension_summary(rows: Iterable[dict[str, Any]], metric: str) -> d def _dimension_rate(metrics: dict[str, Any], metric: str) -> float | None: + derived_rate_key = _DERIVED_PERMISSIBILITY_RATE_KEYS.get(metric) + if derived_rate_key is not None: + rate = metrics.get(derived_rate_key) + return float(rate) if isinstance(rate, (int, float)) else None dimensions = metrics.get("dimensions") if not isinstance(dimensions, dict): return None @@ -438,6 +481,30 @@ def _dimension_rate(metrics: dict[str, Any], metric: str) -> float | None: return float(rate) if isinstance(rate, (int, float)) else None +def _resolve_compare_metric(metric: str | None, run_summaries: Iterable[dict[str, Any]]) -> str: + if metric: + return metric + for run_summary in run_summaries: + if _has_permissibility_split( + run_summary.get("prompt_metrics") or {}, + run_summary.get("scenario_metrics") or {}, + ): + return _POLICY_VIOLATION_NOT_PERMISSIBLE + return DEFAULT_COMPARE_METRIC + + +def _available_compare_metrics(run_summaries: Iterable[dict[str, Any]]) -> set[str]: + available: set[str] = set() + for run_summary in run_summaries: + available.update(_detect_dimensions(run_summary.get("prompt_rows") or [])) + if _has_permissibility_split( + run_summary.get("prompt_metrics") or {}, + run_summary.get("scenario_metrics") or {}, + ): + available.update(_DERIVED_PERMISSIBILITY_RATE_KEYS) + return available + + def _reject_ordinal_compare(run_summaries: Iterable[dict[str, Any]], metric: str) -> None: for run_summary in run_summaries: for key in ("prompt_metrics", "scenario_metrics"): @@ -588,13 +655,18 @@ def _compute_scenario_metrics( return metrics +def _load_behavior_categories(suite_dir: Path) -> list[dict[str, Any]]: + taxonomy = load_json(suite_dir / "taxonomy.json") + behavior_categories = (taxonomy or {}).get("behavior_categories") + if not isinstance(behavior_categories, list): + return [] + return [entry for entry in behavior_categories if isinstance(entry, dict)] + + def _load_run_summary(run_dir: Path) -> dict[str, Any] | None: manifest = load_json(run_dir / "manifest.json") score_rows = load_jsonl(run_dir / "scores.jsonl") - taxonomy = load_json(run_dir.parent / "taxonomy.json") - behavior_categories = (taxonomy or {}).get("behavior_categories") - if not isinstance(behavior_categories, list): - behavior_categories = [] + behavior_categories = _load_behavior_categories(run_dir.parent) prompt_rows = [row for row in score_rows if not row.get("tester_model")] scenario_rows = [row for row in score_rows if row.get("tester_model")] @@ -698,12 +770,33 @@ def _load_all_suites(results_dir: Path) -> list[dict[str, Any]]: return suites -def _behavior_category_metric_map(rows: Iterable[dict[str, Any]], metric: str) -> dict[str, dict[str, Any]]: +def _row_metric_value( + row: dict[str, Any], + metric: str, + behavior_categories: Iterable[dict[str, Any]] = (), +) -> bool | int | None: + if metric in _DERIVED_PERMISSIBILITY_RATE_KEYS: + split = compute_policy_violation_by_permissibility([row], behavior_categories) + bucket = "not_permissible" if metric == _POLICY_VIOLATION_NOT_PERMISSIBLE else "permissible" + summary = split.get(bucket) + if not isinstance(summary, dict) or not summary.get("count"): + return None + return bool(summary.get("flagged_count")) + value = get_verdict_dimension(row.get("verdict"), metric) + return value if is_valid_event_flag(value) else None + + +def _behavior_category_metric_map( + rows: Iterable[dict[str, Any]], + metric: str, + behavior_categories: Iterable[dict[str, Any]] = (), +) -> dict[str, dict[str, Any]]: grouped: dict[str, dict[str, Any]] = {} + categories = list(behavior_categories) for row in rows: if infer_judge_status(row) != "ok": continue - value = get_verdict_dimension(row.get("verdict"), metric) + value = _row_metric_value(row, metric, categories) if not is_valid_event_flag(value): continue behavior_category = row_behavior(row) @@ -715,7 +808,7 @@ def _behavior_category_metric_map(rows: Iterable[dict[str, Any]], metric: str) - "permissible": get_permissible_flag(row), }, ) - bucket["true_count"] += int(value) + bucket["true_count"] += int(bool(value)) bucket["count"] += 1 result = {} for behavior_category, bucket in grouped.items(): @@ -914,14 +1007,12 @@ def results_list(results_dir: Path, suite: Optional[str], as_json: bool, no_colo ) for run_summary in suite_summary["runs"] ) - prompt_primary, prompt_secondary, scenario_primary = _violation_column_titles(split) table = Table(title=f"Runs in {suite}", box=None, show_header=True, show_edge=False, pad_edge=False) table.add_column("Run", style="cyan", no_wrap=True) table.add_column("Status", style="white", no_wrap=True) table.add_column("Started", style="dim", no_wrap=True) - table.add_column(prompt_primary, style="white", no_wrap=True) - table.add_column(prompt_secondary, style="white", no_wrap=True) - table.add_column(scenario_primary, style="white", no_wrap=True) + for title in _violation_column_titles(split): + table.add_column(title, style="white", no_wrap=True) table.add_column("Judge failures", style="white", no_wrap=True) table.add_column("Target", style="white") for run_summary in suite_summary["runs"]: @@ -1019,14 +1110,12 @@ def results_status(suite: str, run: Optional[str], results_dir: Path, as_json: b ) for run_summary in suite_summary["runs"] ) - prompt_primary, prompt_secondary, scenario_primary = _violation_column_titles(split) table = Table(title="Runs", box=None, show_header=True, show_edge=False, pad_edge=False) table.add_column("Run", style="cyan", no_wrap=True) table.add_column("Status", style="white", no_wrap=True) table.add_column("Current Stage", style="white", no_wrap=True) - table.add_column(prompt_primary, style="white", no_wrap=True) - table.add_column(prompt_secondary, style="white", no_wrap=True) - table.add_column(scenario_primary, style="white", no_wrap=True) + for title in _violation_column_titles(split): + table.add_column(title, style="white", no_wrap=True) for run_summary in suite_summary["runs"]: prompt_metrics = run_summary.get("prompt_metrics") or {} scenario_metrics = run_summary.get("scenario_metrics") or {} @@ -1095,13 +1184,14 @@ def results_status(suite: str, run: Optional[str], results_dir: Path, as_json: b table.add_row("Scored", str(prompt_metrics["scored_total"])) table.add_row(label_metric("judge_failure_rate"), _fmt_percent(prompt_metrics.get("judge_failure_rate"))) console.print(table) - if prompt_metrics.get("dimensions"): + prompt_dimensions = _visible_dimension_summaries(prompt_metrics) + if prompt_dimensions: dim_table = Table(title="Prompt Dimensions", box=None, show_header=True, show_edge=False, pad_edge=False) dim_table.add_column("Dimension", style="cyan", no_wrap=True) dim_table.add_column("Summary", style="white", no_wrap=True) dim_table.add_column("Scored", style="white", no_wrap=True) dim_table.add_column("Distribution", style="white", no_wrap=True) - for name, summary in sorted(prompt_metrics["dimensions"].items()): + for name, summary in prompt_dimensions: summary_text, scored_text, distribution_text = _fmt_dimension_summary(summary) dim_table.add_row( label_metric(name), @@ -1123,13 +1213,14 @@ def results_status(suite: str, run: Optional[str], results_dir: Path, as_json: b table.add_row("Scored", str(scenario_metrics["scored_total"])) table.add_row(label_metric("judge_failure_rate"), _fmt_percent(scenario_metrics.get("judge_failure_rate"))) console.print(table) - if scenario_metrics.get("dimensions"): + scenario_dimensions = _visible_dimension_summaries(scenario_metrics) + if scenario_dimensions: dim_table = Table(title="Scenario Dimensions", box=None, show_header=True, show_edge=False, pad_edge=False) dim_table.add_column("Dimension", style="cyan", no_wrap=True) dim_table.add_column("Summary", style="white", no_wrap=True) dim_table.add_column("Scored", style="white", no_wrap=True) dim_table.add_column("Distribution", style="white", no_wrap=True) - for name, summary in sorted(scenario_metrics["dimensions"].items()): + for name, summary in scenario_dimensions: summary_text, scored_text, distribution_text = _fmt_dimension_summary(summary) dim_table.add_row( label_metric(name), @@ -1151,10 +1242,12 @@ def results_status(suite: str, run: Optional[str], results_dir: Path, as_json: b ) @click.option( "--metric", - default=DEFAULT_COMPARE_METRIC, + default=None, shell_complete=_complete_metric, - show_default=True, - help="Judge dimension to use for the top behavior-category delta table.", + help=( + "Judge dimension to use for the top behavior-category delta table. " + "Defaults to impermissible behavior violations when the split is available." + ), ) @click.option("--limit", default=8, show_default=True, type=int, help="Maximum behavior categories to show in the delta table.") @click.option("--json", "as_json", is_flag=True, help="Emit machine-readable JSON instead of tables.") @@ -1164,7 +1257,7 @@ def results_compare( ctx: click.Context, args: tuple[str, ...], results_dir: Path, - metric: str, + metric: str | None, limit: int, as_json: bool, no_color: bool, @@ -1224,7 +1317,7 @@ def _run_within_suite_compare( suite: str, runs: tuple[str, ...] | list[str], results_dir: Path, - metric: str, + metric: str | None, limit: int, as_json: bool, no_color: bool, @@ -1243,17 +1336,25 @@ def _run_within_suite_compare( _error(f"Run not found or unreadable: {suite}/{run_id}") run_summaries.append(run_summary) - available_metrics: set[str] = set() - for run_summary in run_summaries: - available_metrics.update(_detect_dimensions(run_summary.get("prompt_rows") or [])) + metric = _resolve_compare_metric(metric, run_summaries) + available_metrics = _available_compare_metrics(run_summaries) if metric not in available_metrics: _error(f"Metric '{metric}' was not found in the compared prompt judgments. Available: {sorted(available_metrics)}") _reject_ordinal_compare(run_summaries, metric) behavior_category_deltas: list[dict[str, Any]] = [] if all(run_summary.get("prompt_rows") for run_summary in run_summaries): - first_map = _behavior_category_metric_map(run_summaries[0]["prompt_rows"], metric) - last_map = _behavior_category_metric_map(run_summaries[-1]["prompt_rows"], metric) + behavior_categories = _load_behavior_categories(suite_dir) + first_map = _behavior_category_metric_map( + run_summaries[0]["prompt_rows"], + metric, + behavior_categories, + ) + last_map = _behavior_category_metric_map( + run_summaries[-1]["prompt_rows"], + metric, + behavior_categories, + ) for behavior_category in sorted(set(first_map) | set(last_map)): first = first_map.get(behavior_category) last = last_map.get(behavior_category) @@ -1368,16 +1469,19 @@ def _run_within_suite_compare( ) @click.option( "--metric", - default=DEFAULT_COMPARE_METRIC, - show_default=True, - help="Judge dimension to compare.", + default=None, + shell_complete=_complete_metric, + help=( + "Judge dimension to compare. Defaults to impermissible behavior " + "violations when the split is available." + ), ) @click.option("--json", "as_json", is_flag=True, help="Emit machine-readable JSON instead of tables.") @click.option("--no-color", is_flag=True, help="Disable colored terminal output.") def results_compare_suites( suite_runs: tuple[str, ...], results_dir: Path, - metric: str, + metric: str | None, as_json: bool, no_color: bool, ): @@ -1420,6 +1524,7 @@ def results_compare_suites( run_summaries.append(run_summary) labels.append(f"{suite_id}/{run_id}") + metric = _resolve_compare_metric(metric, run_summaries) _reject_ordinal_compare(run_summaries, metric) # Count structural visibility from inference rows diff --git a/tests/test_results.py b/tests/test_results.py index 3ecd0969..99bfee43 100644 --- a/tests/test_results.py +++ b/tests/test_results.py @@ -1,12 +1,18 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. +import json import unittest +from pathlib import Path +from tempfile import TemporaryDirectory + +from click.testing import CliRunner from assert_ai.cli import ( _has_permissibility_split, _violation_cells, _violation_column_titles, + cli, ) from assert_ai.results import ( compute_policy_violation_by_permissibility, @@ -198,9 +204,10 @@ def test_all_permissible_taxonomy_still_reports_the_split(self) -> None: "Prompt impermissible violations", "Prompt permissible violations", "Scenario impermissible violations", + "Scenario permissible violations", ), ) - self.assertEqual(_violation_cells(metrics, metrics, True), ("-", "50.0%", "-")) + self.assertEqual(_violation_cells(metrics, metrics, True), ("-", "50.0%", "-", "50.0%")) def test_runs_without_a_taxonomy_keep_the_superseded_pair(self) -> None: metrics = compute_prompt_metrics( @@ -229,5 +236,258 @@ def test_runs_without_a_taxonomy_keep_the_superseded_pair(self) -> None: ) +def _score_row( + *, + row_type: str, + test_case_id: str, + permissible_violation: bool, + impermissible_violation: bool, +) -> dict: + row = { + "type": row_type, + "test_case_id": test_case_id, + "dimensions": {"behavior": "allowed" if permissible_violation else "blocked"}, + "target": "target-model", + "judge_model": "judge-model", + "judge_status": "ok", + "verdict": { + "dimensions": { + "policy_violation": permissible_violation or impermissible_violation, + "overrefusal": permissible_violation, + }, + "node_judgments": [ + { + "node_index": 0, + "node_name": "allowed", + "relevant": True, + "violated": permissible_violation, + }, + { + "node_index": 1, + "node_name": "blocked", + "relevant": True, + "violated": impermissible_violation, + }, + ], + }, + } + if row_type == "scenario": + row["tester_model"] = "tester-model" + return row + + +def _write_split_results(results_root: Path, suite_id: str = "metrics-suite") -> None: + suite_dir = results_root / suite_id + suite_dir.mkdir(parents=True) + (suite_dir / "suite.json").write_text( + json.dumps({"created_at": "2026-08-06T00:00:00Z"}), + encoding="utf-8", + ) + (suite_dir / "taxonomy.json").write_text( + json.dumps( + { + "behavior": {"name": "Risk"}, + "behavior_categories": [ + {"name": "allowed", "permissible": True}, + {"name": "blocked", "permissible": False}, + ], + } + ), + encoding="utf-8", + ) + (suite_dir / "test_set.jsonl").write_text( + json.dumps({"type": "prompt", "test_case_id": "prompt", "dimensions": {"behavior": "allowed"}}) + + "\n", + encoding="utf-8", + ) + + run_rows = { + "run-1": [ + _score_row( + row_type="prompt", + test_case_id="prompt-1", + permissible_violation=True, + impermissible_violation=False, + ), + _score_row( + row_type="scenario", + test_case_id="scenario-1", + permissible_violation=False, + impermissible_violation=True, + ), + ], + "run-2": [ + _score_row( + row_type="prompt", + test_case_id="prompt-2", + permissible_violation=False, + impermissible_violation=True, + ), + _score_row( + row_type="scenario", + test_case_id="scenario-2", + permissible_violation=True, + impermissible_violation=False, + ), + ], + } + for run_id, rows in run_rows.items(): + run_dir = suite_dir / run_id + run_dir.mkdir() + (run_dir / "manifest.json").write_text( + json.dumps( + { + "status": "completed", + "started_at": f"2026-08-06T00:0{run_id[-1]}:00Z", + "ended_at": f"2026-08-06T00:0{run_id[-1]}:30Z", + "stages": {"inference": "completed", "judge": "completed"}, + } + ), + encoding="utf-8", + ) + (run_dir / "scores.jsonl").write_text( + "\n".join(json.dumps(row) for row in rows) + "\n", + encoding="utf-8", + ) + + +class ResultsCliTest(unittest.TestCase): + def setUp(self) -> None: + self.runner = CliRunner() + + def test_list_shows_both_split_metrics_for_prompts_and_scenarios(self) -> None: + with TemporaryDirectory() as tmp_dir: + results_root = Path(tmp_dir) / "results" + _write_split_results(results_root) + + result = self.runner.invoke( + cli, + [ + "results", + "list", + "--results-dir", + str(results_root), + "--suite", + "metrics-suite", + "--no-color", + ], + env={"COLUMNS": "220"}, + terminal_width=220, + ) + + self.assertEqual(result.exit_code, 0, result.output) + self.assertIn("Prompt impermissible violations", result.output) + self.assertIn("Prompt permissible violations", result.output) + self.assertIn("Scenario impermissible violations", result.output) + self.assertIn("Scenario permissible violations", result.output) + + def test_run_detail_defaults_to_split_metrics_but_json_keeps_legacy_metrics(self) -> None: + with TemporaryDirectory() as tmp_dir: + results_root = Path(tmp_dir) / "results" + _write_split_results(results_root) + args = [ + "results", + "status", + "metrics-suite", + "run-1", + "--results-dir", + str(results_root), + ] + + text_result = self.runner.invoke(cli, [*args, "--no-color"], terminal_width=180) + json_result = self.runner.invoke(cli, [*args, "--json"]) + + self.assertEqual(text_result.exit_code, 0, text_result.output) + self.assertIn("Impermissible behavior violated", text_result.output) + self.assertIn("Permissible behavior violated", text_result.output) + self.assertNotIn("Policy violation", text_result.output) + self.assertNotIn("Overrefusal", text_result.output) + + self.assertEqual(json_result.exit_code, 0, json_result.output) + payload = json.loads(json_result.output) + prompt = payload["prompt_metrics"] + self.assertIn("policy_violation_rate", prompt) + self.assertIn("overrefusal_rate", prompt) + self.assertIn("not_permissible_policy_violation_rate", prompt) + self.assertIn("permissible_policy_violation_rate", prompt) + + def test_compare_defaults_to_impermissible_split_and_accepts_permissible_split(self) -> None: + with TemporaryDirectory() as tmp_dir: + results_root = Path(tmp_dir) / "results" + _write_split_results(results_root) + base_args = [ + "results", + "compare", + "metrics-suite", + "run-1", + "run-2", + "--results-dir", + str(results_root), + "--no-color", + ] + + default_result = self.runner.invoke(cli, base_args, terminal_width=180) + permissible_result = self.runner.invoke( + cli, + [*base_args, "--metric", "policy_violation_permissible"], + terminal_width=180, + ) + + self.assertEqual(default_result.exit_code, 0, default_result.output) + self.assertIn("Run Comparison (metrics-suite, Impermissible behavior violated)", default_result.output) + self.assertIn("0.0%", default_result.output) + self.assertIn("100.0%", default_result.output) + + self.assertEqual(permissible_result.exit_code, 0, permissible_result.output) + self.assertIn("Run Comparison (metrics-suite, Permissible behavior violated)", permissible_result.output) + + def test_compare_without_taxonomy_keeps_policy_violation_default(self) -> None: + with TemporaryDirectory() as tmp_dir: + results_root = Path(tmp_dir) / "results" + _write_split_results(results_root) + (results_root / "metrics-suite" / "taxonomy.json").unlink() + + result = self.runner.invoke( + cli, + [ + "results", + "compare", + "metrics-suite", + "run-1", + "run-2", + "--results-dir", + str(results_root), + "--no-color", + ], + terminal_width=180, + ) + + self.assertEqual(result.exit_code, 0, result.output) + self.assertIn("Run Comparison (metrics-suite, Policy violation)", result.output) + + def test_cross_suite_compare_defaults_to_impermissible_split(self) -> None: + with TemporaryDirectory() as tmp_dir: + results_root = Path(tmp_dir) / "results" + _write_split_results(results_root, "metrics-a") + _write_split_results(results_root, "metrics-b") + + result = self.runner.invoke( + cli, + [ + "results", + "compare-suites", + "metrics-a/run-1", + "metrics-b/run-2", + "--results-dir", + str(results_root), + "--no-color", + ], + terminal_width=180, + ) + + self.assertEqual(result.exit_code, 0, result.output) + self.assertIn("Cross-suite comparison (impermissible behavior violated)", result.output) + + if __name__ == "__main__": unittest.main() From 4eeba4a741718c59d51589ecc2411574ef283f20 Mon Sep 17 00:00:00 2001 From: Jake Present Date: Thu, 6 Aug 2026 17:55:15 -0400 Subject: [PATCH 2/4] fix(metrics): finish permissibility display transition --- assert_ai/cli.py | 2 - assert_ai/runner.py | 56 ++++++++---- docs/cli/commands.md | 4 +- tests/test_runner_headline.py | 90 +++++++++++++++++++ viewer/src/lib/server/data.ts | 24 ----- viewer/src/lib/server/metrics.ts | 21 ----- viewer/src/lib/types.ts | 6 -- .../src/routes/suite/[suite_id]/+page.svelte | 14 --- .../suite/[suite_id]/compare/+page.svelte | 8 +- 9 files changed, 137 insertions(+), 88 deletions(-) create mode 100644 tests/test_runner_headline.py diff --git a/assert_ai/cli.py b/assert_ai/cli.py index 7914108a..978abf55 100644 --- a/assert_ai/cli.py +++ b/assert_ai/cli.py @@ -547,7 +547,6 @@ def _compute_prompt_metrics( if isinstance(row.get("judge_model"), str) and row.get("judge_model") ), ) or "-" - permissible_rows = [row for row in scored_rows if get_permissible_flag(row, default=False)] permissibility_split = compute_policy_violation_by_permissibility( scored_rows, behavior_categories, @@ -560,7 +559,6 @@ def _compute_prompt_metrics( "judge_failure_rate": judge_failures / len(rows) if rows else 0.0, "policy_violation_rate": _dimension_rate({"dimensions": dimensions}, "policy_violation"), "overrefusal_rate": _dimension_rate({"dimensions": dimensions}, "overrefusal"), - "permissible_overrefusal_rate": _compute_dimension_summary(permissible_rows, "overrefusal")["rate"], "dimensions": dimensions, "target": target, "judge_model": judge_model, diff --git a/assert_ai/runner.py b/assert_ai/runner.py index 9e98264e..65ef1285 100644 --- a/assert_ai/runner.py +++ b/assert_ai/runner.py @@ -514,9 +514,10 @@ def _log_run_headline(run_root: Path) -> None: """Log the same headline numbers a user sees on the viewer's run page. Pulls scores from ``run_root/scores.jsonl`` and prints target/judge plus the - headline rates (policy violation, overrefusal, judge failure). Silently - does nothing if the judge stage hasn't produced scores yet — that matches - the viewer's behavior, which only shows the headline once scores exist. + permissibility-split rates and judge failure. Runs without a behavior + taxonomy retain the legacy policy-violation/overrefusal fallback. Silently + does nothing if the judge stage hasn't produced scores yet, matching the + viewer's behavior. """ # Imported lazily to avoid a hard dependency for callers that import the # runner without ever invoking it (e.g. test scaffolding). @@ -524,7 +525,7 @@ def _log_run_headline(run_root: Path) -> None: compute_prompt_metrics, compute_scenario_metrics, ) - from assert_ai.core.io import load_jsonl + from assert_ai.core.io import load_json, load_jsonl scores_path = run_root / "scores.jsonl" if not scores_path.exists(): @@ -535,8 +536,15 @@ def _log_run_headline(run_root: Path) -> None: prompt_rows = [row for row in score_rows if not row.get("tester_model")] scenario_rows = [row for row in score_rows if row.get("tester_model")] - prompt_metrics = compute_prompt_metrics(prompt_rows) - scenario_metrics = compute_scenario_metrics(scenario_rows) + taxonomy = load_json(run_root.parent / "taxonomy.json") + raw_categories = (taxonomy or {}).get("behavior_categories") + behavior_categories = ( + [entry for entry in raw_categories if isinstance(entry, dict)] + if isinstance(raw_categories, list) + else [] + ) + prompt_metrics = compute_prompt_metrics(prompt_rows, behavior_categories) + scenario_metrics = compute_scenario_metrics(scenario_rows, behavior_categories) primary = prompt_metrics or scenario_metrics if primary is None: return @@ -565,16 +573,34 @@ def _emit(label: str, prompt_value: Any, scenario_value: Any) -> None: if parts: log.info(f" {label}: {' · '.join(parts)}") - _emit( - label_metric("policy_violation_rate"), - (prompt_metrics or {}).get("policy_violation_rate"), - (scenario_metrics or {}).get("policy_violation_rate"), - ) - _emit( - label_metric("overrefusal_rate"), - (prompt_metrics or {}).get("overrefusal_rate"), - (scenario_metrics or {}).get("overrefusal_rate"), + metric_sets = (prompt_metrics or {}, scenario_metrics or {}) + has_permissibility_split = any( + "not_permissible_policy_violation_rate" in metrics + or "permissible_policy_violation_rate" in metrics + for metrics in metric_sets ) + if has_permissibility_split: + _emit( + label_metric("not_permissible_policy_violation_rate"), + metric_sets[0].get("not_permissible_policy_violation_rate"), + metric_sets[1].get("not_permissible_policy_violation_rate"), + ) + _emit( + label_metric("permissible_policy_violation_rate"), + metric_sets[0].get("permissible_policy_violation_rate"), + metric_sets[1].get("permissible_policy_violation_rate"), + ) + else: + _emit( + label_metric("policy_violation_rate"), + metric_sets[0].get("policy_violation_rate"), + metric_sets[1].get("policy_violation_rate"), + ) + _emit( + label_metric("overrefusal_rate"), + metric_sets[0].get("overrefusal_rate"), + metric_sets[1].get("overrefusal_rate"), + ) _emit( label_metric("judge_failure_rate"), (prompt_metrics or {}).get("judge_failure_rate"), diff --git a/docs/cli/commands.md b/docs/cli/commands.md index 35053eb9..13213645 100644 --- a/docs/cli/commands.md +++ b/docs/cli/commands.md @@ -117,7 +117,7 @@ assert-ai results compare / / [suite3/run3 ...] [OPT Options: - `--results-dir ` optional -- `--metric ` optional, default `policy_violation` +- `--metric ` optional; defaults to `policy_violation_not_permissible` when the permissibility split is available, otherwise `policy_violation` - `--limit ` optional, default `8` - `--json` optional flag - `--no-color` optional flag @@ -133,7 +133,7 @@ assert-ai results compare-suites / / [OPTIONS] Options: - `--results-dir ` optional -- `--metric ` optional +- `--metric ` optional; defaults to `policy_violation_not_permissible` when the permissibility split is available, otherwise `policy_violation` - `--json` optional flag - `--no-color` optional flag diff --git a/tests/test_runner_headline.py b/tests/test_runner_headline.py new file mode 100644 index 00000000..01f3ebe5 --- /dev/null +++ b/tests/test_runner_headline.py @@ -0,0 +1,90 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +import json +import unittest +from pathlib import Path +from tempfile import TemporaryDirectory + +from assert_ai.runner import _log_run_headline + + +def _score_row() -> dict: + return { + "type": "prompt", + "test_case_id": "prompt-1", + "dimensions": {"behavior": "allowed"}, + "target": "target-model", + "judge_model": "judge-model", + "judge_status": "ok", + "verdict": { + "dimensions": { + "policy_violation": True, + "overrefusal": True, + }, + "node_judgments": [ + { + "node_index": 0, + "node_name": "allowed", + "relevant": True, + "violated": True, + }, + { + "node_index": 1, + "node_name": "blocked", + "relevant": True, + "violated": False, + }, + ], + }, + } + + +def _write_run(root: Path, *, with_taxonomy: bool) -> Path: + suite_dir = root / "suite" + run_dir = suite_dir / "run-1" + run_dir.mkdir(parents=True) + (run_dir / "scores.jsonl").write_text(json.dumps(_score_row()) + "\n", encoding="utf-8") + if with_taxonomy: + (suite_dir / "taxonomy.json").write_text( + json.dumps( + { + "behavior_categories": [ + {"name": "allowed", "permissible": True}, + {"name": "blocked", "permissible": False}, + ] + } + ), + encoding="utf-8", + ) + return run_dir + + +class RunHeadlineTest(unittest.TestCase): + def test_taxonomy_run_logs_permissibility_split(self) -> None: + with TemporaryDirectory() as tmp_dir: + run_dir = _write_run(Path(tmp_dir), with_taxonomy=True) + with self.assertLogs("assert_ai.runner", level="INFO") as captured: + _log_run_headline(run_dir) + + output = "\n".join(captured.output) + self.assertIn("Impermissible behavior violated rate: prompt 0.0%", output) + self.assertIn("Permissible behavior violated rate: prompt 100.0%", output) + self.assertNotIn("Policy violation rate", output) + self.assertNotIn("Overrefusal rate", output) + + def test_run_without_taxonomy_keeps_legacy_headline(self) -> None: + with TemporaryDirectory() as tmp_dir: + run_dir = _write_run(Path(tmp_dir), with_taxonomy=False) + with self.assertLogs("assert_ai.runner", level="INFO") as captured: + _log_run_headline(run_dir) + + output = "\n".join(captured.output) + self.assertIn("Policy violation rate: prompt 100.0%", output) + self.assertIn("Overrefusal rate: prompt 100.0%", output) + self.assertNotIn("Impermissible behavior violated rate", output) + self.assertNotIn("Permissible behavior violated rate", output) + + +if __name__ == "__main__": + unittest.main() diff --git a/viewer/src/lib/server/data.ts b/viewer/src/lib/server/data.ts index afea5f20..d40698e1 100644 --- a/viewer/src/lib/server/data.ts +++ b/viewer/src/lib/server/data.ts @@ -70,9 +70,6 @@ interface PromptMetricView { scoredTotal: number; judgeFailures: number; judgeFailureRate: number; - counts: BinaryCounts; - policyViolationRate: number | null; - overrefusalRate: number | null; policyViolationOnPermissible: DimensionMetrics | null; policyViolationOnNotPermissible: DimensionMetrics | null; dimensions: Record; @@ -85,9 +82,6 @@ interface AuditMetricView { scoredTotal: number; judgeFailures: number; judgeFailureRate: number; - counts: BinaryCounts; - policyViolationRate: number | null; - overrefusalRate: number | null; policyViolationOnPermissible: DimensionMetrics | null; policyViolationOnNotPermissible: DimensionMetrics | null; dimensions: Record; @@ -122,9 +116,6 @@ interface CompareRunSummary { scoredTotal: number; judgeFailures: number; judgeFailureRate: number; - policyViolationRate: number | null; - overrefusalRate: number | null; - counts: BinaryCounts; dimensions: Record; samples: JudgedSample[]; meanAgreement: number | null; @@ -763,9 +754,6 @@ function buildZeroPromptMetrics(): PromptMetricView { scoredTotal: 0, judgeFailures: 0, judgeFailureRate: 0, - counts: emptyScoreCounts(), - policyViolationRate: null, - overrefusalRate: null, policyViolationOnPermissible: null, policyViolationOnNotPermissible: null, dimensions: {}, @@ -780,9 +768,6 @@ function buildZeroAuditMetrics(): AuditMetricView { scoredTotal: 0, judgeFailures: 0, judgeFailureRate: 0, - counts: emptyScoreCounts(), - policyViolationRate: null, - overrefusalRate: null, policyViolationOnPermissible: null, policyViolationOnNotPermissible: null, dimensions: {}, @@ -799,9 +784,6 @@ function toPromptMetricView(metrics: RunMetrics | null): PromptMetricView { scoredTotal: metrics.scored_total, judgeFailures: metrics.judge_failures, judgeFailureRate: metrics.judge_failure_rate, - counts: metrics.counts, - policyViolationRate: metrics.policy_violation_rate, - overrefusalRate: metrics.overrefusal_rate, policyViolationOnPermissible: metrics.policy_violation_on_permissible, policyViolationOnNotPermissible: metrics.policy_violation_on_not_permissible, dimensions: metrics.dimensions, @@ -817,9 +799,6 @@ function toAuditMetricView(metrics: AuditRunMetrics | null): AuditMetricView { scoredTotal: metrics.scored_total, judgeFailures: metrics.judge_failures, judgeFailureRate: metrics.judge_failure_rate, - counts: metrics.counts, - policyViolationRate: metrics.policy_violation_rate, - overrefusalRate: metrics.overrefusal_rate, policyViolationOnPermissible: metrics.policy_violation_on_permissible, policyViolationOnNotPermissible: metrics.policy_violation_on_not_permissible, dimensions: metrics.dimensions, @@ -939,9 +918,6 @@ function buildCompareRunSummary( scoredTotal: metrics.scored_total, judgeFailures: metrics.judge_failures, judgeFailureRate: metrics.judge_failure_rate, - policyViolationRate: metrics.policy_violation_rate, - overrefusalRate: metrics.overrefusal_rate, - counts: metrics.counts, dimensions, samples, meanAgreement, diff --git a/viewer/src/lib/server/metrics.ts b/viewer/src/lib/server/metrics.ts index 82017e0a..0e65de1e 100644 --- a/viewer/src/lib/server/metrics.ts +++ b/viewer/src/lib/server/metrics.ts @@ -2,7 +2,6 @@ // Licensed under the MIT License. import { - getRecordFlag, getRecordMetricValue, getRequiredBaseMetricNames, isBooleanFlag, @@ -228,10 +227,6 @@ function addDimensionValue(aggregate: EventDimensionAggregate, value: unknown): } } -function dimensionRate(dimensions: Record, name: string): number | null { - return dimensions[name]?.rate ?? null; -} - export function computeAuditRunMetrics( scores: AuditScore[], behaviors: Behavior[] = [] @@ -247,12 +242,7 @@ export function computeAuditRunMetrics( .map((score) => withPermissibilitySplit(score, permissibilityIndex)); const dimensionNames = collectDimensionNames(scoredScores); const dimensionAggregates = initDimensionAggregates(dimensionNames, scoredScores); - const counts = emptyScoreCounts(); - for (const score of scoredScores) { - const policyViolation = getRecordFlag(score, 'policy_violation'); - if (policyViolation !== null) counts[policyViolation ? 1 : 0] += 1; - for (const dimensionName of dimensionNames) { const dimensionValue = getRecordMetricValue(score, dimensionName); if (dimensionValue === null) { @@ -275,9 +265,6 @@ export function computeAuditRunMetrics( scored_total: scoredTotal, judge_failures: total - scoredTotal, judge_failure_rate: total > 0 ? (total - scoredTotal) / total : 0, - counts, - policy_violation_rate: dimensionRate(dimensions, 'policy_violation'), - overrefusal_rate: dimensionRate(dimensions, 'overrefusal'), policy_violation_on_permissible: permissibilitySplit.permissible, policy_violation_on_not_permissible: permissibilitySplit.not_permissible, dimensions, @@ -302,12 +289,7 @@ export function computeRunMetrics( .map((sample) => withPermissibilitySplit(sample, permissibilityIndex)); const dimensionNames = collectDimensionNames(scoredSamples); const dimensionAggregates = initDimensionAggregates(dimensionNames, scoredSamples); - const counts = emptyScoreCounts(); - for (const sample of scoredSamples) { - const policyViolation = getRecordFlag(sample, 'policy_violation'); - if (policyViolation !== null) counts[policyViolation ? 1 : 0] += 1; - for (const dimensionName of dimensionNames) { const dimensionValue = getRecordMetricValue(sample, dimensionName); if (dimensionValue === null) { @@ -329,9 +311,6 @@ export function computeRunMetrics( judge_failures: samples.length - scoredSamples.length, judge_failure_rate: samples.length > 0 ? (samples.length - scoredSamples.length) / samples.length : 0, - counts, - policy_violation_rate: dimensionRate(dimensions, 'policy_violation'), - overrefusal_rate: dimensionRate(dimensions, 'overrefusal'), policy_violation_on_permissible: permissibilitySplit.permissible, policy_violation_on_not_permissible: permissibilitySplit.not_permissible, target: samples[0]?.target ?? '—', diff --git a/viewer/src/lib/types.ts b/viewer/src/lib/types.ts index efa63715..370b1ec1 100644 --- a/viewer/src/lib/types.ts +++ b/viewer/src/lib/types.ts @@ -303,9 +303,6 @@ export interface RunMetrics { scored_total: number; judge_failures: number; judge_failure_rate: number; - counts: BinaryCounts; - policy_violation_rate: number | null; - overrefusal_rate: number | null; policy_violation_on_permissible: DimensionMetrics | null; policy_violation_on_not_permissible: DimensionMetrics | null; target: string; @@ -333,9 +330,6 @@ export interface AuditRunMetrics { scored_total: number; judge_failures: number; judge_failure_rate: number; - counts: BinaryCounts; - policy_violation_rate: number | null; - overrefusal_rate: number | null; policy_violation_on_permissible: DimensionMetrics | null; policy_violation_on_not_permissible: DimensionMetrics | null; dimensions: Record; diff --git a/viewer/src/routes/suite/[suite_id]/+page.svelte b/viewer/src/routes/suite/[suite_id]/+page.svelte index af95aa54..126ee63a 100644 --- a/viewer/src/routes/suite/[suite_id]/+page.svelte +++ b/viewer/src/routes/suite/[suite_id]/+page.svelte @@ -416,20 +416,6 @@ expandedRunIds = next; } - function aggregateRunViolationRate(run: CombinedRunEntry): number | null { - const promptTotal = run.prompt?.metrics?.total ?? 0; - const auditTotal = run.audit?.metrics?.total ?? 0; - const total = promptTotal + auditTotal; - if (total === 0) return null; - const promptRate = run.prompt?.metrics?.policy_violation_rate; - const auditRate = run.audit?.metrics?.policy_violation_rate; - const applicableTotal = (promptRate == null ? 0 : promptTotal) + (auditRate == null ? 0 : auditTotal); - if (applicableTotal === 0) return null; - const promptViolations = promptRate == null ? 0 : promptTotal * promptRate; - const auditViolations = auditRate == null ? 0 : auditTotal * auditRate; - return (promptViolations + auditViolations) / applicableTotal; - } - function aggregateRunDimensionRate(run: CombinedRunEntry, dimension: string): number | null { const promptDim = run.prompt?.metrics?.dimensions?.[dimension]; const auditDim = run.audit?.metrics?.dimensions?.[dimension]; diff --git a/viewer/src/routes/suite/[suite_id]/compare/+page.svelte b/viewer/src/routes/suite/[suite_id]/compare/+page.svelte index 7c1c6b06..d44a81d2 100644 --- a/viewer/src/routes/suite/[suite_id]/compare/+page.svelte +++ b/viewer/src/routes/suite/[suite_id]/compare/+page.svelte @@ -128,10 +128,10 @@ let orderedRuns = $derived([ let runColor = $derived( Object.fromEntries(data.runs.map((r, i) => [r.run_id, RUN_COLORS[i]])) as Record ); -function baselineDeltaFor(run: { run_id: string; policyViolationRate: number | null; dimensions: Record }) { +function baselineDeltaFor(run: { run_id: string; dimensions: Record }) { const baseline = data.runs[baselineIdx]; - const avg = activeMetric === 'policy_violation' ? run.policyViolationRate : (run.dimensions[activeMetric]?.rate ?? null); - const baselineAvg = activeMetric === 'policy_violation' ? baseline.policyViolationRate : (baseline.dimensions[activeMetric]?.rate ?? null); + const avg = run.dimensions[activeMetric]?.rate ?? null; + const baselineAvg = baseline.dimensions[activeMetric]?.rate ?? null; const delta = avg !== null && baselineAvg !== null && run.run_id !== baseline.run_id ? avg - baselineAvg : 0; return { avg, baselineAvg, delta }; } @@ -319,7 +319,7 @@ function sampleGridMinWidth(runCount: number): string { {@const dInfo = baselineDeltaFor(run)} {@const avg = dInfo.avg} {@const delta = dInfo.delta} - {@const runScores = activeMetric === 'policy_violation' ? run.counts : (run.dimensions[activeMetric]?.counts ?? { 0: 0, 1: 0 })} + {@const runScores = run.dimensions[activeMetric]?.counts ?? { 0: 0, 1: 0 }} {@const normalizedScores = binaryCounts(runScores)} {@const pct = pctBar(normalizedScores)} {@const totalSamples = normalizedScores[0] + normalizedScores[1]} From c70f33d704946d96f8374a37cf8f92858ed64152 Mon Sep 17 00:00:00 2001 From: Jake Present Date: Wed, 12 Aug 2026 14:19:46 -0400 Subject: [PATCH 3/4] fix(cli): preserve legacy metrics for stale taxonomy --- assert_ai/cli.py | 42 +++++++------ assert_ai/results.py | 26 ++++++++ assert_ai/runner.py | 7 +-- docs/cli/commands.md | 4 +- tests/test_results.py | 115 ++++++++++++++++++++++++++++++++++ tests/test_runner_headline.py | 21 ++++++- 6 files changed, 188 insertions(+), 27 deletions(-) diff --git a/assert_ai/cli.py b/assert_ai/cli.py index 978abf55..3600a0f0 100644 --- a/assert_ai/cli.py +++ b/assert_ai/cli.py @@ -27,6 +27,7 @@ compute_dimension_summary, compute_policy_violation_by_permissibility, detect_dimensions, + has_permissibility_split_data, ) from assert_ai.stages import STAGE_NAMES @@ -254,7 +255,7 @@ def _visible_dimension_summaries(metrics: dict[str, Any]) -> list[tuple[str, dic dimensions = raw_dimensions if isinstance(raw_dimensions, dict) else {} visible: list[tuple[str, dict[str, Any]]] = [] - if _has_permissibility_split(metrics): + if has_permissibility_split_data(metrics): for metric in (_POLICY_VIOLATION_NOT_PERMISSIBLE, _POLICY_VIOLATION_PERMISSIBLE): summary = metrics.get(_DERIVED_PERMISSIBILITY_SUMMARY_KEYS[metric]) if isinstance(summary, dict): @@ -484,12 +485,15 @@ def _dimension_rate(metrics: dict[str, Any], metric: str) -> float | None: def _resolve_compare_metric(metric: str | None, run_summaries: Iterable[dict[str, Any]]) -> str: if metric: return metric - for run_summary in run_summaries: - if _has_permissibility_split( + summaries = list(run_summaries) + if summaries and all( + has_permissibility_split_data( run_summary.get("prompt_metrics") or {}, run_summary.get("scenario_metrics") or {}, - ): - return _POLICY_VIOLATION_NOT_PERMISSIBLE + ) + for run_summary in summaries + ): + return _POLICY_VIOLATION_NOT_PERMISSIBLE return DEFAULT_COMPARE_METRIC @@ -497,7 +501,7 @@ def _available_compare_metrics(run_summaries: Iterable[dict[str, Any]]) -> set[s available: set[str] = set() for run_summary in run_summaries: available.update(_detect_dimensions(run_summary.get("prompt_rows") or [])) - if _has_permissibility_split( + if has_permissibility_split_data( run_summary.get("prompt_metrics") or {}, run_summary.get("scenario_metrics") or {}, ): @@ -998,12 +1002,13 @@ def results_list(results_dir: Path, suite: Optional[str], as_json: bool, no_colo return console = _console(no_color=no_color) - split = any( - _has_permissibility_split( - run_summary.get("prompt_metrics") or {}, - run_summary.get("scenario_metrics") or {}, + runs = suite_summary["runs"] + split = bool(runs) and all( + has_permissibility_split_data( + (run_summary or {}).get("prompt_metrics") or {}, + (run_summary or {}).get("scenario_metrics") or {}, ) - for run_summary in suite_summary["runs"] + for run_summary in runs ) table = Table(title=f"Runs in {suite}", box=None, show_header=True, show_edge=False, pad_edge=False) table.add_column("Run", style="cyan", no_wrap=True) @@ -1101,12 +1106,13 @@ def results_status(suite: str, run: Optional[str], results_dir: Path, as_json: b console.print(summary) if suite_summary["runs"]: - split = any( - _has_permissibility_split( - run_summary.get("prompt_metrics") or {}, - run_summary.get("scenario_metrics") or {}, + runs = suite_summary["runs"] + split = all( + has_permissibility_split_data( + (run_summary or {}).get("prompt_metrics") or {}, + (run_summary or {}).get("scenario_metrics") or {}, ) - for run_summary in suite_summary["runs"] + for run_summary in runs ) table = Table(title="Runs", box=None, show_header=True, show_edge=False, pad_edge=False) table.add_column("Run", style="cyan", no_wrap=True) @@ -1244,7 +1250,7 @@ def results_status(suite: str, run: Optional[str], results_dir: Path, as_json: b shell_complete=_complete_metric, help=( "Judge dimension to use for the top behavior-category delta table. " - "Defaults to impermissible behavior violations when the split is available." + "Defaults to impermissible behavior violations when every compared run has split data." ), ) @click.option("--limit", default=8, show_default=True, type=int, help="Maximum behavior categories to show in the delta table.") @@ -1471,7 +1477,7 @@ def _run_within_suite_compare( shell_complete=_complete_metric, help=( "Judge dimension to compare. Defaults to impermissible behavior " - "violations when the split is available." + "violations when every compared run has split data." ), ) @click.option("--json", "as_json", is_flag=True, help="Emit machine-readable JSON instead of tables.") diff --git a/assert_ai/results.py b/assert_ai/results.py index 8d8bbb3e..86059a89 100644 --- a/assert_ai/results.py +++ b/assert_ai/results.py @@ -264,6 +264,32 @@ def summarize(permissible: bool) -> dict[str, Any]: } +def has_permissibility_split_data(*metric_sets: Any) -> bool: + """Return whether a computed permissibility split contains usable rows. + + Key presence alone is insufficient after a suite taxonomy is regenerated: + older runs can still receive split-shaped summaries whose two buckets both + have zero matching judgments. A one-sided taxonomy remains valid because + its populated bucket has data even though the other bucket is empty. + """ + key_pairs = ( + ("policy_violation_on_permissible", "permissible_policy_violation_rate"), + ("policy_violation_on_not_permissible", "not_permissible_policy_violation_rate"), + ) + for metrics in metric_sets: + if not isinstance(metrics, dict): + continue + for summary_key, rate_key in key_pairs: + summary = metrics.get(summary_key) + count = summary.get("count") if isinstance(summary, dict) else None + if isinstance(count, (int, float)) and not isinstance(count, bool) and count > 0: + return True + rate = metrics.get(rate_key) + if isinstance(rate, (int, float)) and not isinstance(rate, bool): + return True + return False + + def _first_str(rows: Iterable[dict[str, Any]], key: str) -> str: for row in rows: value = row.get(key) diff --git a/assert_ai/runner.py b/assert_ai/runner.py index 65ef1285..7de31e69 100644 --- a/assert_ai/runner.py +++ b/assert_ai/runner.py @@ -524,6 +524,7 @@ def _log_run_headline(run_root: Path) -> None: from assert_ai.results import ( compute_prompt_metrics, compute_scenario_metrics, + has_permissibility_split_data, ) from assert_ai.core.io import load_json, load_jsonl @@ -574,11 +575,7 @@ def _emit(label: str, prompt_value: Any, scenario_value: Any) -> None: log.info(f" {label}: {' · '.join(parts)}") metric_sets = (prompt_metrics or {}, scenario_metrics or {}) - has_permissibility_split = any( - "not_permissible_policy_violation_rate" in metrics - or "permissible_policy_violation_rate" in metrics - for metrics in metric_sets - ) + has_permissibility_split = has_permissibility_split_data(*metric_sets) if has_permissibility_split: _emit( label_metric("not_permissible_policy_violation_rate"), diff --git a/docs/cli/commands.md b/docs/cli/commands.md index 13213645..a8731d67 100644 --- a/docs/cli/commands.md +++ b/docs/cli/commands.md @@ -117,7 +117,7 @@ assert-ai results compare / / [suite3/run3 ...] [OPT Options: - `--results-dir ` optional -- `--metric ` optional; defaults to `policy_violation_not_permissible` when the permissibility split is available, otherwise `policy_violation` +- `--metric ` optional; defaults to `policy_violation_not_permissible` when every compared run has permissibility-split data, otherwise `policy_violation` - `--limit ` optional, default `8` - `--json` optional flag - `--no-color` optional flag @@ -133,7 +133,7 @@ assert-ai results compare-suites / / [OPTIONS] Options: - `--results-dir ` optional -- `--metric ` optional; defaults to `policy_violation_not_permissible` when the permissibility split is available, otherwise `policy_violation` +- `--metric ` optional; defaults to `policy_violation_not_permissible` when every compared run has permissibility-split data, otherwise `policy_violation` - `--json` optional flag - `--no-color` optional flag diff --git a/tests/test_results.py b/tests/test_results.py index 99bfee43..51e7ae0b 100644 --- a/tests/test_results.py +++ b/tests/test_results.py @@ -17,6 +17,7 @@ from assert_ai.results import ( compute_policy_violation_by_permissibility, compute_prompt_metrics, + has_permissibility_split_data, ) @@ -351,6 +352,32 @@ def _write_split_results(results_root: Path, suite_id: str = "metrics-suite") -> ) +def _make_run_judgments_stale(results_root: Path, run_id: str) -> None: + """Make one run predate the suite taxonomy without changing legacy scores.""" + scores_path = results_root / "metrics-suite" / run_id / "scores.jsonl" + rows = [json.loads(line) for line in scores_path.read_text(encoding="utf-8").splitlines()] + for row in rows: + for judgment in row["verdict"]["node_judgments"]: + judgment["node_index"] = 100 + int(judgment["node_index"]) + judgment["node_name"] = f"stale-{judgment['node_name']}" + scores_path.write_text( + "\n".join(json.dumps(row) for row in rows) + "\n", + encoding="utf-8", + ) + + +def _set_prompt_behavior(results_root: Path, run_id: str, behavior: str) -> None: + scores_path = results_root / "metrics-suite" / run_id / "scores.jsonl" + rows = [json.loads(line) for line in scores_path.read_text(encoding="utf-8").splitlines()] + for row in rows: + if not row.get("tester_model"): + row["dimensions"]["behavior"] = behavior + scores_path.write_text( + "\n".join(json.dumps(row) for row in rows) + "\n", + encoding="utf-8", + ) + + class ResultsCliTest(unittest.TestCase): def setUp(self) -> None: self.runner = CliRunner() @@ -381,6 +408,32 @@ def test_list_shows_both_split_metrics_for_prompts_and_scenarios(self) -> None: self.assertIn("Scenario impermissible violations", result.output) self.assertIn("Scenario permissible violations", result.output) + def test_list_falls_back_when_any_run_lacks_current_split_data(self) -> None: + with TemporaryDirectory() as tmp_dir: + results_root = Path(tmp_dir) / "results" + _write_split_results(results_root) + _make_run_judgments_stale(results_root, "run-1") + + result = self.runner.invoke( + cli, + [ + "results", + "list", + "--results-dir", + str(results_root), + "--suite", + "metrics-suite", + "--no-color", + ], + env={"COLUMNS": "220"}, + terminal_width=220, + ) + + self.assertEqual(result.exit_code, 0, result.output) + self.assertIn("Prompt policy violations", result.output) + self.assertIn("Prompt overrefusals", result.output) + self.assertNotIn("Prompt impermissible violations", result.output) + def test_run_detail_defaults_to_split_metrics_but_json_keeps_legacy_metrics(self) -> None: with TemporaryDirectory() as tmp_dir: results_root = Path(tmp_dir) / "results" @@ -411,6 +464,32 @@ def test_run_detail_defaults_to_split_metrics_but_json_keeps_legacy_metrics(self self.assertIn("not_permissible_policy_violation_rate", prompt) self.assertIn("permissible_policy_violation_rate", prompt) + def test_run_detail_falls_back_when_current_taxonomy_matches_no_judgments(self) -> None: + with TemporaryDirectory() as tmp_dir: + results_root = Path(tmp_dir) / "results" + _write_split_results(results_root) + _make_run_judgments_stale(results_root, "run-1") + + result = self.runner.invoke( + cli, + [ + "results", + "status", + "metrics-suite", + "run-1", + "--results-dir", + str(results_root), + "--no-color", + ], + terminal_width=180, + ) + + self.assertEqual(result.exit_code, 0, result.output) + self.assertIn("Policy violation", result.output) + self.assertIn("Overrefusal", result.output) + self.assertNotIn("Impermissible behavior violated", result.output) + self.assertNotIn("Permissible behavior violated", result.output) + def test_compare_defaults_to_impermissible_split_and_accepts_permissible_split(self) -> None: with TemporaryDirectory() as tmp_dir: results_root = Path(tmp_dir) / "results" @@ -441,6 +520,42 @@ def test_compare_defaults_to_impermissible_split_and_accepts_permissible_split(s self.assertEqual(permissible_result.exit_code, 0, permissible_result.output) self.assertIn("Run Comparison (metrics-suite, Permissible behavior violated)", permissible_result.output) + def test_compare_falls_back_when_any_run_lacks_current_split_data(self) -> None: + with TemporaryDirectory() as tmp_dir: + results_root = Path(tmp_dir) / "results" + _write_split_results(results_root) + _make_run_judgments_stale(results_root, "run-1") + _set_prompt_behavior(results_root, "run-2", "allowed") + + result = self.runner.invoke( + cli, + [ + "results", + "compare", + "metrics-suite", + "run-1", + "run-2", + "--results-dir", + str(results_root), + "--no-color", + ], + terminal_width=180, + ) + + self.assertEqual(result.exit_code, 0, result.output) + self.assertIn("Run Comparison (metrics-suite, Policy violation)", result.output) + self.assertIn("Top behavior category deltas", result.output) + + def test_empty_bucket_does_not_hide_a_valid_one_sided_split(self) -> None: + metrics = { + "policy_violation_on_permissible": {"count": 1, "rate": 0.0}, + "policy_violation_on_not_permissible": {"count": 0, "rate": None}, + "permissible_policy_violation_rate": 0.0, + "not_permissible_policy_violation_rate": None, + } + + self.assertTrue(has_permissibility_split_data(metrics)) + def test_compare_without_taxonomy_keeps_policy_violation_default(self) -> None: with TemporaryDirectory() as tmp_dir: results_root = Path(tmp_dir) / "results" diff --git a/tests/test_runner_headline.py b/tests/test_runner_headline.py index 01f3ebe5..9b1f4723 100644 --- a/tests/test_runner_headline.py +++ b/tests/test_runner_headline.py @@ -40,11 +40,16 @@ def _score_row() -> dict: } -def _write_run(root: Path, *, with_taxonomy: bool) -> Path: +def _write_run(root: Path, *, with_taxonomy: bool, stale_judgments: bool = False) -> Path: suite_dir = root / "suite" run_dir = suite_dir / "run-1" run_dir.mkdir(parents=True) - (run_dir / "scores.jsonl").write_text(json.dumps(_score_row()) + "\n", encoding="utf-8") + score_row = _score_row() + if stale_judgments: + for judgment in score_row["verdict"]["node_judgments"]: + judgment["node_index"] = 100 + int(judgment["node_index"]) + judgment["node_name"] = f"stale-{judgment['node_name']}" + (run_dir / "scores.jsonl").write_text(json.dumps(score_row) + "\n", encoding="utf-8") if with_taxonomy: (suite_dir / "taxonomy.json").write_text( json.dumps( @@ -85,6 +90,18 @@ def test_run_without_taxonomy_keeps_legacy_headline(self) -> None: self.assertNotIn("Impermissible behavior violated rate", output) self.assertNotIn("Permissible behavior violated rate", output) + def test_run_with_stale_judgments_keeps_legacy_headline(self) -> None: + with TemporaryDirectory() as tmp_dir: + run_dir = _write_run(Path(tmp_dir), with_taxonomy=True, stale_judgments=True) + with self.assertLogs("assert_ai.runner", level="INFO") as captured: + _log_run_headline(run_dir) + + output = "\n".join(captured.output) + self.assertIn("Policy violation rate: prompt 100.0%", output) + self.assertIn("Overrefusal rate: prompt 100.0%", output) + self.assertNotIn("Impermissible behavior violated rate", output) + self.assertNotIn("Permissible behavior violated rate", output) + if __name__ == "__main__": unittest.main() From 0c4a401821fd2f2de1fe6f3edb7d37efb6a8677c Mon Sep 17 00:00:00 2001 From: Jake Present Date: Fri, 7 Aug 2026 16:46:12 -0400 Subject: [PATCH 4/4] fix(ci): cap broken Phoenix releases --- pyproject.toml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 655a78b7..254c7221 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,7 +44,10 @@ dependencies = [ [project.optional-dependencies] otel = [ - "arize-phoenix>=15.0.0", + # Phoenix 19.18.0-19.19.0 cannot import on supported Python 3.11: its + # dataclass uses MappingProxyType values as mutable defaults. Keep the last + # working release until upstream switches those fields to default_factory. + "arize-phoenix>=15.0.0,<19.18.0", "arize-phoenix-otel>=0.15.0", "openinference-instrumentation-langchain>=0.1.62", ]