diff --git a/assert_ai/cli.py b/assert_ai/cli.py index 3600a0f0..0bef09bd 100644 --- a/assert_ai/cli.py +++ b/assert_ai/cli.py @@ -151,6 +151,17 @@ def _fmt_percent(value: Optional[float]) -> str: _PERMISSIBILITY_SPLIT_RATE_KEYS = tuple(_DERIVED_PERMISSIBILITY_RATE_KEYS.values()) +_PERMISSIBILITY_METRIC_ALIASES = { + rate_key: metric + for metric, rate_key in _DERIVED_PERMISSIBILITY_RATE_KEYS.items() +} + +#: The half of the split the matrix leads with. ``policy_violation`` unions +#: permissible and impermissible behaviors, so ranking behaviors by it can order +#: them by the wrong thing entirely -- a behavior can carry a high union rate +#: made up almost wholly of mishandled *permissible* work while another with a +#: lower union rate is nearly all genuine impermissible failure. +_MATRIX_SPLIT_METRIC = _POLICY_VIOLATION_NOT_PERMISSIBLE def _has_permissibility_split(*metric_sets: Any) -> bool: @@ -696,6 +707,131 @@ def _load_run_summary(run_dir: Path) -> dict[str, Any] | None: } +def _run_behavior_name(run_dir: Path, suite_id: str) -> str: + config_path = run_dir / "config.yaml" + config = None + if config_path.exists(): + try: + config = yaml.safe_load(config_path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, yaml.YAMLError): + pass + behavior = config.get("behavior") if isinstance(config, dict) else None + if isinstance(behavior, dict) and isinstance(behavior.get("name"), str) and behavior.get("name"): + return behavior["name"] + + manifest = load_json(run_dir / "manifest.json") + manifest_candidates: list[Any] = [] + if isinstance(manifest, dict): + manifest_behavior = manifest.get("behavior") + manifest_candidates.append(manifest.get("behavior_name")) + if isinstance(manifest_behavior, dict): + manifest_candidates.append(manifest_behavior.get("name")) + manifest_config = manifest.get("config") + if isinstance(manifest_config, dict): + manifest_behavior = manifest_config.get("behavior") + if isinstance(manifest_behavior, dict): + manifest_candidates.append(manifest_behavior.get("name")) + for candidate in manifest_candidates: + if isinstance(candidate, str) and candidate: + return candidate + return suite_id + + +def _run_arm_label(run_id: str, suite_id: str) -> str: + prefix = f"{suite_id}-" + if run_id.startswith(prefix): + return run_id[len(prefix):] or run_id + return run_id + + +def _ordered_arm_labels(arms: Iterable[str]) -> list[str]: + known_order = {"baseline": 0, "prompted": 1, "acs": 2} + return sorted( + arms, + key=lambda arm: ( + 0 if arm.lower() in known_order else 1, + known_order.get(arm.lower(), 0), + arm.lower(), + ), + ) + + +def _pooled_rate(pairs: Iterable[tuple[int, int]]) -> float | None: + """Pool ``(flagged, scored)`` pairs instead of averaging rates.""" + flagged = scored = 0 + for hit, total in pairs: + flagged += hit + scored += total + return flagged / scored if scored else None + + +def _binary_counts(summary: Any) -> tuple[int, int] | None: + """Return ``(flagged, scored)`` for a binary dimension summary.""" + if not isinstance(summary, dict) or summary.get("kind") == "ordinal": + return None + counts = summary.get("counts") + if not isinstance(counts, dict): + return None + flagged = counts.get(1, counts.get("1", 0)) + passed = counts.get(0, counts.get("0", 0)) + if not isinstance(flagged, int) or not isinstance(passed, int): + return None + total = flagged + passed + return (flagged, total) if total else None + + +def _run_dimension_rate(run_summary: dict[str, Any], metric: str) -> float | None: + """Pool one metric across a run's prompt and scenario rows. + + The permissibility split is derived and stored in top-level bucket summaries, + not in ``dimensions``. All rates are recombined from counts when possible. + """ + prompt_metrics = run_summary.get("prompt_metrics") or {} + scenario_metrics = run_summary.get("scenario_metrics") or {} + halves = [metrics for metrics in (prompt_metrics, scenario_metrics) if isinstance(metrics, dict)] + + canonical_metric = _PERMISSIBILITY_METRIC_ALIASES.get(metric, metric) + split_key = _DERIVED_PERMISSIBILITY_RATE_KEYS.get(canonical_metric) + if split_key is not None: + bucket_key = _DERIVED_PERMISSIBILITY_SUMMARY_KEYS[canonical_metric] + pairs = [ + counts + for metrics in halves + if (counts := _binary_counts(metrics.get(bucket_key))) is not None + ] + if pairs: + return _pooled_rate(pairs) + for metrics in halves: + rate = metrics.get(split_key) + if isinstance(rate, (int, float)): + return float(rate) + return None + + pairs = [ + counts + for metrics in halves + if isinstance(metrics.get("dimensions"), dict) + and (counts := _binary_counts(metrics["dimensions"].get(metric))) is not None + ] + if pairs: + return _pooled_rate(pairs) + + prompt_rate = _dimension_rate(prompt_metrics, metric) + if prompt_rate is not None: + return prompt_rate + return _dimension_rate(scenario_metrics, metric) + + +def _parse_suite_run_arg(suite_run: str) -> tuple[str, str]: + parts = suite_run.strip("/").split("/") + if len(parts) == 1: + return parts[0], "run-1" + if len(parts) == 2: + return parts[0], parts[1] + _error(f"Invalid format: '{suite_run}'. Use SUITE/RUN (e.g., my-suite/run-1).") + raise AssertionError("unreachable") + + def _count_test_case_types(path: Path) -> tuple[int, int]: rows = load_jsonl(path) prompt_count = 0 @@ -1462,6 +1598,195 @@ def _run_within_suite_compare( console.print(delta_table) +@results.command("matrix", short_help="Compare behaviors across arms as a matrix") +@click.argument("suite_runs", nargs=-1) +@click.option( + "--suite", + "suites", + multiple=True, + shell_complete=_complete_suite, + help="Suite ID under artifacts/results. May be repeated; expands to all runs with scores.", +) +@click.option( + "--results-dir", + type=click.Path(path_type=Path), + default=DEFAULT_RESULTS_DIR, + show_default=True, + help="Results root to inspect.", +) +@click.option( + "--metric", + default=None, + shell_complete=_complete_metric, + help=( + "Judge dimension to compare. Defaults to the impermissible half of the " + "permissibility split when every run reports it, otherwise " + f"'{DEFAULT_COMPARE_METRIC}'." + ), +) +@click.option("--json", "as_json", is_flag=True, help="Emit machine-readable JSON instead of tables.") +@click.option("--no-color", is_flag=True, help="Disable colored terminal output.") +def results_matrix( + suite_runs: tuple[str, ...], + suites: tuple[str, ...], + results_dir: Path, + metric: str | None, + as_json: bool, + no_color: bool, +): + """Render a behavior x arm pivot over multiple runs.""" + if not suites and len(suite_runs) < 2: + _error("Provide at least two SUITE/RUN arguments to compare, or use --suite SUITE.") + + results_root = _resolve_results_dir(results_dir) + resolved_suite_runs: list[tuple[str, str]] = [] + seen_suite_runs: set[tuple[str, str]] = set() + for suite_run in suite_runs: + parsed = _parse_suite_run_arg(suite_run) + if parsed not in seen_suite_runs: + resolved_suite_runs.append(parsed) + seen_suite_runs.add(parsed) + for suite in suites: + suite_dir = results_root / suite + if not suite_dir.exists(): + _error(f"Suite not found: {suite}") + for child in sorted(suite_dir.iterdir()): + if child.is_dir() and (child / "scores.jsonl").exists(): + parsed = (suite, child.name) + if parsed not in seen_suite_runs: + resolved_suite_runs.append(parsed) + seen_suite_runs.add(parsed) + + if len(resolved_suite_runs) < 2: + _error("Provide at least two runs with scores to compare.") + + run_summaries: list[dict[str, Any]] = [] + behaviors: list[str] = [] + arms: list[str] = [] + cells: dict[str, dict[str, float | None]] = {} + cell_sources: dict[tuple[str, str], str] = {} + seen_behaviors: set[str] = set() + seen_arms: set[str] = set() + + loaded: list[tuple[str, str, dict[str, Any]]] = [] + for suite_id, run_id in resolved_suite_runs: + run_dir = results_root / suite_id / run_id + if not run_dir.exists(): + _error(f"Not found: {suite_id}/{run_id}") + run_summary = _load_run_summary(run_dir) + if run_summary is None: + _error(f"No scores in {suite_id}/{run_id}") + matrix_summary = { + "prompt_metrics": run_summary.get("prompt_metrics"), + "scenario_metrics": run_summary.get("scenario_metrics"), + } + run_summaries.append(matrix_summary) + + behavior = _run_behavior_name(run_dir, suite_id) + arm = _run_arm_label(run_id, suite_id) + source = f"{suite_id}/{run_id}" + cell_key = (behavior, arm) + if previous_source := cell_sources.get(cell_key): + _error( + f"Runs '{previous_source}' and '{source}' both resolve to " + f"behavior '{behavior}' and arm '{arm}'. Use distinct run IDs " + "or select only one run for that cell." + ) + cell_sources[cell_key] = source + if behavior not in seen_behaviors: + behaviors.append(behavior) + seen_behaviors.add(behavior) + if arm not in seen_arms: + arms.append(arm) + seen_arms.add(arm) + loaded.append((behavior, arm, matrix_summary)) + + split_by_run = [ + any( + _run_dimension_rate(run_summary, split_metric) is not None + for split_metric in _DERIVED_PERMISSIBILITY_RATE_KEYS + ) + for run_summary in run_summaries + ] + + # Requiring every run to have usable split data keeps the matrix from mixing + # an impermissible-only rate with the legacy union. Split-shaped summaries + # whose taxonomy no longer matches their judgments do not count as usable. + if metric is None: + metric = ( + _MATRIX_SPLIT_METRIC + if split_by_run and all(split_by_run) + else DEFAULT_COMPARE_METRIC + ) + metric = _PERMISSIBILITY_METRIC_ALIASES.get(metric, metric) + + available_metrics: set[str] = set() + for run_summary in run_summaries: + prompt_metrics = run_summary.get("prompt_metrics") + scenario_metrics = run_summary.get("scenario_metrics") + for metrics in (prompt_metrics, scenario_metrics): + dimensions = metrics.get("dimensions") if isinstance(metrics, dict) else None + if isinstance(dimensions, dict): + available_metrics.update(dimensions) + if any(split_by_run): + available_metrics.update(_DERIVED_PERMISSIBILITY_RATE_KEYS) + if metric not in available_metrics: + _error( + f"Metric '{metric}' was not found in the compared judgments. " + f"Available: {sorted(available_metrics)}" + ) + + for behavior, arm, run_summary in loaded: + cells.setdefault(behavior, {})[arm] = _run_dimension_rate(run_summary, metric) + + _reject_ordinal_compare(run_summaries, metric) + arms = _ordered_arm_labels(arms) + + if as_json: + _echo_json({ + "metric": metric, + "behaviors": behaviors, + "arms": arms, + "cells": { + behavior: { + arm: cells.get(behavior, {}).get(arm) + for arm in arms + } + for behavior in behaviors + }, + }) + return + + console = _console(no_color=no_color) + table = Table( + title=f"Behavior × arm matrix ({_metric_label(metric)})", + box=None, + show_header=True, + show_edge=False, + pad_edge=False, + ) + table.add_column("Behavior", style="cyan", no_wrap=True) + for arm in arms: + table.add_column(arm, style="white", no_wrap=True) + for behavior in behaviors: + row = [behavior] + for arm in arms: + row.append(_fmt_percent(cells.get(behavior, {}).get(arm))) + table.add_row(*row) + console.print(table) + + if metric in _DERIVED_PERMISSIBILITY_RATE_KEYS: + # Each half is scored only over the rows where a behavior in that bucket + # was relevant, so the two halves have different denominators from each + # other and from `policy_violation`. Say so, or a reader will try to add + # them and find they do not reconcile to the union. + console.print( + "[dim]Rate is over rows where a behavior in this bucket was relevant, " + "not all scored rows. The two halves of the split therefore have " + "different denominators and do not sum to policy_violation.[/dim]" + ) + + @results.command("compare-suites", short_help="Compare runs across different suites (e.g., approach A vs B vs C)") @click.argument("suite_runs", nargs=-1) @click.option( diff --git a/tests/test_cli_results_matrix.py b/tests/test_cli_results_matrix.py new file mode 100644 index 00000000..462a6ae5 --- /dev/null +++ b/tests/test_cli_results_matrix.py @@ -0,0 +1,744 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +import json +from pathlib import Path +from typing import Any + +import pytest +from click.testing import CliRunner + +from assert_ai.cli import cli + + +def _write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None: + path.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") + + +def _score_row(policy_violation: bool) -> dict[str, Any]: + return { + "judge_status": "ok", + "target": "test-target", + "judge_model": "test-judge", + "verdict": { + "dimensions": { + "policy_violation": policy_violation, + "overrefusal": False, + }, + "node_judgments": [], + }, + } + + +def _make_run( + results_root: Path, + suite_id: str, + run_id: str, + behavior_name: str | None, + policy_violations: list[bool], +) -> None: + run_dir = results_root / suite_id / run_id + run_dir.mkdir(parents=True) + (run_dir / "manifest.json").write_text( + json.dumps({"status": "completed", "stages": {"judge": "completed"}}), + encoding="utf-8", + ) + (run_dir / "config.yaml").write_text( + "\n".join([ + "behavior:", + f" name: {behavior_name}", + ]) if behavior_name is not None else "behavior: {}\n", + encoding="utf-8", + ) + _write_jsonl(run_dir / "scores.jsonl", [_score_row(value) for value in policy_violations]) + + +def test_results_matrix_json_renders_two_behaviors_by_two_arms(tmp_path: Path) -> None: + results_root = tmp_path / "results" + _make_run(results_root, "behavior-a", "behavior-a-baseline", "behavior_a", [True, False]) + _make_run(results_root, "behavior-a", "behavior-a-prompted", "behavior_a", [False, False]) + _make_run(results_root, "behavior-b", "behavior-b-baseline", "behavior_b", [True, True]) + _make_run(results_root, "behavior-b", "behavior-b-prompted", "behavior_b", [False, True]) + + result = CliRunner().invoke( + cli, + [ + "results", + "matrix", + "behavior-a/behavior-a-baseline", + "behavior-a/behavior-a-prompted", + "behavior-b/behavior-b-baseline", + "behavior-b/behavior-b-prompted", + "--results-dir", + str(results_root), + "--json", + ], + ) + + assert result.exit_code == 0, result.output + assert json.loads(result.output) == { + "metric": "policy_violation", + "behaviors": ["behavior_a", "behavior_b"], + "arms": ["baseline", "prompted"], + "cells": { + "behavior_a": {"baseline": 0.5, "prompted": 0.0}, + "behavior_b": {"baseline": 1.0, "prompted": 0.5}, + }, + } + + +def test_results_matrix_missing_cell_renders_null_and_dash(tmp_path: Path) -> None: + results_root = tmp_path / "results" + _make_run(results_root, "behavior-a", "behavior-a-baseline", "behavior_a", [True]) + _make_run(results_root, "behavior-a", "behavior-a-prompted", "behavior_a", [False]) + _make_run(results_root, "behavior-b", "behavior-b-baseline", "behavior_b", [False]) + + args = [ + "results", + "matrix", + "behavior-a/behavior-a-baseline", + "behavior-a/behavior-a-prompted", + "behavior-b/behavior-b-baseline", + "--results-dir", + str(results_root), + ] + runner = CliRunner() + + json_result = runner.invoke(cli, [*args, "--json"]) + assert json_result.exit_code == 0, json_result.output + payload = json.loads(json_result.output) + assert payload["cells"]["behavior_b"]["prompted"] is None + + text_result = runner.invoke(cli, [*args, "--no-color"]) + assert text_result.exit_code == 0, text_result.output + behavior_b_row = next(line for line in text_result.output.splitlines() if "behavior_b" in line) + assert behavior_b_row.rstrip().endswith("-") + + +def test_results_matrix_suite_auto_expand_matches_explicit_args(tmp_path: Path) -> None: + results_root = tmp_path / "results" + _make_run(results_root, "suite-a", "suite-a-baseline", "behavior_a", [True, False]) + _make_run(results_root, "suite-a", "suite-a-prompted", "behavior_a", [False, False]) + + runner = CliRunner() + explicit = runner.invoke( + cli, + [ + "results", + "matrix", + "suite-a/suite-a-baseline", + "suite-a/suite-a-prompted", + "--results-dir", + str(results_root), + "--json", + ], + ) + expanded = runner.invoke( + cli, + [ + "results", + "matrix", + "--suite", + "suite-a", + "--results-dir", + str(results_root), + "--json", + ], + ) + + assert explicit.exit_code == 0, explicit.output + assert expanded.exit_code == 0, expanded.output + assert json.loads(explicit.output) == json.loads(expanded.output) + + +def test_results_matrix_repeated_suite_expands_multiple_suites_with_known_arm_order(tmp_path: Path) -> None: + results_root = tmp_path / "results" + _make_run(results_root, "suite-a", "suite-a-acs", "behavior_a", [False]) + _make_run(results_root, "suite-a", "suite-a-baseline", "behavior_a", [True]) + _make_run(results_root, "suite-b", "suite-b-prompted", "behavior_b", [True, False]) + _make_run(results_root, "suite-b", "suite-b-acs", "behavior_b", [False, False]) + _make_run(results_root, "suite-b", "suite-b-baseline", "behavior_b", [True, True]) + + result = CliRunner().invoke( + cli, + [ + "results", + "matrix", + "--suite", + "suite-a", + "--suite", + "suite-b", + "--results-dir", + str(results_root), + "--json", + ], + ) + + assert result.exit_code == 0, result.output + payload = json.loads(result.output) + assert payload["behaviors"] == ["behavior_a", "behavior_b"] + assert payload["arms"] == ["baseline", "prompted", "acs"] + assert payload["cells"] == { + "behavior_a": {"baseline": 1.0, "prompted": None, "acs": 0.0}, + "behavior_b": {"baseline": 1.0, "prompted": 0.5, "acs": 0.0}, + } + + +def test_results_matrix_behavior_name_falls_back_to_suite_id(tmp_path: Path) -> None: + results_root = tmp_path / "results" + _make_run(results_root, "fallback-suite", "fallback-suite-baseline", None, [True]) + _make_run(results_root, "fallback-suite", "fallback-suite-prompted", None, [False]) + + result = CliRunner().invoke( + cli, + [ + "results", + "matrix", + "fallback-suite/fallback-suite-baseline", + "fallback-suite/fallback-suite-prompted", + "--results-dir", + str(results_root), + "--json", + ], + ) + + assert result.exit_code == 0, result.output + assert json.loads(result.output)["behaviors"] == ["fallback-suite"] + + +def test_results_matrix_preserves_full_non_prefixed_run_ids(tmp_path: Path) -> None: + results_root = tmp_path / "results" + _make_run( + results_root, + "suite-a", + "variant-c-baseline-prompt", + "behavior_a", + [True], + ) + _make_run( + results_root, + "suite-b", + "baseline-weak-prompt", + "behavior_b", + [False], + ) + + result = CliRunner().invoke( + cli, + [ + "results", + "matrix", + "suite-a/variant-c-baseline-prompt", + "suite-b/baseline-weak-prompt", + "--results-dir", + str(results_root), + "--json", + ], + ) + + assert result.exit_code == 0, result.output + arms = json.loads(result.output)["arms"] + assert arms == ["baseline-weak-prompt", "variant-c-baseline-prompt"] + assert "prompt" not in arms + + +def test_results_matrix_rejects_duplicate_behavior_arm_cells(tmp_path: Path) -> None: + results_root = tmp_path / "results" + _make_run(results_root, "suite-a", "suite-a-baseline", "shared_behavior", [True]) + _make_run(results_root, "suite-b", "suite-b-baseline", "shared_behavior", [False]) + + result = CliRunner().invoke( + cli, + [ + "results", + "matrix", + "suite-a/suite-a-baseline", + "suite-b/suite-b-baseline", + "--results-dir", + str(results_root), + "--json", + ], + ) + + assert result.exit_code == 1 + assert "suite-a/suite-a-baseline" in result.output + assert "suite-b/suite-b-baseline" in result.output + assert "behavior 'shared_behavior' and arm 'baseline'" in result.output + + +def test_results_matrix_rejects_unknown_metric(tmp_path: Path) -> None: + results_root = tmp_path / "results" + _make_run(results_root, "beh", "beh-baseline", "beh", [True]) + _make_run(results_root, "beh", "beh-prompted", "beh", [False]) + + result = CliRunner().invoke( + cli, + [ + "results", + "matrix", + "--suite", + "beh", + "--results-dir", + str(results_root), + "--metric", + "policy_violaton", + "--json", + ], + ) + + assert result.exit_code == 1 + assert "Metric 'policy_violaton' was not found" in result.output + assert "policy_violation" in result.output + + +def test_results_matrix_malformed_config_falls_back_to_manifest(tmp_path: Path) -> None: + results_root = tmp_path / "results" + for run_id, flagged in (("suite-a-baseline", True), ("suite-a-prompted", False)): + _make_run(results_root, "suite-a", run_id, None, [flagged]) + run_dir = results_root / "suite-a" / run_id + (run_dir / "config.yaml").write_text("behavior: [\n", encoding="utf-8") + (run_dir / "manifest.json").write_text( + json.dumps({ + "status": "completed", + "stages": {"judge": "completed"}, + "behavior_name": "manifest_behavior", + }), + encoding="utf-8", + ) + + result = CliRunner().invoke( + cli, + [ + "results", + "matrix", + "--suite", + "suite-a", + "--results-dir", + str(results_root), + "--json", + ], + ) + + assert result.exit_code == 0, result.output + assert json.loads(result.output)["behaviors"] == ["manifest_behavior"] + + +def test_results_matrix_deduplicates_explicit_and_suite_expansion(tmp_path: Path) -> None: + results_root = tmp_path / "results" + _make_run(results_root, "suite-a", "suite-a-baseline", "behavior_a", [True]) + _make_run(results_root, "suite-a", "suite-a-prompted", "behavior_a", [False]) + + result = CliRunner().invoke( + cli, + [ + "results", + "matrix", + "suite-a/suite-a-baseline", + "--suite", + "suite-a", + "--results-dir", + str(results_root), + "--json", + ], + ) + + assert result.exit_code == 0, result.output + assert json.loads(result.output)["cells"] == { + "behavior_a": {"baseline": 1.0, "prompted": 0.0} + } + + +def test_results_matrix_missing_suite_is_a_cli_error(tmp_path: Path) -> None: + result = CliRunner().invoke( + cli, + [ + "results", + "matrix", + "--suite", + "missing-suite", + "--results-dir", + str(tmp_path / "results"), + ], + ) + + assert result.exit_code == 1 + assert "Suite not found: missing-suite" in result.output + + +# --- permissibility split ------------------------------------------------- +# +# `policy_violation` unions permissible and impermissible behaviors, so ranking +# behaviors by it can order them by the wrong thing entirely. The matrix +# therefore leads with the impermissible half when every run reports it. +# +# The split is derived from node judgments plus the taxonomy and is stored as a +# top-level rate, not under `dimensions`. Asking for it used to render an empty +# cell under a correct-looking "Impermissible behavior violated" heading, which +# reads as "no violations" rather than "not wired up". + + +def _split_score_row(*, impermissible: bool, permissible: bool) -> dict[str, Any]: + """A row whose node judgments cover both buckets.""" + return { + "judge_status": "ok", + "target": "test-target", + "judge_model": "test-judge", + "verdict": { + "dimensions": { + "policy_violation": impermissible or permissible, + "overrefusal": False, + }, + "node_judgments": [ + {"node_index": 0, "node_name": "must never", "relevant": True, "violated": impermissible}, + {"node_index": 1, "node_name": "allowed", "relevant": True, "violated": permissible}, + ], + }, + } + + +def _make_split_run( + results_root: Path, + suite_id: str, + run_id: str, + behavior_name: str, + rows: list[tuple[bool, bool]], + *, with_taxonomy: bool = True, +) -> None: + run_dir = results_root / suite_id / run_id + run_dir.mkdir(parents=True) + (run_dir / "manifest.json").write_text( + json.dumps({"status": "completed", "stages": {"judge": "completed"}}), encoding="utf-8" + ) + (run_dir / "config.yaml").write_text(f"behavior:\n name: {behavior_name}\n", encoding="utf-8") + if with_taxonomy: + (run_dir.parent / "taxonomy.json").write_text( + json.dumps({ + "behavior_categories": [ + {"name": "must never", "permissible": False}, + {"name": "allowed", "permissible": True}, + ] + }), + encoding="utf-8", + ) + _write_jsonl( + run_dir / "scores.jsonl", + [_split_score_row(impermissible=i, permissible=p) for i, p in rows], + ) + + +def test_matrix_defaults_to_the_impermissible_half_when_every_run_has_the_split(tmp_path: Path) -> None: + results_root = tmp_path / "results" + # 1 of 4 impermissible, 3 of 4 permissible: the union would rank this high + # for the wrong reason. + _make_split_run(results_root, "beh", "beh-baseline", "beh", + [(True, True), (False, True), (False, True), (False, False)]) + _make_split_run(results_root, "beh", "beh-governed", "beh", + [(False, False), (False, False), (False, True), (False, False)]) + + result = CliRunner().invoke( + cli, ["results", "matrix", "--suite", "beh", "--results-dir", str(results_root), "--json"] + ) + + assert result.exit_code == 0, result.output + payload = json.loads(result.output) + assert payload["metric"] == "policy_violation_not_permissible" + assert payload["cells"]["beh"]["baseline"] == 0.25 + assert payload["cells"]["beh"]["governed"] == 0.0 + + +def test_matrix_split_cells_are_populated_not_dashes(tmp_path: Path) -> None: + """The regression: the metric resolved and labelled, but every cell was None.""" + results_root = tmp_path / "results" + _make_split_run(results_root, "beh", "beh-baseline", "beh", [(True, False), (False, True)]) + _make_split_run(results_root, "beh", "beh-governed", "beh", [(False, False), (False, True)]) + + for metric in ("policy_violation_not_permissible", "policy_violation_permissible"): + result = CliRunner().invoke( + cli, + ["results", "matrix", "--suite", "beh", "--results-dir", str(results_root), + "--metric", metric, "--json"], + ) + assert result.exit_code == 0, result.output + cells = json.loads(result.output)["cells"]["beh"] + assert all(value is not None for value in cells.values()), (metric, cells) + + +def test_matrix_accepts_the_artifact_key_spelling_of_the_split(tmp_path: Path) -> None: + results_root = tmp_path / "results" + _make_split_run(results_root, "beh", "beh-baseline", "beh", [(True, False), (False, False)]) + _make_split_run(results_root, "beh", "beh-governed", "beh", [(False, False), (False, False)]) + + result = CliRunner().invoke( + cli, + ["results", "matrix", "--suite", "beh", "--results-dir", str(results_root), + "--metric", "not_permissible_policy_violation_rate", "--json"], + ) + + assert result.exit_code == 0, result.output + payload = json.loads(result.output) + assert payload["metric"] == "policy_violation_not_permissible" + assert payload["cells"]["beh"]["baseline"] == 0.5 + + +def test_matrix_falls_back_to_policy_violation_without_a_taxonomy(tmp_path: Path) -> None: + """Quality suites repurpose policy_violation and have no taxonomy; they must + keep reporting the union rather than a table of blanks.""" + results_root = tmp_path / "results" + _make_split_run(results_root, "beh", "beh-baseline", "beh", + [(True, False), (False, False)], with_taxonomy=False) + _make_split_run(results_root, "beh", "beh-governed", "beh", + [(False, False), (False, False)], with_taxonomy=False) + + result = CliRunner().invoke( + cli, ["results", "matrix", "--suite", "beh", "--results-dir", str(results_root), "--json"] + ) + + assert result.exit_code == 0, result.output + payload = json.loads(result.output) + assert payload["metric"] == "policy_violation" + assert payload["cells"]["beh"]["baseline"] == 0.5 + + +def test_matrix_falls_back_when_taxonomy_no_longer_matches_judgments(tmp_path: Path) -> None: + results_root = tmp_path / "results" + for run_id, violated in (("beh-baseline", True), ("beh-governed", False)): + _make_split_run( + results_root, + "beh", + run_id, + "beh", + [(violated, False)], + ) + row = _split_score_row(impermissible=violated, permissible=False) + for index, node in enumerate(row["verdict"]["node_judgments"]): + node["node_index"] = index + 10 + node["node_name"] = f"stale-{index}" + _write_jsonl(results_root / "beh" / run_id / "scores.jsonl", [row]) + + result = CliRunner().invoke( + cli, + [ + "results", + "matrix", + "--suite", + "beh", + "--results-dir", + str(results_root), + "--json", + ], + ) + + assert result.exit_code == 0, result.output + payload = json.loads(result.output) + assert payload["metric"] == "policy_violation" + assert payload["cells"]["beh"]["baseline"] == 1.0 + + +def test_matrix_does_not_mix_halves_when_only_some_runs_have_the_split(tmp_path: Path) -> None: + """One run contributing an impermissible-only rate while another contributes + the union would put non-comparable numbers in one table.""" + results_root = tmp_path / "results" + _make_split_run(results_root, "with-tax", "with-tax-baseline", "beh_a", [(True, False)]) + _make_split_run(results_root, "no-tax", "no-tax-baseline", "beh_b", + [(True, False)], with_taxonomy=False) + + result = CliRunner().invoke( + cli, + ["results", "matrix", "with-tax/with-tax-baseline", "no-tax/no-tax-baseline", + "--results-dir", str(results_root), "--json"], + ) + + assert result.exit_code == 0, result.output + assert json.loads(result.output)["metric"] == "policy_violation" + + +def test_matrix_warns_that_split_halves_have_different_denominators(tmp_path: Path) -> None: + """Each half is scored only over rows where its bucket was relevant, so the + halves do not sum to policy_violation.""" + results_root = tmp_path / "results" + _make_split_run(results_root, "beh", "beh-baseline", "beh", [(True, False), (False, True)]) + _make_split_run(results_root, "beh", "beh-governed", "beh", [(False, False), (False, True)]) + + result = CliRunner().invoke( + cli, ["results", "matrix", "--suite", "beh", "--results-dir", str(results_root)] + ) + + assert result.exit_code == 0, result.output + assert "denominator" in result.output.lower() + + +# --- prompt + scenario pooling ------------------------------------------- +# +# A run's prompt and scenario rows are separate metric sets. Reporting whichever +# was present first silently drops the other, and they are not interchangeable: +# on the career-health CV-injection baseline the prompt rows score 64% and the +# scenario rows 88%, so a prompt-only cell understated the run by 12 points with +# nothing on screen to say half the data was excluded. + + +def _mixed_run( + results_root: Path, + suite_id: str, + run_id: str, + behavior_name: str, + prompt_flags: list[bool], + scenario_flags: list[bool], +) -> None: + run_dir = results_root / suite_id / run_id + run_dir.mkdir(parents=True) + (run_dir / "manifest.json").write_text( + json.dumps({"status": "completed", "stages": {"judge": "completed"}}), encoding="utf-8" + ) + (run_dir / "config.yaml").write_text(f"behavior:\n name: {behavior_name}\n", encoding="utf-8") + rows = [_score_row(flag) for flag in prompt_flags] + for flag in scenario_flags: + row = _score_row(flag) + row["tester_model"] = "test-tester" + rows.append(row) + _write_jsonl(run_dir / "scores.jsonl", rows) + + +def test_matrix_pools_prompt_and_scenario_rows(tmp_path: Path) -> None: + results_root = tmp_path / "results" + # 1/4 prompt + 3/4 scenario = 4/8 pooled. Prompt-only would report 0.25. + _mixed_run(results_root, "beh", "beh-baseline", "beh", + [True, False, False, False], [True, True, True, False]) + _mixed_run(results_root, "beh", "beh-governed", "beh", + [False, False, False, False], [False, False, False, False]) + + result = CliRunner().invoke( + cli, + ["results", "matrix", "--suite", "beh", "--results-dir", str(results_root), + "--metric", "policy_violation", "--json"], + ) + + assert result.exit_code == 0, result.output + assert json.loads(result.output)["cells"]["beh"]["baseline"] == 0.5 + + +def test_matrix_pools_from_counts_not_by_averaging_rates(tmp_path: Path) -> None: + """Unequal halves: the mean of the two rates is not the rate of the whole.""" + results_root = tmp_path / "results" + # 1/1 prompt (100%) + 1/9 scenario (11.1%) = 2/10 pooled (20%). + # Averaging the two rates would give 55.6%. + _mixed_run(results_root, "beh", "beh-baseline", "beh", + [True], [True] + [False] * 8) + _mixed_run(results_root, "beh", "beh-governed", "beh", [False], [False]) + + result = CliRunner().invoke( + cli, + ["results", "matrix", "--suite", "beh", "--results-dir", str(results_root), + "--metric", "policy_violation", "--json"], + ) + + assert result.exit_code == 0, result.output + baseline = json.loads(result.output)["cells"]["beh"]["baseline"] + assert baseline == pytest.approx(0.2), baseline + + +def test_matrix_pools_the_permissibility_split_across_both_halves(tmp_path: Path) -> None: + results_root = tmp_path / "results" + run_dir = results_root / "beh" / "beh-baseline" + run_dir.mkdir(parents=True) + (run_dir / "manifest.json").write_text( + json.dumps({"status": "completed", "stages": {"judge": "completed"}}), encoding="utf-8" + ) + (run_dir / "config.yaml").write_text("behavior:\n name: beh\n", encoding="utf-8") + (results_root / "beh" / "taxonomy.json").write_text( + json.dumps({"behavior_categories": [ + {"name": "must never", "permissible": False}, + {"name": "allowed", "permissible": True}, + ]}), + encoding="utf-8", + ) + # 1/2 impermissible in prompt, 1/2 in scenario -> 2/4 pooled. + rows = [ + _split_score_row(impermissible=True, permissible=False), + _split_score_row(impermissible=False, permissible=False), + ] + for flag in (True, False): + row = _split_score_row(impermissible=flag, permissible=False) + row["tester_model"] = "test-tester" + rows.append(row) + _write_jsonl(run_dir / "scores.jsonl", rows) + + _make_split_run(results_root, "beh2", "beh2-baseline", "beh2", [(False, False)]) + + result = CliRunner().invoke( + cli, + ["results", "matrix", "beh/beh-baseline", "beh2/beh2-baseline", + "--results-dir", str(results_root), "--json"], + ) + + assert result.exit_code == 0, result.output + payload = json.loads(result.output) + assert payload["metric"] == "policy_violation_not_permissible" + assert payload["cells"]["beh"]["baseline"] == 0.5 + + +def test_matrix_matches_cv_injection_prompt_scenario_totals(tmp_path: Path) -> None: + results_root = tmp_path / "results" + run_dir = results_root / "cv-injection" / "cv-injection-baseline" + run_dir.mkdir(parents=True) + (run_dir / "manifest.json").write_text( + json.dumps({"status": "completed", "stages": {"judge": "completed"}}), + encoding="utf-8", + ) + (run_dir / "config.yaml").write_text( + "behavior:\n name: cv_injection\n", + encoding="utf-8", + ) + (run_dir.parent / "taxonomy.json").write_text( + json.dumps({ + "behavior_categories": [ + {"name": "must never", "permissible": False}, + {"name": "allowed", "permissible": True}, + ] + }), + encoding="utf-8", + ) + + rows: list[dict[str, Any]] = [] + for index in range(25): + row = _split_score_row( + impermissible=index < 4, + permissible=index < 16, + ) + if index >= 21: + row["verdict"]["node_judgments"][0]["relevant"] = False + rows.append(row) + for index in range(25): + row = _split_score_row( + impermissible=index < 18, + permissible=index < 22, + ) + row["tester_model"] = "test-tester" + rows.append(row) + _write_jsonl(run_dir / "scores.jsonl", rows) + _make_split_run( + results_root, + "control", + "control-baseline", + "control", + [(False, False)], + ) + + base_args = [ + "results", + "matrix", + "cv-injection/cv-injection-baseline", + "control/control-baseline", + "--results-dir", + str(results_root), + "--json", + ] + runner = CliRunner() + union_result = runner.invoke(cli, [*base_args, "--metric", "policy_violation"]) + split_result = runner.invoke(cli, base_args) + + assert union_result.exit_code == 0, union_result.output + assert split_result.exit_code == 0, split_result.output + union = json.loads(union_result.output) + split = json.loads(split_result.output) + assert union["cells"]["cv_injection"]["baseline"] == pytest.approx(38 / 50) + assert split["cells"]["cv_injection"]["baseline"] == pytest.approx(22 / 46)