diff --git a/apps/presentation/dashboard/smoke/benchmark-study-contract-smoke.ts b/apps/presentation/dashboard/smoke/benchmark-study-contract-smoke.ts index b34dd59e3f..b3d86812cf 100644 --- a/apps/presentation/dashboard/smoke/benchmark-study-contract-smoke.ts +++ b/apps/presentation/dashboard/smoke/benchmark-study-contract-smoke.ts @@ -26,6 +26,27 @@ if (packet.contrasts.goal_hint?.binary_metric_transitions.reward?.same !== 2) th if (packet.cases[1]?.largest_eligible_primary_contrast?.metric_deltas.feature?.delta !== 2) throw new Error("case-level eligible contrast missing"); if (packet.cases[0]?.arms.some((cell) => cell.score_countable && !cell.metrics.preservation)) throw new Error("countable cell guardrail missing"); +const ratioFixture = structuredClone(fixture); +const ratioMetric = ratioFixture.cases[1]!.largest_eligible_primary_contrast!.metric_deltas.feature; +Object.assign(ratioMetric, { delta: -41, delta_rate: 0.4, direction: "improved" }); +const ratioPacket = parseBenchmarkStudyDashboard(ratioFixture); +if (ratioPacket.cases[1]?.largest_eligible_primary_contrast?.metric_deltas.feature?.delta_rate !== 0.4) throw new Error("ratio comparison scale lost"); +const unavailable = structuredClone(ratioPacket); +const unavailableMetrics = unavailable.cases[1]!.largest_eligible_primary_contrast!.metric_deltas; +unavailableMetrics.preservation = { + baseline_value: 1, candidate_value: 2, comparison_unavailable_reason: "unit_mismatch", +}; +unavailable.runs[0]!.metrics.feature!.total = 0; +parseBenchmarkStudyDashboard(unavailable); +unavailableMetrics.preservation.delta = 1; +if (parseUnexpectedComparison(unavailable)) throw new Error("unavailable comparison exposed a numeric conclusion"); +Object.assign(ratioMetric, { delta_rate: Number.POSITIVE_INFINITY }); +if (parseUnexpectedComparison(ratioFixture)) throw new Error("non-finite ratio accepted"); + +function parseUnexpectedComparison(value: unknown) { + try { parseBenchmarkStudyDashboard(value); return true; } catch { return false; } +} + const resolved = resolveBenchmarkStudyDashboardUrl("/study.json", "http://127.0.0.1:5173/benchmarks/study"); if (resolved !== "http://127.0.0.1:5173/study.json") throw new Error("local readback URL drifted"); const packaged = resolveBenchmarkStudyDashboardUrl("/chat/benchmark-study.example.json", "http://127.0.0.1:5173/chat/benchmarks/study"); diff --git a/apps/presentation/dashboard/src/data/benchmark-study.ts b/apps/presentation/dashboard/src/data/benchmark-study.ts index ffda1dc700..ef085b20f7 100644 --- a/apps/presentation/dashboard/src/data/benchmark-study.ts +++ b/apps/presentation/dashboard/src/data/benchmark-study.ts @@ -2,7 +2,7 @@ import { z } from "zod"; const metricValueSchema = z.object({ value: z.number().finite(), - total: z.number().finite().positive().optional(), + total: z.number().finite().nonnegative().optional(), unit: z.string().optional(), higher_is_better: z.boolean(), }).passthrough(); @@ -97,9 +97,13 @@ const armSchema = z.object({ const metricDeltaSchema = z.object({ baseline_value: z.number().finite(), candidate_value: z.number().finite(), - delta: z.number().finite(), + delta: z.number().finite().optional(), + delta_rate: z.number().finite().optional(), + comparison_unavailable_reason: z.string().optional(), direction: z.enum(["improved", "flat", "regressed"]).optional(), -}).passthrough(); +}).passthrough().refine((metric) => metric.comparison_unavailable_reason + ? metric.delta === undefined && metric.delta_rate === undefined && metric.direction === undefined + : metric.delta !== undefined, "Metric comparison must contain a delta or an unavailable reason"); const matchedComparisonSchema = z.object({ comparison_id: z.string(), diff --git a/apps/presentation/dashboard/src/views/benchmark-study-page.tsx b/apps/presentation/dashboard/src/views/benchmark-study-page.tsx index a4928bee29..91eba5ef1d 100644 --- a/apps/presentation/dashboard/src/views/benchmark-study-page.tsx +++ b/apps/presentation/dashboard/src/views/benchmark-study-page.tsx @@ -67,11 +67,13 @@ function metricAggregate(arm: BenchmarkStudyArm, metricName: string) { function largestContrastLabel(item: BenchmarkStudyCase, primaryMetric: string) { const contrast = item.largest_eligible_primary_contrast; const metric = contrast?.metric_deltas[primaryMetric]; - if (!contrast || !metric) return null; - const prefix = metric.delta > 0 ? "+" : ""; + if (!contrast || !metric || metric.comparison_unavailable_reason) return null; + const value = metric.delta_rate == null ? metric.delta : metric.delta_rate * 100; + if (value == null) return null; + const prefix = value > 0 ? "+" : ""; return { direction: metric.direction, - text: `${contrast.candidate_arm_id}: ${prefix}${compactNumber(metric.delta)}`, + text: `${contrast.candidate_arm_id}: ${prefix}${compactNumber(value)}${metric.delta_rate == null ? "" : " pp"}`, }; } diff --git a/docs/assets/benchmark-study/metric-comparison-after.png b/docs/assets/benchmark-study/metric-comparison-after.png new file mode 100644 index 0000000000..69a9712baf Binary files /dev/null and b/docs/assets/benchmark-study/metric-comparison-after.png differ diff --git a/docs/assets/benchmark-study/metric-comparison-before.png b/docs/assets/benchmark-study/metric-comparison-before.png new file mode 100644 index 0000000000..d516e93bb1 Binary files /dev/null and b/docs/assets/benchmark-study/metric-comparison-before.png differ diff --git a/examples/dashboard-benchmark-study-browser-smoke.mjs b/examples/dashboard-benchmark-study-browser-smoke.mjs index 00f7a07b30..4218a610f0 100644 --- a/examples/dashboard-benchmark-study-browser-smoke.mjs +++ b/examples/dashboard-benchmark-study-browser-smoke.mjs @@ -142,6 +142,28 @@ async function assertCaseScopedRunSelection(page, url) { await page.getByRole("complementary").getByText("case-state-transition ยท goal_plain", { exact: true }).waitFor(); } +async function assertRatioComparisonReadback(page, url) { + // Synthetic projection isolates formatting from Python's independently tested selection. + const packet = JSON.parse(readFileSync(resolve(dashboardDir, "public/benchmark-study.example.json"), "utf8")); + const metric = packet.cases[1].largest_eligible_primary_contrast.metric_deltas.feature; + await page.route("**/ratio-comparison.json", (route) => route.fulfill({ json: packet })); + const target = new URL(url); + target.search = new URLSearchParams({ dashboardUrl: "/ratio-comparison.json", view: "cases" }).toString(); + for (const [delta, rate, direction, label, className] of [ + [-41, 0.4, "improved", "goal_hint: +40 pp", "benchmark-positive"], + [10, -0.2, "regressed", "goal_hint: -20 pp", "benchmark-negative"], + [50, 0, "flat", "goal_hint: 0 pp", ""], + ]) { + Object.assign(metric, { delta, delta_rate: rate, direction }); + await page.goto(target.toString(), { waitUntil: "networkidle" }); + const cell = page.getByRole("cell", { name: label, exact: true }); + await cell.waitFor(); + if ((await cell.getAttribute("class") ?? "") !== className) { + throw new Error(`ratio comparison direction style is inconsistent: ${label}`); + } + } +} + async function main() { const { chromium } = loadPlaywright(); const server = process.env.LOOPX_BENCHMARK_STUDY_URL ? null : startDashboardServer(); @@ -195,6 +217,7 @@ async function main() { await rejected.getByText("Benchmark dashboard source must use same-origin local readback", { exact: true }).waitFor(); await assertCaseScopedRunSelection(desktop, url); + await assertRatioComparisonReadback(desktop, url); if (pageErrors.length) throw new Error(`browser errors: ${pageErrors.join(" | ")}`); console.log("benchmark study browser smoke passed"); } finally { diff --git a/loopx/capabilities/benchmark_toolkit/README.md b/loopx/capabilities/benchmark_toolkit/README.md index 0f647c9220..ef972c1bdb 100644 --- a/loopx/capabilities/benchmark_toolkit/README.md +++ b/loopx/capabilities/benchmark_toolkit/README.md @@ -924,6 +924,20 @@ metric, comparison protocol, score countability, and treatment fidelity. Exact protocol revisions remain visible as a warning even when a declared comparison protocol says an older credible score remains semantically comparable. +Metric comparisons preserve the raw `delta` (candidate value minus baseline +value). When both metrics have positive totals, `delta_rate` determines improvement +and the dashboard's largest eligible contrast; Markdown and dashboard labels show +this change in percentage points (`pp`). Scalar metrics use the raw delta. +Units, improvement direction, and the presence of a denominator must agree. +A mismatch, zero denominator, or non-finite difference retains the endpoint values +with `comparison_unavailable_reason`, without a delta or direction. An unavailable +primary metric excludes the matched pair; an unavailable auxiliary metric does +not invalidate an otherwise eligible primary comparison. Optional unit and +direction metadata may remain absent on both sides of historical board rows. +Factorial interactions use the same scale across all four cells and retain the +existing fixed-total requirement for their primary metric. The study dashboard +does not rank a mixture of scalar and ratio contrasts against one another. + Full post-run analysis stays in private `benchmark_case_insight_v0` storage. The board records only its compact status or handle, so reading the board cannot widen the solving agent's evidence boundary. diff --git a/loopx/capabilities/benchmark_toolkit/experiment_board.py b/loopx/capabilities/benchmark_toolkit/experiment_board.py index 3e4bc9f00b..9eced596f6 100644 --- a/loopx/capabilities/benchmark_toolkit/experiment_board.py +++ b/loopx/capabilities/benchmark_toolkit/experiment_board.py @@ -14,6 +14,7 @@ experiment_token_text as _token, ) from .factorial_contrast import ( + benchmark_metric_comparison_value, build_benchmark_factorial_contrasts, build_benchmark_metric_delta, ) @@ -636,6 +637,10 @@ def _build_comparison( metric_deltas[name] = build_benchmark_metric_delta( anchor_metrics[name], candidate_metrics[name] ) + if metric_deltas.get(candidate.get("primary_metric"), {}).get( + "comparison_unavailable_reason" + ): + reasons.append("primary_metric_definition_mismatch") return { "comparison_id": ( @@ -944,9 +949,11 @@ def render_benchmark_experiment_board_markdown(payload: Mapping[str, Any]) -> st else {} ) primary_delta = deltas.get(item.get("primary_metric")) - delta_value = ( - primary_delta.get("delta") if isinstance(primary_delta, Mapping) else "n/a" - ) + delta_value: str | float = "n/a" + if isinstance(primary_delta, Mapping): + value = benchmark_metric_comparison_value(primary_delta) + if value is not None: + delta_value = f"{value * 100:g} pp" if "delta_rate" in primary_delta else value reasons = ( ", ".join(str(value) for value in item.get("reason_codes", []) or []) or "none" @@ -997,10 +1004,12 @@ def render_benchmark_experiment_board_markdown(payload: Mapping[str, Any]) -> st ) primary = metrics.get(item.get("primary_metric")) interaction_value = ( - primary.get("difference_in_differences") + primary.get("difference_in_differences", "n/a") if isinstance(primary, Mapping) else "n/a" ) + if isinstance(primary, Mapping) and "difference_in_differences_rate" in primary: + interaction_value = f"{primary['difference_in_differences_rate'] * 100:g} pp" reasons = ( ", ".join(str(value) for value in item.get("reason_codes", []) or []) or "none" diff --git a/loopx/capabilities/benchmark_toolkit/factorial_contrast.py b/loopx/capabilities/benchmark_toolkit/factorial_contrast.py index 2c641f30be..52ef54d82e 100644 --- a/loopx/capabilities/benchmark_toolkit/factorial_contrast.py +++ b/loopx/capabilities/benchmark_toolkit/factorial_contrast.py @@ -3,6 +3,7 @@ from __future__ import annotations from collections.abc import Iterable, Mapping +import math from typing import Any from .experiment_identity import experiment_token_text as _token @@ -23,17 +24,38 @@ def _optional_token(value: Any, *, field: str) -> str | None: return _token(value, field=field) +def _metric_comparison_issue( + baseline: Mapping[str, Any], candidate: Mapping[str, Any] +) -> str | None: + """Check metric meaning, without imposing a study's fixed-total policy.""" + + if baseline.get("unit") != candidate.get("unit"): + return "unit_mismatch" + if baseline.get("higher_is_better") != candidate.get("higher_is_better"): + return "direction_mismatch" + if ("total" in baseline) != ("total" in candidate): + return "denominator_presence_mismatch" + if "total" in baseline and any(metric["total"] <= 0 for metric in (baseline, candidate)): + return "non_positive_denominator" + return None + + def build_benchmark_metric_delta( baseline: Mapping[str, Any], candidate: Mapping[str, Any] ) -> dict[str, Any]: """Compare two normalized experiment-board metric values.""" - delta = float(candidate["value"]) - float(baseline["value"]) result: dict[str, Any] = { "baseline_value": baseline["value"], "candidate_value": candidate["value"], - "delta": delta, } + issue = _metric_comparison_issue(baseline, candidate) + if issue is not None: + return {**result, "comparison_unavailable_reason": issue} + delta = float(candidate["value"]) - float(baseline["value"]) + if not math.isfinite(delta): + return {**result, "comparison_unavailable_reason": "non_finite_difference"} + result["delta"] = delta baseline_total = baseline.get("total") candidate_total = candidate.get("total") if ( @@ -46,6 +68,15 @@ def build_benchmark_metric_delta( ): baseline_rate = float(baseline["value"]) / float(baseline_total) candidate_rate = float(candidate["value"]) / float(candidate_total) + if not all( + math.isfinite(value) + for value in (baseline_rate, candidate_rate, candidate_rate - baseline_rate) + ): + return { + "baseline_value": baseline["value"], + "candidate_value": candidate["value"], + "comparison_unavailable_reason": "non_finite_difference", + } result.update( { "baseline_total": baseline_total, @@ -60,15 +91,25 @@ def build_benchmark_metric_delta( isinstance(higher_is_better, bool) and baseline.get("higher_is_better") == higher_is_better ): - if delta == 0: + directional_delta = result.get("delta_rate", delta) + if directional_delta == 0: result["direction"] = "flat" - elif (delta > 0) == higher_is_better: + elif (directional_delta > 0) == higher_is_better: result["direction"] = "improved" else: result["direction"] = "regressed" return result +def benchmark_metric_comparison_value(metric: Mapping[str, Any]) -> float | None: + """Read the comparison scale while retaining the raw delta in the packet.""" + + if metric.get("comparison_unavailable_reason"): + return None + value = metric.get("delta_rate", metric.get("delta")) + return float(value) if isinstance(value, (int, float)) and math.isfinite(value) else None + + def _normalize_four_arm_design(contract: Mapping[str, Any]) -> dict[str, Any]: if not isinstance(contract, Mapping): raise TypeError("four-arm contract must be an object") @@ -252,6 +293,9 @@ def _interaction_metric( anchor_effect: Mapping[str, Any], higher_is_better: bool | None, ) -> dict[str, Any]: + for effect in (candidate_effect, anchor_effect): + if effect.get("comparison_unavailable_reason"): + return {"comparison_unavailable_reason": effect["comparison_unavailable_reason"]} candidate_delta = candidate_effect.get("delta") anchor_delta = anchor_effect.get("delta") result: dict[str, Any] = { @@ -263,12 +307,6 @@ def _interaction_metric( ): difference = candidate_delta - anchor_delta result["difference_in_differences"] = difference - if difference == 0: - result["direction"] = "flat" - elif isinstance(higher_is_better, bool): - result["direction"] = ( - "improved" if (difference > 0) == higher_is_better else "regressed" - ) candidate_rate = candidate_effect.get("delta_rate") anchor_rate = anchor_effect.get("delta_rate") if isinstance(candidate_rate, (int, float)) and isinstance( @@ -277,6 +315,17 @@ def _interaction_metric( result["candidate_effect_delta_rate"] = candidate_rate result["anchor_effect_delta_rate"] = anchor_rate result["difference_in_differences_rate"] = candidate_rate - anchor_rate + if any(not math.isfinite(value) for value in result.values()): + return {"comparison_unavailable_reason": "non_finite_difference"} + difference = result.get( + "difference_in_differences_rate", result.get("difference_in_differences") + ) + if difference == 0: + result["direction"] = "flat" + elif isinstance(higher_is_better, bool): + result["direction"] = ( + "improved" if (difference > 0) == higher_is_better else "regressed" + ) return result @@ -353,17 +402,18 @@ def build_benchmark_factorial_contrasts( design=design, effect_by_pair=effect_by_pair, ) + primary_metric = ( + next(iter(selected.values())).get("primary_metric") if selected else None + ) + if interaction_metrics.get(primary_metric, {}).get("comparison_unavailable_reason"): + reasons.append("primary_metric_definition_mismatch") contrasts.append( { "schema_version": BENCHMARK_FACTORIAL_CONTRAST_SCHEMA_VERSION, "benchmark_id": benchmark_id, "study_id": study_id, "case_id": case_id, - "primary_metric": ( - next(iter(selected.values())).get("primary_metric") - if selected - else None - ), + "primary_metric": primary_metric, "factorial_contrast_countable": not reasons, "qualification_scope": "declared_factor_design_and_board_results", "reason_codes": sorted(set(reasons)), @@ -454,7 +504,13 @@ def reject_cell(arm_id: str, reason: str) -> None: } for row in selected_rows ] - if any(signature != metric_signatures[0] for signature in metric_signatures[1:]): + if ( + any(signature != metric_signatures[0] for signature in metric_signatures[1:]) + or build_benchmark_metric_delta( + selected_rows[0]["metrics"][primary_metric], + selected_rows[0]["metrics"][primary_metric], + ).get("comparison_unavailable_reason") + ): reasons.append("primary_metric_definition_mismatch") @@ -504,6 +560,19 @@ def _interaction_metrics( interaction_metrics: dict[str, Any] = {} for name in sorted(set(candidate_metrics) & set(anchor_metrics)): metric_source = selected[candidate_effect["candidate_arm_id"]]["metrics"][name] + # Conditional effects can each be valid while using different units or + # scales. Check all four cells before subtracting those effects. + issue = next( + ( + reason + for row in selected.values() + if (reason := _metric_comparison_issue(metric_source, row["metrics"][name])) + ), + None, + ) + if issue is not None: + interaction_metrics[name] = {"comparison_unavailable_reason": issue} + continue interaction_metrics[name] = _interaction_metric( candidate_effect=candidate_metrics[name], anchor_effect=anchor_metrics[name], diff --git a/loopx/capabilities/benchmark_toolkit/study_projection.py b/loopx/capabilities/benchmark_toolkit/study_projection.py index 76db741896..85d1dc586b 100644 --- a/loopx/capabilities/benchmark_toolkit/study_projection.py +++ b/loopx/capabilities/benchmark_toolkit/study_projection.py @@ -23,6 +23,7 @@ normalize_benchmark_experiment_board_row, preview_benchmark_experiment_board_upsert, ) +from .factorial_contrast import benchmark_metric_comparison_value from .experiment_identity import ( ARM_ROLES, experiment_run_key, @@ -1240,16 +1241,22 @@ def build_benchmark_study_dashboard( ranked = [ comparison for comparison in eligible - if isinstance( - comparison.get("metric_deltas", {}) - .get(primary_metric, {}) - .get("delta"), - (int, float), + if ( + benchmark_metric_comparison_value( + comparison.get("metric_deltas", {}).get(primary_metric, {}) + ) + is not None ) ] + # A raw count and a rate have no meaningful shared magnitude ordering. + scales = {"delta_rate" in item["metric_deltas"][primary_metric] for item in ranked} + if len(scales) > 1: + ranked = [] largest = max( ranked, - key=lambda item: abs(item["metric_deltas"][primary_metric]["delta"]), + key=lambda item: abs( + benchmark_metric_comparison_value(item["metric_deltas"][primary_metric]) or 0 + ), default=None, ) case["eligible_comparisons"] = eligible @@ -1276,7 +1283,9 @@ def build_benchmark_study_dashboard( directions[direction] += 1 for metric_name, transitions in binary_transitions.items(): binary_delta = item.get("metric_deltas", {}).get(metric_name) - if not isinstance(binary_delta, Mapping): + if not isinstance(binary_delta, Mapping) or binary_delta.get( + "comparison_unavailable_reason" + ): continue before, after = ( binary_delta.get("baseline_value"), diff --git a/tests/capabilities/test_benchmark_experiment_board.py b/tests/capabilities/test_benchmark_experiment_board.py index 30319fe82b..a566ef0b98 100644 --- a/tests/capabilities/test_benchmark_experiment_board.py +++ b/tests/capabilities/test_benchmark_experiment_board.py @@ -128,6 +128,217 @@ def _running_baseline(**overrides: object) -> dict[str, object]: return payload +@pytest.mark.parametrize( + "before,after,higher,expected", + [ + (50, 60, True, "regressed"), + (50, 100, True, "flat"), + (50, 120, True, "improved"), + (50, 60, False, "improved"), + (50, 100, False, "flat"), + (50, 120, False, "regressed"), + ], +) +def test_metric_direction_uses_ratio_scale(before, after, higher, expected): + baseline = _baseline() + candidate = _baseline( + run_id="candidate", + arm_id="treatment", + arm_role="treatment", + treatment_fidelity="qualified", + comparison_anchor_run_id=baseline["run_id"], + ) + baseline["metrics"]["feature_pass"] = { + "value": before, + "total": 100, + "higher_is_better": higher, + } + candidate["metrics"]["feature_pass"] = { + "value": after, + "total": 200, + "higher_is_better": higher, + } + comparison = build_benchmark_experiment_board([baseline, candidate])["comparisons"][ + 0 + ] + metric = comparison["metric_deltas"]["feature_pass"] + assert comparison["matched_pair_countable"] is True + assert metric["delta"] == after - before + assert metric["delta_rate"] == pytest.approx(after / 200 - before / 100) + assert metric["direction"] == expected + rendered = render_benchmark_experiment_board_markdown( + build_benchmark_experiment_board([baseline, candidate]) + ) + assert f"{(after / 200 - before / 100) * 100:g} pp" in rendered + + +@pytest.mark.parametrize( + "before,after,reason", + [ + ({"value": 1000, "unit": "ms"}, {"value": 2, "unit": "s"}, "unit_mismatch"), + ({"value": 1, "unit": "ms"}, {"value": 2}, "unit_mismatch"), + ( + {"value": 1, "higher_is_better": True}, + {"value": 2, "higher_is_better": False}, + "direction_mismatch", + ), + ({"value": 1, "higher_is_better": True}, {"value": 2}, "direction_mismatch"), + ({"value": 1, "total": 10}, {"value": 2}, "denominator_presence_mismatch"), + ( + {"value": 1, "total": 10}, + {"value": 0, "total": 0}, + "non_positive_denominator", + ), + ], +) +def test_primary_metric_incompatibility_excludes_pair(before, after, reason): + baseline = _baseline() + candidate = _baseline( + run_id="candidate", + arm_id="treatment", + arm_role="treatment", + treatment_fidelity="qualified", + comparison_anchor_run_id=baseline["run_id"], + ) + baseline["metrics"]["feature_pass"] = before + candidate["metrics"]["feature_pass"] = after + comparison = build_benchmark_experiment_board([baseline, candidate])["comparisons"][ + 0 + ] + assert comparison["matched_pair_countable"] is False + assert "primary_metric_definition_mismatch" in comparison["reason_codes"] + metric = comparison["metric_deltas"]["feature_pass"] + assert metric["comparison_unavailable_reason"] == reason + assert "delta" not in metric and "direction" not in metric + + +def test_incompatible_auxiliary_metric_preserves_primary_pair(): + baseline = _baseline() + candidate = _baseline( + run_id="candidate", + arm_id="treatment", + arm_role="treatment", + treatment_fidelity="qualified", + comparison_anchor_run_id=baseline["run_id"], + ) + candidate["metrics"]["preservation_pass"]["unit"] = "seconds" + comparison = build_benchmark_experiment_board([baseline, candidate])["comparisons"][ + 0 + ] + assert comparison["matched_pair_countable"] is True + assert comparison["metric_deltas"]["feature_pass"]["direction"] == "flat" + assert ( + comparison["metric_deltas"]["preservation_pass"][ + "comparison_unavailable_reason" + ] + == "unit_mismatch" + ) + # Historical scalar rows need not acquire optional metadata to remain readable. + baseline["metrics"]["feature_pass"] = {"value": 1} + candidate["metrics"]["feature_pass"] = {"value": 2} + comparison = build_benchmark_experiment_board([baseline, candidate])["comparisons"][ + 0 + ] + assert comparison["matched_pair_countable"] is True + assert comparison["metric_deltas"]["feature_pass"] == { + "baseline_value": 1, + "candidate_value": 2, + "delta": 1.0, + } + + +def test_factorial_ratio_interaction_uses_one_scale(): + rows = _four_arm_rows() + for row, value, total in zip(rows, [50, 60, 50, 55], [100, 200, 100, 100]): + row["metrics"]["preservation_pass"] = { + "value": value, + "total": total, + "higher_is_better": True, + } + contrast = build_benchmark_experiment_board( + rows, + four_arm_contract=_four_arm_contract(), + )["factorial_contrasts"][0] + assert contrast["factorial_contrast_countable"] is True + interaction = contrast["interaction_contrast"]["metric_contrasts"][ + "preservation_pass" + ] + assert interaction["difference_in_differences"] == -5 + assert interaction["difference_in_differences_rate"] == pytest.approx(0.25) + assert interaction["direction"] == "improved" + # Two individually compatible effects still cannot be subtracted across units. + for row in rows[2:]: + row["metrics"]["preservation_pass"]["unit"] = "different-unit" + contrast = build_benchmark_experiment_board( + rows, + four_arm_contract=_four_arm_contract(), + )["factorial_contrasts"][0] + assert contrast["factorial_contrast_countable"] is True + assert contrast["interaction_contrast"]["metric_contrasts"][ + "preservation_pass" + ] == { + "comparison_unavailable_reason": "unit_mismatch", + } + + +def test_factorial_zero_primary_denominator_is_not_countable(): + rows = _four_arm_rows() + for row in rows: + row["metrics"]["feature_pass"].update(value=0, total=0) + contrast = build_benchmark_experiment_board( + rows, + four_arm_contract=_four_arm_contract(), + )["factorial_contrasts"][0] + assert contrast["factorial_contrast_countable"] is False + assert "primary_metric_definition_mismatch" in contrast["reason_codes"] + + +@pytest.mark.parametrize("total", [None, 1e-308]) +def test_non_finite_comparison_does_not_publish_a_direction(total): + baseline = _baseline() + candidate = _baseline( + run_id="candidate", + arm_id="treatment", + arm_role="treatment", + treatment_fidelity="qualified", + comparison_anchor_run_id=baseline["run_id"], + ) + for row, value in [(baseline, -1e308), (candidate, 1e308)]: + row["metrics"]["feature_pass"] = {"value": value, "higher_is_better": True} + if total is not None: + row["metrics"]["feature_pass"].update( + value=1 if row is baseline else 2, total=total + ) + comparison = build_benchmark_experiment_board([baseline, candidate])["comparisons"][ + 0 + ] + assert comparison["matched_pair_countable"] is False + assert comparison["metric_deltas"]["feature_pass"] == { + "baseline_value": baseline["metrics"]["feature_pass"]["value"], + "candidate_value": candidate["metrics"]["feature_pass"]["value"], + "comparison_unavailable_reason": "non_finite_difference", + } + + +@pytest.mark.parametrize("metric_name", ["feature_pass", "preservation_pass"]) +def test_factorial_non_finite_raw_interaction_is_unavailable(metric_name): + rows = _four_arm_rows() + for row, value in zip(rows, [1e308, 0, 0, 1e308]): + row["metrics"][metric_name] = { + "value": value, + "total": 1e308, + "higher_is_better": True, + } + contrast = build_benchmark_experiment_board( + rows, + four_arm_contract=_four_arm_contract(), + )["factorial_contrasts"][0] + assert contrast["factorial_contrast_countable"] is (metric_name != "feature_pass") + assert contrast["interaction_contrast"]["metric_contrasts"][metric_name] == { + "comparison_unavailable_reason": "non_finite_difference", + } + + def _four_arm_contract() -> dict[str, object]: return compact_benchmark_four_arm_contract( build_benchmark_four_arm_contract( diff --git a/tests/capabilities/test_benchmark_study_projection.py b/tests/capabilities/test_benchmark_study_projection.py index a51e479734..42c55ffca2 100644 --- a/tests/capabilities/test_benchmark_study_projection.py +++ b/tests/capabilities/test_benchmark_study_projection.py @@ -157,6 +157,128 @@ def _five_mode_manifest() -> dict[str, object]: return manifest +def test_ratio_comparisons_round_trip_and_rank_on_the_same_scale(tmp_path: Path): + manifest = _manifest() + manifest["factors"][0]["levels"].append("alternative") + manifest["arms"].append( + { + "arm_id": "alternative", + "arm_role": "treatment", + "factor_assignments": {"orchestrator": "alternative"}, + } + ) + baseline = _row(arm_id="goal_plain", arm_role="baseline", feature=50, reward=0) + candidate = _row( + arm_id="loopx_plain", + arm_role="treatment", + feature=600, + reward=1, + anchor=baseline["run_id"], + ) + alternative = _row( + arm_id="alternative", + arm_role="treatment", + feature=9, + reward=1, + anchor=baseline["run_id"], + ) + store = tmp_path / "uploads.jsonl" + for row, total in zip([baseline, candidate, alternative], [100, 1000, 10]): + row["metrics"]["requirements_passed"]["total"] = total + simulate_benchmark_upload( + store, + _envelope( + row, + record_kind="experiment_board_row", + key=row["run_id"], + ), + execute=True, + ) + manifest_path = tmp_path / "manifest.json" + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + packet = _loopx_json( + "benchmark", + "study-dashboard", + "--manifest-json", + str(manifest_path), + "--store", + str(store), + "--format", + "json", + ) + assert packet == build_benchmark_study_dashboard( + manifest, read_benchmark_local_upload_records(store) + ) + largest = packet["cases"][0]["largest_eligible_primary_contrast"] + assert largest["candidate_arm_id"] == "alternative" + assert largest["metric_deltas"]["requirements_passed"]["delta"] == -41 + assert largest["metric_deltas"]["requirements_passed"][ + "delta_rate" + ] == pytest.approx(0.4) + assert packet["contrasts"]["alternative"]["primary_metric_directions"] == { + "improved": 1, + "flat": 0, + "regressed": 0, + } + + +def test_dashboard_does_not_rank_mixed_comparison_scales(): + manifest = _manifest() + baseline = _row(arm_id="goal_plain", arm_role="baseline", feature=5, reward=0) + candidate = _row( + arm_id="loopx_plain", + arm_role="treatment", + feature=9, + reward=1, + anchor=baseline["run_id"], + ) + scalar_baseline, scalar_candidate = ( + copy.deepcopy(baseline), + copy.deepcopy(candidate), + ) + scalar_baseline["run_id"] = "scalar-baseline" + scalar_candidate["run_id"] = "scalar-candidate" + scalar_candidate["comparison_anchor_run_id"] = scalar_baseline["run_id"] + for row in [scalar_baseline, scalar_candidate]: + del row["metrics"]["requirements_passed"]["total"] + packet = build_benchmark_study_dashboard( + manifest, + [ + _envelope(row, record_kind="experiment_board_row", key=row["run_id"]) + for row in [baseline, candidate, scalar_baseline, scalar_candidate] + ], + ) + assert len(packet["cases"][0]["eligible_comparisons"]) == 2 + assert packet["cases"][0]["largest_eligible_primary_contrast"] is None + + +def test_unavailable_auxiliary_metric_is_not_counted_as_a_binary_transition(): + baseline = _row(arm_id="goal_plain", arm_role="baseline", feature=5, reward=0) + candidate = _row( + arm_id="loopx_plain", + arm_role="treatment", + feature=9, + reward=1, + anchor=baseline["run_id"], + ) + baseline["metrics"]["reward"]["total"] = 0 + packet = build_benchmark_study_dashboard( + _manifest(), + [ + _envelope(row, record_kind="experiment_board_row", key=row["run_id"]) + for row in [baseline, candidate] + ], + ) + contrast = packet["contrasts"]["loopx_plain"] + assert contrast["matched_pair_denominator"] == 1 + assert contrast["primary_metric_directions"]["improved"] == 1 + assert contrast["binary_metric_transitions"]["reward"] == { + "0_to_1": 0, + "1_to_0": 0, + "same": 0, + } + + def _row( *, arm_id: str,