From 5f72905c55f33a67a84900f9c09838ecc7ebaf77 Mon Sep 17 00:00:00 2001 From: OpenCode Date: Fri, 10 Jul 2026 10:41:48 +0700 Subject: [PATCH] feat: Phase C C9 cheap enemy comparison report -- standardizes anti-hype by requiring all declared enemies to be beaten (#889) --- .../evidence/cheap_enemy_comparison.py | 159 ++++++++ tests/evidence/test_cheap_enemy_comparison.py | 352 ++++++++++++++++++ tests/test_test_count_regression.py | 2 +- 3 files changed, 512 insertions(+), 1 deletion(-) create mode 100644 src/openamp_foundry/evidence/cheap_enemy_comparison.py create mode 100644 tests/evidence/test_cheap_enemy_comparison.py diff --git a/src/openamp_foundry/evidence/cheap_enemy_comparison.py b/src/openamp_foundry/evidence/cheap_enemy_comparison.py new file mode 100644 index 00000000..1841eede --- /dev/null +++ b/src/openamp_foundry/evidence/cheap_enemy_comparison.py @@ -0,0 +1,159 @@ +"""Cheap enemy comparison report — Phase C C9. + +Standardizes anti-hype: compares an advanced scorer's metric against every +declared cheap enemy baseline from a benchmark card. An advanced scorer +earns ranking authority ONLY if it beats ALL cheap enemies on the primary metric. + +This prevents performance claims that would be matched or exceeded by trivial +heuristics (charge threshold, length filter, hydrophobicity cutoff). +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + + +@dataclass +class EnemyResult: + enemy_name: str + enemy_score: float + advanced_score: float + margin: float + beats_enemy: bool + + +@dataclass +class CheapEnemyComparisonReport: + bmc_id: str + metric: str + advanced_scorer_name: str + advanced_scorer_score: float + enemy_results: list[EnemyResult] = field(default_factory=list) + n_enemies: int = 0 + n_beaten: int = 0 + n_failed: int = 0 + all_enemies_beaten: bool = False + ranking_authority_granted: bool = False + comparison_summary: str = "" + failed_enemies: list[str] = field(default_factory=list) + + +def compute_cheap_enemy_comparison( + bmc_id: str, + metric: str, + advanced_scorer_name: str, + advanced_scorer_score: float, + enemy_scores: dict[str, float], + higher_is_better: bool = True, +) -> CheapEnemyComparisonReport: + """Compare an advanced scorer against all declared cheap enemy baselines. + + Args: + bmc_id: BMC- card ID that governs this benchmark. + metric: The evaluation metric being compared (e.g. "precision_at_k"). + advanced_scorer_name: Name of the advanced scoring method being evaluated. + advanced_scorer_score: The advanced scorer's metric value. + enemy_scores: Dict mapping cheap enemy name to its metric value. + higher_is_better: Whether higher metric values are better (True for precision, + AUC, etc.; False for calibration error, etc.). + + Returns: + CheapEnemyComparisonReport with per-enemy pass/fail and overall authority decision. + + Raises: + ValueError: If bmc_id does not start with 'BMC-'. + ValueError: If enemy_scores is empty. + """ + if not bmc_id.startswith("BMC-"): + raise ValueError(f"bmc_id must start with 'BMC-', got '{bmc_id}'") + if not enemy_scores: + raise ValueError("enemy_scores must not be empty — at least one cheap enemy is required") + + results: list[EnemyResult] = [] + failed: list[str] = [] + + for enemy_name, enemy_score in sorted(enemy_scores.items()): + if higher_is_better: + beats = advanced_scorer_score > enemy_score + margin = advanced_scorer_score - enemy_score + else: + beats = advanced_scorer_score < enemy_score + margin = enemy_score - advanced_scorer_score + + result = EnemyResult( + enemy_name=enemy_name, + enemy_score=enemy_score, + advanced_score=advanced_scorer_score, + margin=margin, + beats_enemy=beats, + ) + results.append(result) + if not beats: + failed.append(enemy_name) + + n_beaten = sum(1 for r in results if r.beats_enemy) + n_failed = len(failed) + all_beaten = n_failed == 0 + authority = all_beaten + + direction = "↑ higher" if higher_is_better else "↓ lower" + if all_beaten: + summary = ( + f"{advanced_scorer_name} beats ALL {len(results)} cheap enemies on {metric} " + f"({direction} is better). Ranking authority GRANTED for {bmc_id}." + ) + else: + failed_str = ", ".join(failed) + summary = ( + f"{advanced_scorer_name} FAILS to beat {n_failed}/{len(results)} cheap enemies " + f"on {metric} ({direction} is better): [{failed_str}]. " + f"Ranking authority DENIED for {bmc_id}. " + "Do not promote this scorer until all enemies are beaten." + ) + + return CheapEnemyComparisonReport( + bmc_id=bmc_id, + metric=metric, + advanced_scorer_name=advanced_scorer_name, + advanced_scorer_score=advanced_scorer_score, + enemy_results=results, + n_enemies=len(results), + n_beaten=n_beaten, + n_failed=n_failed, + all_enemies_beaten=all_beaten, + ranking_authority_granted=authority, + comparison_summary=summary, + failed_enemies=failed, + ) + + +def format_cheap_enemy_comparison(report: CheapEnemyComparisonReport) -> str: + """Format a CheapEnemyComparisonReport as a human-readable string.""" + lines = [ + "=== CHEAP ENEMY COMPARISON REPORT ===", + f"Benchmark card: {report.bmc_id}", + f"Metric: {report.metric}", + f"Advanced scorer: {report.advanced_scorer_name} ({report.advanced_scorer_score:.4f})", + "", + f"-- CHEAP ENEMY RESULTS ({report.n_beaten}/{report.n_enemies} beaten) --", + ] + for result in sorted(report.enemy_results, key=lambda r: r.enemy_name): + status = "PASS" if result.beats_enemy else "FAIL" + sign = "+" if result.margin >= 0 else "" + lines.append( + f" [{status}] {result.enemy_name}: " + f"enemy={result.enemy_score:.4f}, " + f"advanced={result.advanced_score:.4f}, " + f"margin={sign}{result.margin:.4f}" + ) + lines.extend([ + "", + f"ALL ENEMIES BEATEN: {'YES' if report.all_enemies_beaten else 'NO'}", + f"RANKING AUTHORITY: {'GRANTED' if report.ranking_authority_granted else 'DENIED'}", + "", + report.comparison_summary, + "", + "NOTICE: Ranking authority requires beating ALL declared cheap enemies.", + "Partial victory is NOT sufficient — see CLAUDE.md anti-hype clause.", + ]) + return "\n".join(lines) diff --git a/tests/evidence/test_cheap_enemy_comparison.py b/tests/evidence/test_cheap_enemy_comparison.py new file mode 100644 index 00000000..9d64250e --- /dev/null +++ b/tests/evidence/test_cheap_enemy_comparison.py @@ -0,0 +1,352 @@ +"""Tests for cheap_enemy_comparison module (C9).""" +import pytest + +from openamp_foundry.evidence.cheap_enemy_comparison import ( + CheapEnemyComparisonReport, + EnemyResult, + compute_cheap_enemy_comparison, + format_cheap_enemy_comparison, +) + + +# --- EnemyResult --- + +class TestEnemyResult: + def test_is_dataclass(self): + r = EnemyResult( + enemy_name="charge_threshold", + enemy_score=0.5, + advanced_score=0.7, + margin=0.2, + beats_enemy=True, + ) + assert isinstance(r, EnemyResult) + + def test_fields_accessible(self): + r = EnemyResult("test", 0.3, 0.5, 0.2, True) + assert r.enemy_name == "test" + assert r.enemy_score == 0.3 + assert r.advanced_score == 0.5 + assert r.margin == pytest.approx(0.2) + assert r.beats_enemy is True + + +# --- compute_cheap_enemy_comparison --- + +class TestComputeCheapEnemyComparison: + def test_beats_all_enemies(self): + report = compute_cheap_enemy_comparison( + bmc_id="BMC-0001", + metric="precision_at_k", + advanced_scorer_name="ensemble_v1", + advanced_scorer_score=0.80, + enemy_scores={"charge_threshold": 0.60, "length_filter": 0.55}, + ) + assert report.all_enemies_beaten is True + assert report.ranking_authority_granted is True + assert report.n_beaten == 2 + assert report.n_failed == 0 + + def test_fails_one_enemy(self): + report = compute_cheap_enemy_comparison( + bmc_id="BMC-0001", + metric="precision_at_k", + advanced_scorer_name="ensemble_v1", + advanced_scorer_score=0.60, + enemy_scores={"charge_threshold": 0.65, "length_filter": 0.55}, + ) + assert report.all_enemies_beaten is False + assert report.ranking_authority_granted is False + assert report.n_beaten == 1 + assert report.n_failed == 1 + assert "charge_threshold" in report.failed_enemies + + def test_fails_all_enemies(self): + report = compute_cheap_enemy_comparison( + bmc_id="BMC-0001", + metric="precision_at_k", + advanced_scorer_name="weak_scorer", + advanced_scorer_score=0.40, + enemy_scores={"charge_threshold": 0.60, "length_filter": 0.55}, + ) + assert report.all_enemies_beaten is False + assert report.n_failed == 2 + assert len(report.failed_enemies) == 2 + + def test_invalid_bmc_id_raises(self): + with pytest.raises(ValueError, match="BMC-"): + compute_cheap_enemy_comparison( + bmc_id="NOT-VALID", + metric="precision_at_k", + advanced_scorer_name="scorer", + advanced_scorer_score=0.8, + enemy_scores={"enemy": 0.5}, + ) + + def test_empty_enemy_scores_raises(self): + with pytest.raises(ValueError, match="empty"): + compute_cheap_enemy_comparison( + bmc_id="BMC-0001", + metric="precision_at_k", + advanced_scorer_name="scorer", + advanced_scorer_score=0.8, + enemy_scores={}, + ) + + def test_lower_is_better_mode(self): + report = compute_cheap_enemy_comparison( + bmc_id="BMC-0003", + metric="calibration_error", + advanced_scorer_name="calibrated_v1", + advanced_scorer_score=0.05, + enemy_scores={"isotonic_regression": 0.10}, + higher_is_better=False, + ) + assert report.all_enemies_beaten is True + assert report.ranking_authority_granted is True + + def test_lower_is_better_fails(self): + report = compute_cheap_enemy_comparison( + bmc_id="BMC-0003", + metric="calibration_error", + advanced_scorer_name="bad_calibration", + advanced_scorer_score=0.15, + enemy_scores={"isotonic_regression": 0.10}, + higher_is_better=False, + ) + assert report.all_enemies_beaten is False + assert report.ranking_authority_granted is False + + def test_margin_correct_higher_better(self): + report = compute_cheap_enemy_comparison( + bmc_id="BMC-0001", + metric="precision_at_k", + advanced_scorer_name="scorer", + advanced_scorer_score=0.80, + enemy_scores={"charge_threshold": 0.60}, + ) + result = report.enemy_results[0] + assert result.margin == pytest.approx(0.20) + + def test_margin_correct_lower_better(self): + report = compute_cheap_enemy_comparison( + bmc_id="BMC-0003", + metric="calibration_error", + advanced_scorer_name="scorer", + advanced_scorer_score=0.05, + enemy_scores={"baseline": 0.10}, + higher_is_better=False, + ) + result = report.enemy_results[0] + assert result.margin == pytest.approx(0.05) + + def test_bmc_id_stored(self): + report = compute_cheap_enemy_comparison( + bmc_id="BMC-0002", + metric="auc_roc", + advanced_scorer_name="scorer", + advanced_scorer_score=0.75, + enemy_scores={"baseline": 0.70}, + ) + assert report.bmc_id == "BMC-0002" + + def test_metric_stored(self): + report = compute_cheap_enemy_comparison( + bmc_id="BMC-0001", + metric="enrichment_factor", + advanced_scorer_name="scorer", + advanced_scorer_score=3.0, + enemy_scores={"baseline": 2.0}, + ) + assert report.metric == "enrichment_factor" + + def test_scorer_name_stored(self): + report = compute_cheap_enemy_comparison( + bmc_id="BMC-0001", + metric="precision_at_k", + advanced_scorer_name="my_ensemble", + advanced_scorer_score=0.8, + enemy_scores={"baseline": 0.6}, + ) + assert report.advanced_scorer_name == "my_ensemble" + + def test_scorer_score_stored(self): + report = compute_cheap_enemy_comparison( + bmc_id="BMC-0001", + metric="precision_at_k", + advanced_scorer_name="scorer", + advanced_scorer_score=0.777, + enemy_scores={"baseline": 0.5}, + ) + assert report.advanced_scorer_score == pytest.approx(0.777) + + def test_n_enemies_correct(self): + report = compute_cheap_enemy_comparison( + bmc_id="BMC-0001", + metric="precision_at_k", + advanced_scorer_name="scorer", + advanced_scorer_score=0.8, + enemy_scores={"a": 0.5, "b": 0.6, "c": 0.7}, + ) + assert report.n_enemies == 3 + + def test_enemy_results_list_length(self): + report = compute_cheap_enemy_comparison( + bmc_id="BMC-0001", + metric="precision_at_k", + advanced_scorer_name="scorer", + advanced_scorer_score=0.8, + enemy_scores={"a": 0.5, "b": 0.6}, + ) + assert len(report.enemy_results) == 2 + + def test_returns_dataclass(self): + report = compute_cheap_enemy_comparison( + bmc_id="BMC-0001", + metric="precision_at_k", + advanced_scorer_name="scorer", + advanced_scorer_score=0.8, + enemy_scores={"baseline": 0.5}, + ) + assert isinstance(report, CheapEnemyComparisonReport) + + def test_summary_contains_granted_when_all_beaten(self): + report = compute_cheap_enemy_comparison( + bmc_id="BMC-0001", + metric="precision_at_k", + advanced_scorer_name="scorer", + advanced_scorer_score=0.8, + enemy_scores={"baseline": 0.5}, + ) + assert "GRANTED" in report.comparison_summary + + def test_summary_contains_denied_when_failed(self): + report = compute_cheap_enemy_comparison( + bmc_id="BMC-0001", + metric="precision_at_k", + advanced_scorer_name="scorer", + advanced_scorer_score=0.4, + enemy_scores={"baseline": 0.5}, + ) + assert "DENIED" in report.comparison_summary + + def test_summary_contains_scorer_name(self): + report = compute_cheap_enemy_comparison( + bmc_id="BMC-0001", + metric="precision_at_k", + advanced_scorer_name="my_special_scorer", + advanced_scorer_score=0.8, + enemy_scores={"baseline": 0.5}, + ) + assert "my_special_scorer" in report.comparison_summary + + def test_failed_enemies_list_correct(self): + report = compute_cheap_enemy_comparison( + bmc_id="BMC-0001", + metric="precision_at_k", + advanced_scorer_name="scorer", + advanced_scorer_score=0.5, + enemy_scores={"weak": 0.4, "strong": 0.6}, + ) + assert "strong" in report.failed_enemies + assert "weak" not in report.failed_enemies + + def test_exact_tie_does_not_beat(self): + report = compute_cheap_enemy_comparison( + bmc_id="BMC-0001", + metric="precision_at_k", + advanced_scorer_name="scorer", + advanced_scorer_score=0.5, + enemy_scores={"baseline": 0.5}, + ) + assert report.all_enemies_beaten is False + assert report.n_failed == 1 + + def test_bmc_id_must_start_with_bmc(self): + with pytest.raises(ValueError): + compute_cheap_enemy_comparison("bmc-0001", "metric", "s", 0.5, {"e": 0.3}) + + def test_single_enemy_passes(self): + report = compute_cheap_enemy_comparison( + bmc_id="BMC-0001", + metric="precision_at_k", + advanced_scorer_name="scorer", + advanced_scorer_score=0.9, + enemy_scores={"only_enemy": 0.8}, + ) + assert report.all_enemies_beaten is True + assert report.n_enemies == 1 + + +# --- format_cheap_enemy_comparison --- + +class TestFormatCheapEnemyComparison: + def _passing_report(self) -> CheapEnemyComparisonReport: + return compute_cheap_enemy_comparison( + bmc_id="BMC-0001", + metric="precision_at_k", + advanced_scorer_name="ensemble_v1", + advanced_scorer_score=0.80, + enemy_scores={"charge_threshold": 0.60, "length_filter": 0.55}, + ) + + def _failing_report(self) -> CheapEnemyComparisonReport: + return compute_cheap_enemy_comparison( + bmc_id="BMC-0001", + metric="precision_at_k", + advanced_scorer_name="weak_scorer", + advanced_scorer_score=0.50, + enemy_scores={"charge_threshold": 0.60}, + ) + + def test_returns_string(self): + assert isinstance(format_cheap_enemy_comparison(self._passing_report()), str) + + def test_contains_header(self): + assert "CHEAP ENEMY COMPARISON" in format_cheap_enemy_comparison(self._passing_report()) + + def test_contains_bmc_id(self): + assert "BMC-0001" in format_cheap_enemy_comparison(self._passing_report()) + + def test_contains_metric(self): + assert "precision_at_k" in format_cheap_enemy_comparison(self._passing_report()) + + def test_contains_scorer_name(self): + assert "ensemble_v1" in format_cheap_enemy_comparison(self._passing_report()) + + def test_granted_in_passing(self): + assert "GRANTED" in format_cheap_enemy_comparison(self._passing_report()) + + def test_denied_in_failing(self): + assert "DENIED" in format_cheap_enemy_comparison(self._failing_report()) + + def test_pass_label_present(self): + assert "PASS" in format_cheap_enemy_comparison(self._passing_report()) + + def test_fail_label_present(self): + assert "FAIL" in format_cheap_enemy_comparison(self._failing_report()) + + def test_contains_notice(self): + assert "NOTICE" in format_cheap_enemy_comparison(self._passing_report()) + + def test_contains_summary(self): + report = self._passing_report() + text = format_cheap_enemy_comparison(report) + assert report.comparison_summary in text + + def test_enemy_names_in_output(self): + text = format_cheap_enemy_comparison(self._passing_report()) + assert "charge_threshold" in text + assert "length_filter" in text + + def test_scores_in_output(self): + text = format_cheap_enemy_comparison(self._passing_report()) + assert "0.80" in text or "0.8000" in text + + def test_all_enemies_beaten_yes(self): + text = format_cheap_enemy_comparison(self._passing_report()) + assert "YES" in text + + def test_all_enemies_beaten_no(self): + text = format_cheap_enemy_comparison(self._failing_report()) + assert "NO" in text diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index 60adb8f2..fa2d4af1 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -3,7 +3,7 @@ import sys -BASELINE = 7213 +BASELINE = 7276 TOLERANCE = 0.05