From b5204581eceaf50b4a2d1db6a97d202c4f183cfc Mon Sep 17 00:00:00 2001 From: OpenCode Date: Fri, 10 Jul 2026 10:12:42 +0700 Subject: [PATCH] feat: Phase C C8 benchmark deprecation banner system -- prevents stale benchmark authority --- docs/evidence/METRICS_CURRENT.md | 9 + docs/research/NEXT_100_PR_MAP.md | 2 +- .../evidence/benchmark_deprecation.py | 140 +++++++ tests/evidence/test_benchmark_deprecation.py | 381 ++++++++++++++++++ tests/test_test_count_regression.py | 2 +- 5 files changed, 532 insertions(+), 2 deletions(-) create mode 100644 src/openamp_foundry/evidence/benchmark_deprecation.py create mode 100644 tests/evidence/test_benchmark_deprecation.py diff --git a/docs/evidence/METRICS_CURRENT.md b/docs/evidence/METRICS_CURRENT.md index 09281a1c..4ef1fbb1 100644 --- a/docs/evidence/METRICS_CURRENT.md +++ b/docs/evidence/METRICS_CURRENT.md @@ -9,6 +9,15 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak ## Changelog +### v0.10.35 — Phase C C8: Benchmark deprecation banner system +- Added `src/openamp_foundry/evidence/benchmark_deprecation.py` +- Functions: get_deprecated_cards(), get_active_cards(), build_deprecation_banner(), print_all_deprecation_banners(), check_no_deprecated_in_ranking() (raises DeprecatedBenchmarkError), deprecation_status_report() +- DeprecatedBenchmarkError: raised when deprecated benchmarks are used in ranking context +- Main registry confirmed: all 5 BMC- cards are active (no deprecated) +- 63 tests in `tests/evidence/test_benchmark_deprecation.py` +- BASELINE 6772→6835 +- Closes Phase C C8 — stale benchmark authority is now machine-preventable via check_no_deprecated_in_ranking() + ### v0.10.34 — Phase C C1: Machine-readable benchmark registry - Added `src/openamp_foundry/evidence/benchmark_registry.py` — benchmark card registry - 5 BMC- cards: BMC-0001 (precision@k, leakage_aware_split), BMC-0002 (charge-matched, charge_stratified), BMC-0003 (calibration, random_70_30), BMC-0004 (family-stratified, family_stratified), BMC-0005 (cheap enemy comparison, leakage_aware_split) diff --git a/docs/research/NEXT_100_PR_MAP.md b/docs/research/NEXT_100_PR_MAP.md index 73260a5f..45a1a56a 100644 --- a/docs/research/NEXT_100_PR_MAP.md +++ b/docs/research/NEXT_100_PR_MAP.md @@ -65,7 +65,7 @@ Make OpenAMP the hardest repo to fool in AMP dry-lab selection. | C5 | Add `bench-length-matched` control. | Tests length shortcut. | C | | C6 | Add `bench-charge-length-matched` control. | Stronger nontrivial signal test. | C | | C7 | Add family-stratified precision@k. | Helps panel design. | C | -| C8 | Add benchmark-deprecation banner system. | Prevents stale benchmark authority. | B | +| C8 | Add benchmark-deprecation banner system (complete). | benchmark_deprecation.py: get_deprecated_cards(), build_deprecation_banner(), check_no_deprecated_in_ranking() (raises DeprecatedBenchmarkError), deprecation_status_report(); 63 tests; main registry confirmed all-active. | B | | C9 | Add command that compares all advanced scorers against declared cheap enemies. | Standardizes anti-hype. | C | | C10 | Add benchmark governance CI that rejects missing benchmark cards for new benchmark scripts. | Prevents benchmark sprawl. | B/C | diff --git a/src/openamp_foundry/evidence/benchmark_deprecation.py b/src/openamp_foundry/evidence/benchmark_deprecation.py new file mode 100644 index 00000000..19ca6355 --- /dev/null +++ b/src/openamp_foundry/evidence/benchmark_deprecation.py @@ -0,0 +1,140 @@ +"""Benchmark deprecation banner system — Phase C C8. + +Prevents stale benchmark authority by providing: +1. Detection of deprecated benchmark cards in the registry +2. Human-readable deprecation banners for each deprecated card +3. A strict governance check that fails if deprecated benchmarks are used + in official ranking decisions without explicit override + +Usage: + from openamp_foundry.evidence.benchmark_deprecation import ( + check_no_deprecated_in_ranking, + get_deprecated_cards, + build_deprecation_banner, + print_all_deprecation_banners, + ) + + # In CI or ranking pipeline: + check_no_deprecated_in_ranking(registry) # raises if any card is deprecated + + # For user-facing warnings: + print_all_deprecation_banners(registry) +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from openamp_foundry.evidence.benchmark_card import BenchmarkCard + +_BANNER_WIDTH = 72 +_BANNER_CHAR = "!" + + +class DeprecatedBenchmarkError(Exception): + """Raised when a deprecated benchmark is used in a ranking context.""" + + +def get_deprecated_cards( + registry: list["BenchmarkCard"], +) -> list["BenchmarkCard"]: + """Return all deprecated BenchmarkCards from the given registry.""" + return [card for card in registry if card.deprecated] + + +def get_active_cards( + registry: list["BenchmarkCard"], +) -> list["BenchmarkCard"]: + """Return all non-deprecated BenchmarkCards from the given registry.""" + return [card for card in registry if not card.deprecated] + + +def build_deprecation_banner(card: "BenchmarkCard") -> str: + """Build a human-readable deprecation banner for a deprecated benchmark. + + The banner is designed to be impossible to miss: it uses a visible + header, states the benchmark ID and name, and quotes the notes field + which must explain the deprecation reason. + """ + sep = _BANNER_CHAR * _BANNER_WIDTH + lines = [ + sep, + f"DEPRECATED BENCHMARK — {card.bmc_id}", + sep, + f"Name: {card.benchmark_name}", + f"Target: {card.measurement_target}", + f"Reason: {card.notes or '(no reason given)'}", + sep, + "WARNING: This benchmark is DEPRECATED and must NOT be used to make", + "ranking decisions or support candidate selection claims.", + "Remove or replace it before publishing results.", + sep, + ] + return "\n".join(lines) + + +def print_all_deprecation_banners( + registry: list["BenchmarkCard"], +) -> list[str]: + """Print deprecation banners for all deprecated cards. + + Returns the list of banners printed (useful for testing without capturing stdout). + """ + deprecated = get_deprecated_cards(registry) + banners: list[str] = [] + for card in deprecated: + banner = build_deprecation_banner(card) + print(banner) + banners.append(banner) + return banners + + +def check_no_deprecated_in_ranking( + registry: list["BenchmarkCard"], + allow_deprecated: bool = False, +) -> None: + """Strict governance check: raise if any deprecated benchmark is in the registry. + + Args: + registry: List of BenchmarkCard instances to check. + allow_deprecated: If True, skip the check (for testing only — must be + documented in code review). + + Raises: + DeprecatedBenchmarkError: If any card in the registry is deprecated. + """ + if allow_deprecated: + return + + deprecated = get_deprecated_cards(registry) + if deprecated: + names = ", ".join(f"{c.bmc_id} ({c.benchmark_name!r})" for c in deprecated) + raise DeprecatedBenchmarkError( + f"Deprecated benchmarks found in ranking registry: {names}. " + "Remove deprecated cards from the registry before running official " + "ranking. Set allow_deprecated=True only in test contexts with " + "explicit code-review approval." + ) + + +def deprecation_status_report( + registry: list["BenchmarkCard"], +) -> dict: + """Return a machine-readable deprecation status report for the registry. + + Returns a dict with: + total: int — total cards in registry + active: int — non-deprecated cards + deprecated: int — deprecated cards + deprecated_ids: list[str] — BMC IDs of deprecated cards + all_active: bool — True if no deprecated cards + """ + deprecated = get_deprecated_cards(registry) + return { + "total": len(registry), + "active": len(registry) - len(deprecated), + "deprecated": len(deprecated), + "deprecated_ids": [c.bmc_id for c in deprecated], + "all_active": len(deprecated) == 0, + } diff --git a/tests/evidence/test_benchmark_deprecation.py b/tests/evidence/test_benchmark_deprecation.py new file mode 100644 index 00000000..6a7731b3 --- /dev/null +++ b/tests/evidence/test_benchmark_deprecation.py @@ -0,0 +1,381 @@ +"""Tests for benchmark deprecation banner system — Phase C C8. + +63 tests across 7 groups. +""" + +from __future__ import annotations + +import pytest + +from openamp_foundry.evidence.benchmark_card import BenchmarkCard +from openamp_foundry.evidence.benchmark_deprecation import ( + DeprecatedBenchmarkError, + build_deprecation_banner, + check_no_deprecated_in_ranking, + deprecation_status_report, + get_active_cards, + get_deprecated_cards, + print_all_deprecation_banners, +) +from openamp_foundry.evidence.benchmark_registry import BENCHMARK_REGISTRY + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _active_card(bmc_id: str = "BMC-0010") -> BenchmarkCard: + return BenchmarkCard( + bmc_id=bmc_id, + pipeline_version="0.10.34", + benchmark_name="Test Active Benchmark", + measurement_target="antimicrobial_activity_prediction", + split_strategy="leakage_aware_split", + cheap_enemy_baselines=["charge_threshold_gte4", "length_heuristic"], + evaluation_metrics=["precision_at_k", "auc_roc"], + known_limitations=["Test only.", "Synthetic data."], + deprecated=False, + created_date="2024-06-01", + last_updated_date="2024-06-01", + notes="Test card.", + ) + + +def _deprecated_card(bmc_id: str = "BMC-DEPR-01") -> BenchmarkCard: + return BenchmarkCard( + bmc_id=bmc_id, + pipeline_version="0.10.34", + benchmark_name="Old Leaky Benchmark", + measurement_target="antimicrobial_activity_prediction", + split_strategy="random_80_20", + cheap_enemy_baselines=["charge_threshold_gte4", "length_heuristic"], + evaluation_metrics=["precision_at_k"], + known_limitations=["Had train/test leakage.", "Random split only."], + deprecated=True, + created_date="2024-01-01", + last_updated_date="2024-06-01", + notes="Superseded by BMC-0010 which uses leakage_aware_split.", + ) + + +# --------------------------------------------------------------------------- +# Group 1: GetDeprecatedAndActive (8 tests) +# --------------------------------------------------------------------------- + +class TestGetDeprecatedAndActive: + def test_get_deprecated_returns_only_deprecated(self): + registry = [_active_card(), _deprecated_card()] + deprecated = get_deprecated_cards(registry) + assert all(c.deprecated for c in deprecated) + + def test_get_deprecated_empty_for_all_active(self): + registry = [_active_card("BMC-0010"), _active_card("BMC-0011")] + assert get_deprecated_cards(registry) == [] + + def test_get_deprecated_all_for_all_deprecated(self): + registry = [_deprecated_card("BMC-D1"), _deprecated_card("BMC-D2")] + assert len(get_deprecated_cards(registry)) == 2 + + def test_get_active_returns_only_active(self): + registry = [_active_card(), _deprecated_card()] + active = get_active_cards(registry) + assert all(not c.deprecated for c in active) + + def test_get_active_empty_for_all_deprecated(self): + registry = [_deprecated_card()] + assert get_active_cards(registry) == [] + + def test_get_active_all_for_all_active(self): + registry = [_active_card("BMC-0010"), _active_card("BMC-0011")] + assert len(get_active_cards(registry)) == 2 + + def test_active_plus_deprecated_equals_total(self): + registry = [_active_card(), _deprecated_card()] + assert len(get_active_cards(registry)) + len(get_deprecated_cards(registry)) == len(registry) + + def test_main_registry_has_no_deprecated_cards(self): + assert get_deprecated_cards(BENCHMARK_REGISTRY) == [] + + +# --------------------------------------------------------------------------- +# Group 2: BuildDeprecationBanner (10 tests) +# --------------------------------------------------------------------------- + +class TestBuildDeprecationBanner: + def test_banner_returns_string(self): + assert isinstance(build_deprecation_banner(_deprecated_card()), str) + + def test_banner_non_empty(self): + assert len(build_deprecation_banner(_deprecated_card())) > 0 + + def test_banner_contains_deprecated_label(self): + banner = build_deprecation_banner(_deprecated_card()) + assert "DEPRECATED" in banner + + def test_banner_contains_bmc_id(self): + card = _deprecated_card("BMC-DEPR-01") + banner = build_deprecation_banner(card) + assert "BMC-DEPR-01" in banner + + def test_banner_contains_benchmark_name(self): + card = _deprecated_card() + banner = build_deprecation_banner(card) + assert card.benchmark_name in banner + + def test_banner_contains_deprecation_reason(self): + card = _deprecated_card() + banner = build_deprecation_banner(card) + assert "leakage_aware_split" in banner or "Superseded" in banner + + def test_banner_contains_warning_about_ranking(self): + banner = build_deprecation_banner(_deprecated_card()) + assert "ranking" in banner.lower() or "ranking decisions" in banner.lower() + + def test_banner_contains_remove_instruction(self): + banner = build_deprecation_banner(_deprecated_card()) + assert "remove" in banner.lower() or "replace" in banner.lower() + + def test_banner_is_multiline(self): + banner = build_deprecation_banner(_deprecated_card()) + assert "\n" in banner + + def test_banner_measurement_target_in_banner(self): + card = _deprecated_card() + banner = build_deprecation_banner(card) + assert card.measurement_target in banner + + +# --------------------------------------------------------------------------- +# Group 3: CheckNoDeprecatedInRanking (10 tests) +# --------------------------------------------------------------------------- + +class TestCheckNoDeprecatedInRanking: + def test_all_active_registry_passes(self): + registry = [_active_card()] + check_no_deprecated_in_ranking(registry) # should not raise + + def test_main_registry_passes(self): + check_no_deprecated_in_ranking(BENCHMARK_REGISTRY) # should not raise + + def test_deprecated_card_raises(self): + registry = [_deprecated_card()] + with pytest.raises(DeprecatedBenchmarkError): + check_no_deprecated_in_ranking(registry) + + def test_mixed_registry_raises(self): + registry = [_active_card(), _deprecated_card()] + with pytest.raises(DeprecatedBenchmarkError): + check_no_deprecated_in_ranking(registry) + + def test_error_message_contains_bmc_id(self): + registry = [_deprecated_card("BMC-DEPR-01")] + with pytest.raises(DeprecatedBenchmarkError) as exc_info: + check_no_deprecated_in_ranking(registry) + assert "BMC-DEPR-01" in str(exc_info.value) + + def test_error_message_mentions_deprecated(self): + registry = [_deprecated_card()] + with pytest.raises(DeprecatedBenchmarkError) as exc_info: + check_no_deprecated_in_ranking(registry) + assert "deprecated" in str(exc_info.value).lower() + + def test_allow_deprecated_flag_suppresses_error(self): + registry = [_deprecated_card()] + check_no_deprecated_in_ranking(registry, allow_deprecated=True) # should not raise + + def test_empty_registry_passes(self): + check_no_deprecated_in_ranking([]) # should not raise + + def test_two_deprecated_cards_raises(self): + registry = [_deprecated_card("BMC-D1"), _deprecated_card("BMC-D2")] + with pytest.raises(DeprecatedBenchmarkError): + check_no_deprecated_in_ranking(registry) + + def test_deprecated_benchmark_error_is_exception(self): + assert issubclass(DeprecatedBenchmarkError, Exception) + + +# --------------------------------------------------------------------------- +# Group 4: DeprecationStatusReport (8 tests) +# --------------------------------------------------------------------------- + +class TestDeprecationStatusReport: + def test_report_returns_dict(self): + assert isinstance(deprecation_status_report([_active_card()]), dict) + + def test_report_has_expected_keys(self): + report = deprecation_status_report([_active_card()]) + assert "total" in report + assert "active" in report + assert "deprecated" in report + assert "deprecated_ids" in report + assert "all_active" in report + + def test_all_active_registry_all_active_true(self): + report = deprecation_status_report([_active_card()]) + assert report["all_active"] is True + + def test_deprecated_registry_all_active_false(self): + report = deprecation_status_report([_deprecated_card()]) + assert report["all_active"] is False + + def test_counts_correct(self): + registry = [_active_card(), _deprecated_card()] + report = deprecation_status_report(registry) + assert report["total"] == 2 + assert report["active"] == 1 + assert report["deprecated"] == 1 + + def test_deprecated_ids_correct(self): + registry = [_active_card(), _deprecated_card("BMC-D1")] + report = deprecation_status_report(registry) + assert "BMC-D1" in report["deprecated_ids"] + + def test_main_registry_report_all_active(self): + report = deprecation_status_report(BENCHMARK_REGISTRY) + assert report["all_active"] is True + + def test_empty_registry_report(self): + report = deprecation_status_report([]) + assert report["total"] == 0 + assert report["all_active"] is True + + +# --------------------------------------------------------------------------- +# Group 5: PrintBanners (8 tests) +# --------------------------------------------------------------------------- + +class TestPrintBanners: + def test_print_banners_returns_list(self, capsys): + banners = print_all_deprecation_banners([_deprecated_card()]) + assert isinstance(banners, list) + + def test_print_banners_empty_for_active_only(self, capsys): + banners = print_all_deprecation_banners([_active_card()]) + assert banners == [] + + def test_print_banners_one_for_one_deprecated(self, capsys): + banners = print_all_deprecation_banners([_deprecated_card()]) + assert len(banners) == 1 + + def test_print_banners_two_for_two_deprecated(self, capsys): + banners = print_all_deprecation_banners([ + _deprecated_card("BMC-D1"), + _deprecated_card("BMC-D2"), + ]) + assert len(banners) == 2 + + def test_print_banners_outputs_to_stdout(self, capsys): + print_all_deprecation_banners([_deprecated_card()]) + captured = capsys.readouterr() + assert "DEPRECATED" in captured.out + + def test_print_banners_empty_registry(self, capsys): + banners = print_all_deprecation_banners([]) + assert banners == [] + + def test_print_banners_main_registry_no_output(self, capsys): + banners = print_all_deprecation_banners(BENCHMARK_REGISTRY) + assert banners == [] + + def test_banner_content_in_returned_list(self, capsys): + banners = print_all_deprecation_banners([_deprecated_card()]) + assert len(banners) == 1 + assert "DEPRECATED" in banners[0] + + +# --------------------------------------------------------------------------- +# Group 6: IntegrationWithRegistry (10 tests) +# --------------------------------------------------------------------------- + +class TestIntegrationWithRegistry: + def test_main_registry_no_deprecated(self): + assert len(get_deprecated_cards(BENCHMARK_REGISTRY)) == 0 + + def test_main_registry_all_active(self): + assert len(get_active_cards(BENCHMARK_REGISTRY)) == len(BENCHMARK_REGISTRY) + + def test_check_passes_for_main_registry(self): + check_no_deprecated_in_ranking(BENCHMARK_REGISTRY) + + def test_report_for_main_registry(self): + report = deprecation_status_report(BENCHMARK_REGISTRY) + assert report["deprecated"] == 0 + assert report["all_active"] is True + + def test_adding_deprecated_fails_check(self): + registry = list(BENCHMARK_REGISTRY) + [_deprecated_card()] + with pytest.raises(DeprecatedBenchmarkError): + check_no_deprecated_in_ranking(registry) + + def test_deprecated_card_not_in_active_cards(self): + registry = [_active_card(), _deprecated_card()] + active = get_active_cards(registry) + assert not any(c.deprecated for c in active) + + def test_deprecated_ids_list_is_list(self): + report = deprecation_status_report(BENCHMARK_REGISTRY) + assert isinstance(report["deprecated_ids"], list) + + def test_deprecated_ids_empty_for_main_registry(self): + report = deprecation_status_report(BENCHMARK_REGISTRY) + assert report["deprecated_ids"] == [] + + def test_mixed_registry_active_count_correct(self): + registry = [_active_card("BMC-A1"), _active_card("BMC-A2"), _deprecated_card()] + report = deprecation_status_report(registry) + assert report["active"] == 2 + assert report["deprecated"] == 1 + + def test_banner_for_card_without_notes_uses_fallback(self): + card = _deprecated_card() + card.notes = "" + banner = build_deprecation_banner(card) + assert "no reason given" in banner.lower() or "DEPRECATED" in banner + + +# --------------------------------------------------------------------------- +# Group 7: EdgeCases (9 tests) +# --------------------------------------------------------------------------- + +class TestEdgeCases: + def test_get_deprecated_empty_registry(self): + assert get_deprecated_cards([]) == [] + + def test_get_active_empty_registry(self): + assert get_active_cards([]) == [] + + def test_check_no_deprecated_empty_registry(self): + check_no_deprecated_in_ranking([]) # should not raise + + def test_deprecation_error_inherits_from_exception(self): + try: + raise DeprecatedBenchmarkError("test") + except Exception as e: + assert isinstance(e, DeprecatedBenchmarkError) + + def test_allow_deprecated_true_does_not_raise_even_with_deprecated(self): + registry = [_deprecated_card("BMC-D1"), _deprecated_card("BMC-D2")] + check_no_deprecated_in_ranking(registry, allow_deprecated=True) + + def test_report_deprecated_count_matches_get_deprecated(self): + registry = [_active_card(), _deprecated_card("BMC-D1"), _deprecated_card("BMC-D2")] + report = deprecation_status_report(registry) + assert report["deprecated"] == len(get_deprecated_cards(registry)) + + def test_report_active_count_matches_get_active(self): + registry = [_active_card(), _deprecated_card()] + report = deprecation_status_report(registry) + assert report["active"] == len(get_active_cards(registry)) + + def test_banner_width_reasonable(self): + banner = build_deprecation_banner(_deprecated_card()) + lines = banner.split("\n") + # Banner lines should not be unreasonably long + assert all(len(line) <= 120 for line in lines) + + def test_print_banners_returns_correct_banner_content(self, capsys): + card = _deprecated_card("BMC-DEPR-77") + banners = print_all_deprecation_banners([card]) + assert len(banners) == 1 + assert "BMC-DEPR-77" in banners[0] diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index e80a68a8..7c64d889 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -6,7 +6,7 @@ # Note: 3 pre-existing collection errors in test_benchmark_calibration.py, # test_benchmark_charge_distribution.py, test_benchmark_cheap_enemies.py. # Raise threshold if legitimate tests are removed; lower threshold is okay. -BASELINE = 6772 +BASELINE = 6835 TOLERANCE = 0.05 # allow 5% fluctuation