From 48ca0a7e5d72b9d3062001a68cdaad9bf70cdb11 Mon Sep 17 00:00:00 2001 From: OpenCode Date: Fri, 10 Jul 2026 10:05:26 +0700 Subject: [PATCH] feat: Phase C C2 benchmark card schema (BMC-) -- machine-checkable governance for every benchmark --- docs/evidence/BENCHMARK_CARD_GUIDE.md | 57 +++ docs/evidence/METRICS_CURRENT.md | 9 + docs/research/NEXT_100_PR_MAP.md | 2 +- .../evidence/benchmark_card.py | 209 +++++++++ tests/evidence/test_benchmark_card.py | 436 ++++++++++++++++++ tests/test_test_count_regression.py | 2 +- 6 files changed, 713 insertions(+), 2 deletions(-) create mode 100644 docs/evidence/BENCHMARK_CARD_GUIDE.md create mode 100644 src/openamp_foundry/evidence/benchmark_card.py create mode 100644 tests/evidence/test_benchmark_card.py diff --git a/docs/evidence/BENCHMARK_CARD_GUIDE.md b/docs/evidence/BENCHMARK_CARD_GUIDE.md new file mode 100644 index 00000000..e5e225af --- /dev/null +++ b/docs/evidence/BENCHMARK_CARD_GUIDE.md @@ -0,0 +1,57 @@ +# Benchmark Card Guide (BMC-) + +## Purpose + +A `BenchmarkCard` is the machine-readable documentation contract for every benchmark in the pipeline. Before a benchmark can have ranking authority over candidates, it must have a corresponding BMC- record that documents: + +1. What it measures (`measurement_target`) +2. How data is split (`split_strategy`) +3. What cheap baselines it must beat (`cheap_enemy_baselines`) +4. What evaluation metrics are used (`evaluation_metrics`) +5. What its known limitations are (`known_limitations`) + +This prevents benchmark sprawl and enforces anti-hype governance at the schema level. + +## Required Fields + +| Field | Type | Description | +|-------|------|-------------| +| `bmc_id` | str | Unique ID starting with `BMC-` | +| `pipeline_version` | str | Pipeline version when benchmark was defined | +| `benchmark_name` | str | Human-readable name | +| `measurement_target` | str | What's being measured (controlled vocab) | +| `split_strategy` | str | How data is split (controlled vocab) | +| `cheap_enemy_baselines` | List[str] | ≥1 cheap baseline this benchmark must beat | +| `evaluation_metrics` | List[str] | Evaluation metrics (controlled vocab) | +| `known_limitations` | List[str] | ≥1 documented limitation | +| `deprecated` | bool | Whether this benchmark is deprecated | +| `created_date` | str | ISO 8601 date | +| `last_updated_date` | str | ISO 8601 date | + +## Optional Fields + +| Field | Default | Description | +|-------|---------|-------------| +| `notes` | "" | Additional context (max 500 chars). Required when deprecated=True. | + +## Validation Rules + +1. `bmc_id` must start with `BMC-` +2. `pipeline_version` non-empty +3. `benchmark_name` non-empty +4. `measurement_target` in `VALID_MEASUREMENT_TARGETS` +5. `split_strategy` in `VALID_SPLIT_STRATEGIES` +6. `cheap_enemy_baselines` ≥1 entry, all non-empty strings +7. All entries in `cheap_enemy_baselines` non-empty +8. `evaluation_metrics` ≥1 entry, all in `VALID_EVALUATION_METRICS` +9. `known_limitations` ≥1 entry, all non-empty strings +10. `created_date` ISO 8601 format +11. `last_updated_date` ISO 8601 format +12. If `deprecated=True`, `notes` must explain why +13. `notes` ≤500 characters +14. `evaluation_metrics` entries in controlled vocab + +## Anti-Hype Constraints + +- A benchmark with no `cheap_enemy_baselines` cannot prove the method beats trivial rules. **Warnings** are emitted when only one cheap enemy is declared; two or more are preferred. +- Every benchmark has limitations. Documenting zero limitations is not allowed. diff --git a/docs/evidence/METRICS_CURRENT.md b/docs/evidence/METRICS_CURRENT.md index ca1ac2c9..ae78ebb8 100644 --- a/docs/evidence/METRICS_CURRENT.md +++ b/docs/evidence/METRICS_CURRENT.md @@ -9,6 +9,15 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak ## Changelog +### v0.10.33 — Phase C C2: Benchmark card schema (BMC-) +- Added `src/openamp_foundry/evidence/benchmark_card.py` — BenchmarkCard schema +- BMC- schema: 12 fields (bmc_id, pipeline_version, benchmark_name, measurement_target, split_strategy, cheap_enemy_baselines, evaluation_metrics, known_limitations, deprecated, created_date, last_updated_date, notes) +- 14 validation rules: BMC- prefix, controlled vocabs for measurement_target (10 values)/split_strategy (10 values)/evaluation_metrics (12 values), cheap_enemy_baselines ≥1 required, known_limitations ≥1 required, deprecated+notes dependency, notes ≤500 chars +- 2 warnings: <2 cheap enemies, <2 known limitations +- 63 tests in `tests/evidence/test_benchmark_card.py` +- BASELINE 6646→6709 +- Closes Phase C C2 — benchmark documentation is now machine-checkable; incomplete benchmark docs are blocked at schema level + ### v0.10.32 — Phase B B8: Certificates linked to run manifest hashes - Added `run_id` and `run_manifest_hash` optional parameters to `build_certificate()` in `src/openamp_foundry/evidence/certificate.py` - Every certificate now carries the pipeline run_id and the SHA256 hash of the run manifest that produced it diff --git a/docs/research/NEXT_100_PR_MAP.md b/docs/research/NEXT_100_PR_MAP.md index 4c8da557..be866eec 100644 --- a/docs/research/NEXT_100_PR_MAP.md +++ b/docs/research/NEXT_100_PR_MAP.md @@ -59,7 +59,7 @@ Make OpenAMP the hardest repo to fool in AMP dry-lab selection. | PR | Task | Why it matters | Review class | |---:|---|---|---| | C1 | Create machine-readable benchmark cards. | Governance becomes enforceable. | B | -| C2 | Add benchmark-card schema. | Prevents incomplete benchmark docs. | B | +| C2 | Add benchmark-card schema (complete). | BMC- schema: 12 fields, 14 validation rules, VALID_MEASUREMENT_TARGETS (10), VALID_SPLIT_STRATEGIES (10), VALID_EVALUATION_METRICS (12), cheap_enemy_baselines required (≥1, warns if <2), known_limitations required (≥1), deprecated+notes dependency; 63 tests. | B | | C3 | Add charge-distribution report for every benchmark. | Shortcut visibility. | B/C | | C4 | Add similarity-neighbor distribution report. | Novelty honesty. | B/C | | C5 | Add `bench-length-matched` control. | Tests length shortcut. | C | diff --git a/src/openamp_foundry/evidence/benchmark_card.py b/src/openamp_foundry/evidence/benchmark_card.py new file mode 100644 index 00000000..ae72f6b6 --- /dev/null +++ b/src/openamp_foundry/evidence/benchmark_card.py @@ -0,0 +1,209 @@ +"""BenchmarkCard schema — Phase C C2. + +Machine-readable documentation for each benchmark in the pipeline. +Prevents incomplete benchmark docs by requiring: measurement target, +split strategy, at least one cheap enemy baseline (must be beaten before +an advanced scorer gets ranking authority), at least one known limitation. + +ID prefix: BMC- +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from typing import List + +BMC_PREFIX = "BMC-" + +VALID_SPLIT_STRATEGIES = frozenset({ + "random_80_20", + "random_70_30", + "family_stratified", + "charge_stratified", + "length_stratified", + "temporal_split", + "scaffold_split", + "leakage_aware_split", + "cross_validation_5fold", + "cross_validation_10fold", +}) + +VALID_MEASUREMENT_TARGETS = frozenset({ + "antimicrobial_activity_prediction", + "hemolysis_risk_prediction", + "toxicity_prediction", + "novelty_scoring", + "synthesis_feasibility", + "diversity_selection", + "ensemble_ranking", + "safety_scoring", + "selectivity_prediction", + "calibration_accuracy", +}) + +VALID_EVALUATION_METRICS = frozenset({ + "precision_at_k", + "recall_at_k", + "auc_roc", + "auc_pr", + "mcc", + "f1", + "accuracy", + "spearman_rho", + "pearson_r", + "enrichment_factor", + "calibration_error", + "brier_score", +}) + +_MIN_CHEAP_ENEMIES = 1 +_MIN_KNOWN_LIMITATIONS = 1 +_NOTES_MAX_LENGTH = 500 +_ISO_DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}(T[\d:.+Z-]+)?$") + + +@dataclass +class BenchmarkCard: + bmc_id: str + pipeline_version: str + benchmark_name: str + measurement_target: str + split_strategy: str + cheap_enemy_baselines: List[str] + evaluation_metrics: List[str] + known_limitations: List[str] + deprecated: bool + created_date: str + last_updated_date: str + notes: str = "" + + def validate(self) -> list[str]: + errors: list[str] = [] + warnings: list[str] = [] + + # Rule 1: BMC- prefix + if not self.bmc_id.startswith(BMC_PREFIX): + errors.append(f"bmc_id must start with '{BMC_PREFIX}', got {self.bmc_id!r}") + + # Rule 2: pipeline_version non-empty + if not self.pipeline_version.strip(): + errors.append("pipeline_version must not be empty") + + # Rule 3: benchmark_name non-empty + if not self.benchmark_name.strip(): + errors.append("benchmark_name must not be empty") + + # Rule 4: measurement_target in controlled vocab + if self.measurement_target not in VALID_MEASUREMENT_TARGETS: + errors.append( + f"measurement_target must be one of {sorted(VALID_MEASUREMENT_TARGETS)}, " + f"got {self.measurement_target!r}" + ) + + # Rule 5: split_strategy in controlled vocab + if self.split_strategy not in VALID_SPLIT_STRATEGIES: + errors.append( + f"split_strategy must be one of {sorted(VALID_SPLIT_STRATEGIES)}, " + f"got {self.split_strategy!r}" + ) + + # Rule 6: cheap_enemy_baselines non-empty + if not self.cheap_enemy_baselines or len(self.cheap_enemy_baselines) < _MIN_CHEAP_ENEMIES: + errors.append( + f"cheap_enemy_baselines must have at least {_MIN_CHEAP_ENEMIES} entry — " + "a benchmark with no declared cheap enemies cannot prove the method beats " + "trivial baselines" + ) + + # Rule 7: all cheap enemy names non-empty strings + for i, enemy in enumerate(self.cheap_enemy_baselines): + if not isinstance(enemy, str) or not enemy.strip(): + errors.append(f"cheap_enemy_baselines[{i}] must be a non-empty string") + + # Rule 8: evaluation_metrics non-empty and in vocab + if not self.evaluation_metrics: + errors.append("evaluation_metrics must have at least one entry") + else: + for metric in self.evaluation_metrics: + if metric not in VALID_EVALUATION_METRICS: + errors.append( + f"evaluation_metrics entry {metric!r} must be one of " + f"{sorted(VALID_EVALUATION_METRICS)}" + ) + + # Rule 9: known_limitations non-empty + if not self.known_limitations or len(self.known_limitations) < _MIN_KNOWN_LIMITATIONS: + errors.append( + f"known_limitations must have at least {_MIN_KNOWN_LIMITATIONS} entry — " + "every benchmark has limitations; document them or do not ship the benchmark" + ) + + # Rule 10: all known_limitations non-empty strings + for i, lim in enumerate(self.known_limitations): + if not isinstance(lim, str) or not lim.strip(): + errors.append(f"known_limitations[{i}] must be a non-empty string") + + # Rule 11: created_date ISO format + if not _ISO_DATE_RE.match(self.created_date): + errors.append( + f"created_date must be ISO 8601 date, got {self.created_date!r}" + ) + + # Rule 12: last_updated_date ISO format + if not _ISO_DATE_RE.match(self.last_updated_date): + errors.append( + f"last_updated_date must be ISO 8601 date, got {self.last_updated_date!r}" + ) + + # Rule 13: if deprecated, notes must explain why + if self.deprecated and not self.notes.strip(): + errors.append( + "notes must explain why the benchmark is deprecated when deprecated=True" + ) + + # Rule 14: notes length cap + if len(self.notes) > _NOTES_MAX_LENGTH: + errors.append( + f"notes must be ≤{_NOTES_MAX_LENGTH} characters, got {len(self.notes)}" + ) + + # Warning: only one cheap enemy (weak — two or more preferred) + if len(self.cheap_enemy_baselines) < 2: + warnings.append( + "cheap_enemy_baselines has only one entry; two or more cheap enemies " + "provide stronger anti-hype evidence" + ) + + # Warning: only one known limitation (probably incomplete) + if len(self.known_limitations) < 2: + warnings.append( + "known_limitations has only one entry; document at least two limitations " + "for a complete benchmark card" + ) + + if errors: + return errors + return [f"WARNING: {w}" for w in warnings] if warnings else [] + + +def validate_dict(data: dict) -> list[str]: + """Validate a dict representation of BenchmarkCard.""" + try: + record = BenchmarkCard( + bmc_id=data.get("bmc_id", ""), + pipeline_version=data.get("pipeline_version", ""), + benchmark_name=data.get("benchmark_name", ""), + measurement_target=data.get("measurement_target", ""), + split_strategy=data.get("split_strategy", ""), + cheap_enemy_baselines=data.get("cheap_enemy_baselines", []), + evaluation_metrics=data.get("evaluation_metrics", []), + known_limitations=data.get("known_limitations", []), + deprecated=data.get("deprecated", False), + created_date=data.get("created_date", ""), + last_updated_date=data.get("last_updated_date", ""), + notes=data.get("notes", ""), + ) + return record.validate() + except Exception as exc: + return [f"Construction error: {exc}"] diff --git a/tests/evidence/test_benchmark_card.py b/tests/evidence/test_benchmark_card.py new file mode 100644 index 00000000..dd7d3a74 --- /dev/null +++ b/tests/evidence/test_benchmark_card.py @@ -0,0 +1,436 @@ +"""Tests for BenchmarkCard schema — Phase C C2. + +63 tests across 7 groups. +""" + +from __future__ import annotations + +import pytest + +from openamp_foundry.evidence.benchmark_card import ( + BMC_PREFIX, + VALID_EVALUATION_METRICS, + VALID_MEASUREMENT_TARGETS, + VALID_SPLIT_STRATEGIES, + BenchmarkCard, + validate_dict, +) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _valid() -> BenchmarkCard: + return BenchmarkCard( + bmc_id="BMC-0001", + pipeline_version="0.10.32", + benchmark_name="AMP Activity Prediction Benchmark v1", + measurement_target="antimicrobial_activity_prediction", + split_strategy="leakage_aware_split", + cheap_enemy_baselines=[ + "charge_threshold_gte4", + "length_10_to_40_heuristic", + ], + evaluation_metrics=["precision_at_k", "auc_roc"], + known_limitations=[ + "Trained on APD3; may overfit to alpha-helical AMPs.", + "Does not account for post-translational modifications.", + ], + deprecated=False, + created_date="2024-06-01", + last_updated_date="2024-06-01", + notes="Validated against DRAMP3 holdout set.", + ) + + +def _valid_dict() -> dict: + r = _valid() + return { + "bmc_id": r.bmc_id, + "pipeline_version": r.pipeline_version, + "benchmark_name": r.benchmark_name, + "measurement_target": r.measurement_target, + "split_strategy": r.split_strategy, + "cheap_enemy_baselines": r.cheap_enemy_baselines, + "evaluation_metrics": r.evaluation_metrics, + "known_limitations": r.known_limitations, + "deprecated": r.deprecated, + "created_date": r.created_date, + "last_updated_date": r.last_updated_date, + "notes": r.notes, + } + + +# --------------------------------------------------------------------------- +# Group 1: ValidRecord (8 tests) +# --------------------------------------------------------------------------- + +class TestValidRecord: + def test_valid_record_passes(self): + assert _valid().validate() == [] + + def test_valid_dict_passes(self): + assert validate_dict(_valid_dict()) == [] + + def test_bmc_prefix_constant(self): + assert BMC_PREFIX == "BMC-" + + def test_valid_split_strategies_non_empty(self): + assert len(VALID_SPLIT_STRATEGIES) >= 8 + + def test_valid_measurement_targets_non_empty(self): + assert len(VALID_MEASUREMENT_TARGETS) >= 8 + + def test_valid_evaluation_metrics_non_empty(self): + assert len(VALID_EVALUATION_METRICS) >= 8 + + def test_deprecated_false_no_notes_needed(self): + r = _valid() + r.notes = "" + result = r.validate() + # warnings only, no errors + assert all("WARNING" in msg for msg in result) + + def test_valid_record_with_multiple_metrics(self): + r = _valid() + r.evaluation_metrics = ["precision_at_k", "auc_roc", "mcc"] + assert r.validate() == [] + + +# --------------------------------------------------------------------------- +# Group 2: IDAndVersionRules (8 tests) +# --------------------------------------------------------------------------- + +class TestIDAndVersionRules: + def test_wrong_prefix_fails(self): + r = _valid() + r.bmc_id = "WRONG-001" + assert any("BMC-" in e for e in r.validate()) + + def test_empty_bmc_id_fails(self): + r = _valid() + r.bmc_id = "" + assert any("bmc_id" in e for e in r.validate()) + + def test_lowercase_prefix_fails(self): + r = _valid() + r.bmc_id = "bmc-001" + assert any("BMC-" in e for e in r.validate()) + + def test_empty_pipeline_version_fails(self): + r = _valid() + r.pipeline_version = "" + assert any("pipeline_version" in e for e in r.validate()) + + def test_empty_benchmark_name_fails(self): + r = _valid() + r.benchmark_name = "" + assert any("benchmark_name" in e for e in r.validate()) + + def test_whitespace_benchmark_name_fails(self): + r = _valid() + r.benchmark_name = " " + assert any("benchmark_name" in e for e in r.validate()) + + def test_bmc_prefix_with_suffix_passes(self): + r = _valid() + r.bmc_id = "BMC-9999" + assert r.validate() == [] + + def test_validate_dict_wrong_prefix(self): + d = _valid_dict() + d["bmc_id"] = "WRONG-001" + assert any("BMC-" in e for e in validate_dict(d)) + + +# --------------------------------------------------------------------------- +# Group 3: MeasurementTargetAndSplitRules (10 tests) +# --------------------------------------------------------------------------- + +class TestMeasurementTargetAndSplitRules: + def test_invalid_measurement_target_fails(self): + r = _valid() + r.measurement_target = "made_up_target" + assert any("measurement_target" in e for e in r.validate()) + + def test_all_valid_measurement_targets_pass(self): + for target in VALID_MEASUREMENT_TARGETS: + r = _valid() + r.measurement_target = target + errors = [e for e in r.validate() if "measurement_target" in e] + assert errors == [], f"Target {target!r} should be valid" + + def test_empty_measurement_target_fails(self): + r = _valid() + r.measurement_target = "" + assert any("measurement_target" in e for e in r.validate()) + + def test_invalid_split_strategy_fails(self): + r = _valid() + r.split_strategy = "random_split_99" + assert any("split_strategy" in e for e in r.validate()) + + def test_all_valid_split_strategies_pass(self): + for strategy in VALID_SPLIT_STRATEGIES: + r = _valid() + r.split_strategy = strategy + errors = [e for e in r.validate() if "split_strategy" in e] + assert errors == [], f"Strategy {strategy!r} should be valid" + + def test_empty_split_strategy_fails(self): + r = _valid() + r.split_strategy = "" + assert any("split_strategy" in e for e in r.validate()) + + def test_leakage_aware_split_passes(self): + r = _valid() + r.split_strategy = "leakage_aware_split" + errors = [e for e in r.validate() if "split_strategy" in e] + assert errors == [] + + def test_family_stratified_passes(self): + r = _valid() + r.split_strategy = "family_stratified" + errors = [e for e in r.validate() if "split_strategy" in e] + assert errors == [] + + def test_antimicrobial_activity_prediction_target_passes(self): + r = _valid() + r.measurement_target = "antimicrobial_activity_prediction" + errors = [e for e in r.validate() if "measurement_target" in e] + assert errors == [] + + def test_calibration_accuracy_target_passes(self): + r = _valid() + r.measurement_target = "calibration_accuracy" + errors = [e for e in r.validate() if "measurement_target" in e] + assert errors == [] + + +# --------------------------------------------------------------------------- +# Group 4: CheapEnemyRules (10 tests) +# --------------------------------------------------------------------------- + +class TestCheapEnemyRules: + def test_empty_cheap_enemies_fails(self): + r = _valid() + r.cheap_enemy_baselines = [] + assert any("cheap_enemy_baselines" in e for e in r.validate()) + + def test_one_cheap_enemy_passes_with_warning(self): + r = _valid() + r.cheap_enemy_baselines = ["charge_threshold_gte4"] + result = r.validate() + assert all("WARNING" in msg for msg in result) + + def test_one_cheap_enemy_warning_mentions_two(self): + r = _valid() + r.cheap_enemy_baselines = ["charge_threshold_gte4"] + result = r.validate() + assert any("WARNING" in msg and "cheap_enemy" in msg for msg in result) + + def test_two_cheap_enemies_no_warning(self): + r = _valid() + r.cheap_enemy_baselines = ["charge_threshold", "length_heuristic"] + result = r.validate() + assert not any("cheap_enemy" in msg for msg in result) + + def test_empty_string_enemy_fails(self): + r = _valid() + r.cheap_enemy_baselines = [""] + assert any("cheap_enemy_baselines" in e for e in r.validate()) + + def test_whitespace_enemy_fails(self): + r = _valid() + r.cheap_enemy_baselines = [" "] + assert any("cheap_enemy_baselines" in e for e in r.validate()) + + def test_multiple_cheap_enemies_pass(self): + r = _valid() + r.cheap_enemy_baselines = ["enemy1", "enemy2", "enemy3"] + errors = [e for e in r.validate() if "cheap_enemy" in e and "WARNING" not in e] + assert errors == [] + + def test_cheap_enemy_names_are_descriptive(self): + r = _valid() + for enemy in r.cheap_enemy_baselines: + assert len(enemy.strip()) > 0 + + def test_dict_empty_cheap_enemies_fails(self): + d = _valid_dict() + d["cheap_enemy_baselines"] = [] + assert any("cheap_enemy" in e for e in validate_dict(d)) + + def test_non_string_enemy_fails(self): + r = _valid() + r.cheap_enemy_baselines = [123] # type: ignore + errors = r.validate() + # 123 is not a string with strip(), should error + assert len(errors) > 0 + + +# --------------------------------------------------------------------------- +# Group 5: EvalMetricsAndLimitations (10 tests) +# --------------------------------------------------------------------------- + +class TestEvalMetricsAndLimitations: + def test_empty_evaluation_metrics_fails(self): + r = _valid() + r.evaluation_metrics = [] + assert any("evaluation_metrics" in e for e in r.validate()) + + def test_invalid_metric_fails(self): + r = _valid() + r.evaluation_metrics = ["made_up_metric"] + assert any("evaluation_metrics" in e for e in r.validate()) + + def test_all_valid_metrics_pass(self): + for metric in VALID_EVALUATION_METRICS: + r = _valid() + r.evaluation_metrics = [metric] + errors = [e for e in r.validate() if "evaluation_metrics" in e and "WARNING" not in e] + assert errors == [], f"Metric {metric!r} should be valid" + + def test_empty_known_limitations_fails(self): + r = _valid() + r.known_limitations = [] + assert any("known_limitations" in e for e in r.validate()) + + def test_one_known_limitation_warning(self): + r = _valid() + r.known_limitations = ["Only tested on alpha-helical AMPs."] + result = r.validate() + # May have warning about only one limitation + assert all("WARNING" in msg for msg in result) or result == [] + + def test_two_known_limitations_no_warning(self): + r = _valid() + r.known_limitations = [ + "Only tested on alpha-helical AMPs.", + "No temporal split applied.", + ] + result = r.validate() + assert not any("known_limitations" in msg for msg in result) + + def test_empty_limitation_string_fails(self): + r = _valid() + r.known_limitations = [""] + assert any("known_limitations" in e for e in r.validate()) + + def test_precision_at_k_metric_passes(self): + r = _valid() + r.evaluation_metrics = ["precision_at_k"] + errors = [e for e in r.validate() if "evaluation_metrics" in e and "WARNING" not in e] + assert errors == [] + + def test_auc_roc_metric_passes(self): + r = _valid() + r.evaluation_metrics = ["auc_roc"] + errors = [e for e in r.validate() if "evaluation_metrics" in e and "WARNING" not in e] + assert errors == [] + + def test_calibration_error_metric_passes(self): + r = _valid() + r.evaluation_metrics = ["calibration_error"] + errors = [e for e in r.validate() if "evaluation_metrics" in e and "WARNING" not in e] + assert errors == [] + + +# --------------------------------------------------------------------------- +# Group 6: DeprecatedAndDates (9 tests) +# --------------------------------------------------------------------------- + +class TestDeprecatedAndDates: + def test_deprecated_true_requires_notes(self): + r = _valid() + r.deprecated = True + r.notes = "" + assert any("deprecated" in e for e in r.validate()) + + def test_deprecated_true_with_notes_passes(self): + r = _valid() + r.deprecated = True + r.notes = "Superseded by BMC-0002 which uses leakage-aware split." + assert r.validate() == [] + + def test_deprecated_false_empty_notes_only_warning(self): + r = _valid() + r.deprecated = False + r.notes = "" + result = r.validate() + assert all("WARNING" in msg for msg in result) + + def test_invalid_created_date_fails(self): + r = _valid() + r.created_date = "June 2024" + assert any("created_date" in e for e in r.validate()) + + def test_valid_created_date_passes(self): + r = _valid() + r.created_date = "2024-01-15" + assert r.validate() == [] + + def test_invalid_last_updated_date_fails(self): + r = _valid() + r.last_updated_date = "not-a-date" + assert any("last_updated_date" in e for e in r.validate()) + + def test_notes_exceeds_limit_fails(self): + r = _valid() + r.notes = "x" * 501 + assert any("notes" in e for e in r.validate()) + + def test_notes_exactly_at_limit_passes(self): + r = _valid() + r.notes = "x" * 500 + errors = [e for e in r.validate() if "notes" in e and "WARNING" not in e] + assert errors == [] + + def test_datetime_format_date_passes(self): + r = _valid() + r.created_date = "2024-06-01T12:00:00Z" + assert r.validate() == [] + + +# --------------------------------------------------------------------------- +# Group 7: ValidateDictEdgeCases (8 tests) +# --------------------------------------------------------------------------- + +class TestValidateDictEdgeCases: + def test_empty_dict_fails(self): + assert len(validate_dict({})) > 0 + + def test_valid_dict_passes(self): + assert validate_dict(_valid_dict()) == [] + + def test_missing_bmc_id_fails(self): + d = _valid_dict() + del d["bmc_id"] + assert any("bmc_id" in e for e in validate_dict(d)) + + def test_missing_cheap_enemies_fails(self): + d = _valid_dict() + del d["cheap_enemy_baselines"] + assert any("cheap_enemy" in e for e in validate_dict(d)) + + def test_missing_known_limitations_fails(self): + d = _valid_dict() + del d["known_limitations"] + assert any("known_limitations" in e for e in validate_dict(d)) + + def test_missing_deprecated_defaults_false(self): + d = _valid_dict() + del d["deprecated"] + # deprecated defaults False → no error for notes + result = validate_dict(d) + assert all("WARNING" in msg for msg in result) or result == [] + + def test_construction_error_returns_error_list(self): + result = validate_dict({"bmc_id": None}) + assert len(result) > 0 + + def test_dict_with_extra_fields_passes(self): + d = _valid_dict() + d["extra_field"] = "extra_value" + assert validate_dict(d) == [] diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index 47afd161..06c800ae 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -6,7 +6,7 @@ # Note: 3 pre-existing collection errors in test_benchmark_calibration.py, # test_benchmark_charge_distribution.py, test_benchmark_cheap_enemies.py. # Raise threshold if legitimate tests are removed; lower threshold is okay. -BASELINE = 6646 +BASELINE = 6709 TOLERANCE = 0.05 # allow 5% fluctuation