Add metric naming consistency checker

configs/metric_polarity_registry.json

@@ -2,7 +2,38 @@
   "$schema": "https://json-schema.org/draft/2020-12/schema",
   "title": "Metric Polarity Registry",
   "description": "Defines whether each metric improves when higher or lower.",
+  "canonical_names": {
+    "ensemble": "ensemble",
+    "activity": "activity",
+    "safety": "safety",
+    "synthesis": "synthesis",
+    "novelty": "novelty",
+    "rich_selectivity": "rich_selectivity",
+    "hemolysis_risk": "hemolysis_risk",
+    "charge_bias": "charge_bias",
+    "bacterial_binding": "bacterial_binding",
+    "selectivity_ratio": "selectivity_ratio",
+    "helix_weight": "helix_weight",
+    "non_helical": "non_helical",
+    "auroc": "auroc",
+    "auprc": "auprc",
+    "disagreement": "disagreement"
+  },
+  "aliases": {
+    "roc_auc": "auroc",
+    "auc_roc": "auroc",
+    "pr_auc": "auprc",
+    "selectivity": "rich_selectivity",
+    "hemolysis": "hemolysis_risk",
+    "charge_density": "charge_bias",
+    "membrane_binding": "bacterial_binding"
+  },
   "pipeline_metrics": {
+    "ensemble": {"polarity": "higher_is_better", "description": "Weighted ensemble score"},
+    "activity": {"polarity": "higher_is_better", "description": "Activity-likeness score"},
+    "safety": {"polarity": "higher_is_better", "description": "Safety score"},
+    "synthesis": {"polarity": "higher_is_better", "description": "Synthesis feasibility score"},
+    "novelty": {"polarity": "higher_is_better", "description": "Novelty score"},
     "auroc": {"polarity": "higher_is_better", "description": "Area under ROC curve"},
     "auprc": {"polarity": "higher_is_better", "description": "Area under precision-recall curve"},
     "recall_at_10": {"polarity": "higher_is_better", "description": "Recall at top 10"},
@@ -21,7 +52,6 @@
     "uncertainty": {"polarity": "lower_is_better", "description": "Simulation uncertainty score"}
   },
   "safety_metrics": {
-    "safety": {"polarity": "higher_is_better", "description": "Safety score"},
     "rich_selectivity": {"polarity": "higher_is_better", "description": "Selectivity composite"},
     "hemolysis_risk": {"polarity": "lower_is_better", "description": "Predicted hemolysis risk"},
     "toxicity_count": {"polarity": "lower_is_better", "description": "Number of toxic candidates"}

scripts/check_metric_naming.py

@@ -0,0 +1,104 @@
+"""Check metric names against the canonical registry."""
+
+from __future__ import annotations
+
+import json
+import re
+import sys
+from pathlib import Path
+
+
+def load_registry(path: str = "configs/metric_polarity_registry.json") -> dict:
+    return json.loads(Path(path).read_text())
+
+
+def get_all_canonical(registry: dict) -> set[str]:
+    names = set()
+    for key in ["pipeline_metrics", "simulation_metrics", "safety_metrics"]:
+        names.update(registry.get(key, {}).keys())
+    names.update(registry.get("canonical_names", {}).keys())
+    return names
+
+
+def get_aliases(registry: dict) -> dict[str, str]:
+    return registry.get("aliases", {})
+
+
+def check_metric_name(name: str, registry: dict) -> dict:
+    canonicals = get_all_canonical(registry)
+    aliases = get_aliases(registry)
+
+    if name in canonicals:
+        return {"name": name, "status": "canonical", "canonical_name": name}
+    if name in aliases:
+        return {"name": name, "status": "alias", "canonical_name": aliases[name]}
+    return {"name": name, "status": "unknown", "canonical_name": None}
+
+
+def scan_reports(docs_dir: str = "docs") -> list[dict]:
+    registry = load_registry()
+    canonicals = get_all_canonical(registry)
+    aliases = get_aliases(registry)
+    all_known = canonicals | set(aliases.keys())
+    findings = []
+
+    pattern = re.compile(r"\b[a-z_]+(?:_at_\d+)?\b")
+    for md in Path(docs_dir).rglob("*.md"):
+        for match in pattern.finditer(md.read_text()):
+            word = match.group()
+            if word in all_known:
+                continue
+            # Skip common non-metric words
+            if word in {"docs", "make", "path", "file", "true", "false", "null", "none", "src",
+                         "tests", "scripts", "configs", "outputs", "examples", "data", "models",
+                         "schemas", "github", "html", "json", "yaml", "csv", "txt", "png"}:
+                continue
+            # Check if it looks like a metric name (snake_case with known endings)
+            if word.endswith(("_score", "_rate", "_count", "_auroc", "_auc", "_at", "_bias",
+                              "_risk", "_weight", "_ratio", "_flag", "_binding", "_index")):
+                findings.append({
+                    "file": str(md),
+                    "word": word,
+                    "suggestion": aliases.get(word, "unknown — check spelling or add to registry"),
+                })
+
+    return findings
+
+
+def check_names_in_file(file_path: str) -> dict:
+    registry = load_registry()
+    text = Path(file_path).read_text()
+    words = set(re.findall(r"\b[a-z_]+(?:_at_\d+)?\b", text))
+    results = []
+    for word in sorted(words):
+        result = check_metric_name(word, registry)
+        if result["status"] != "canonical":
+            results.append(result)
+    return {"file": file_path, "metrics_found": len(words), "non_canonical": results}
+
+
+def main() -> int:
+    import argparse
+    parser = argparse.ArgumentParser(description="Check metric naming consistency")
+    parser.add_argument("--file", help="Check a specific file")
+    parser.add_argument("--scan-docs", action="store_true", help="Scan docs for unknown metric names")
+    args = parser.parse_args()
+
+    if args.file:
+        result = check_names_in_file(args.file)
+        print(json.dumps(result, indent=2))
+        return 3 if result["non_canonical"] else 0
+
+    if args.scan_docs:
+        findings = scan_reports()
+        print(f"Found {len(findings)} potentially unknown metric names in docs:")
+        for f in findings[:20]:
+            print(f"  ❌ {f['file']}: '{f['word']}' → {f['suggestion']}")
+        return 3 if findings else 0
+
+    print("Specify --file or --scan-docs")
+    return 2
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tests/test_check_metric_naming.py

@@ -0,0 +1,52 @@
+"""Tests for metric naming consistency checker."""
+import json
+import subprocess
+import sys
+from pathlib import Path
+import pytest
+
+from scripts.check_metric_naming import check_metric_name, load_registry
+
+
+@pytest.fixture
+def registry():
+    return load_registry()
+
+
+def test_known_canonical(registry):
+    r = check_metric_name("auroc", registry)
+    assert r["status"] == "canonical"
+
+
+def test_known_alias(registry):
+    r = check_metric_name("roc_auc", registry)
+    assert r["status"] == "alias"
+    assert r["canonical_name"] == "auroc"
+
+
+def test_unknown_name(registry):
+    r = check_metric_name("nonexistent_metric", registry)
+    assert r["status"] == "unknown"
+
+
+def test_cli_known(tmp_path):
+    f = tmp_path / "test.json"
+    f.write_text(json.dumps({"auroc": "ensemble"}))
+    r = subprocess.run([sys.executable, "scripts/check_metric_naming.py", "--file", str(f)],
+                       capture_output=True, text=True, env={"PYTHONPATH": "src"})
+    assert r.returncode == 0
+
+
+def test_registry_has_canonical_names(registry):
+    assert "canonical_names" in registry
+    assert len(registry["canonical_names"]) >= 5
+
+
+def test_registry_has_aliases(registry):
+    assert "aliases" in registry
+    assert len(registry["aliases"]) >= 3
+
+
+def test_aliases_resolve_to_canonical(registry):
+    for alias, canonical in registry["aliases"].items():
+        assert canonical in registry["canonical_names"], f"{alias} -> {canonical} not in canonical_names"

tests/test_metric_polarity_registry.py

@@ -30,7 +30,10 @@ def test_disagreement_is_lower_better():
 
 def test_all_metrics_have_description():
     data = _registry()
-    for category in data.values():
+    metric_cats = {"pipeline_metrics", "simulation_metrics", "safety_metrics"}
+    for key, category in data.items():
+        if key not in metric_cats:
+            continue
         if isinstance(category, dict):
             for name, info in category.items():
                 assert "description" in info, f"{name} missing description"
@@ -40,7 +43,10 @@ def test_all_metrics_have_description():
 def test_polarity_values():
     data = _registry()
     valid = {"higher_is_better", "lower_is_better", "depends"}
-    for category in data.values():
+    metric_cats = {"pipeline_metrics", "simulation_metrics", "safety_metrics"}
+    for key, category in data.items():
+        if key not in metric_cats:
+            continue
         if isinstance(category, dict):
             for name, info in category.items():
                 assert info["polarity"] in valid, f"{name} has invalid polarity: {info['polarity']}"