From a80a83a9b34f35555003107db640344c4bd8fcd9 Mon Sep 17 00:00:00 2001 From: OpenCode Date: Tue, 7 Jul 2026 10:32:50 +0700 Subject: [PATCH] Add metric naming consistency checker Closes #350 - Extends metric_polarity_registry.json with canonical_names and aliases - scripts/check_metric_naming.py -- checks metric names against registry - Reports aliases and unknown names with suggestions - 7 tests covering canonical, alias, unknown, CLI, registry integrity --- configs/metric_polarity_registry.json | 32 +++++++- scripts/check_metric_naming.py | 104 +++++++++++++++++++++++++ tests/test_check_metric_naming.py | 52 +++++++++++++ tests/test_metric_polarity_registry.py | 10 ++- 4 files changed, 195 insertions(+), 3 deletions(-) create mode 100644 scripts/check_metric_naming.py create mode 100644 tests/test_check_metric_naming.py diff --git a/configs/metric_polarity_registry.json b/configs/metric_polarity_registry.json index 3a158d61..46a69ae2 100644 --- a/configs/metric_polarity_registry.json +++ b/configs/metric_polarity_registry.json @@ -2,7 +2,38 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "title": "Metric Polarity Registry", "description": "Defines whether each metric improves when higher or lower.", + "canonical_names": { + "ensemble": "ensemble", + "activity": "activity", + "safety": "safety", + "synthesis": "synthesis", + "novelty": "novelty", + "rich_selectivity": "rich_selectivity", + "hemolysis_risk": "hemolysis_risk", + "charge_bias": "charge_bias", + "bacterial_binding": "bacterial_binding", + "selectivity_ratio": "selectivity_ratio", + "helix_weight": "helix_weight", + "non_helical": "non_helical", + "auroc": "auroc", + "auprc": "auprc", + "disagreement": "disagreement" + }, + "aliases": { + "roc_auc": "auroc", + "auc_roc": "auroc", + "pr_auc": "auprc", + "selectivity": "rich_selectivity", + "hemolysis": "hemolysis_risk", + "charge_density": "charge_bias", + "membrane_binding": "bacterial_binding" + }, "pipeline_metrics": { + "ensemble": {"polarity": "higher_is_better", "description": "Weighted ensemble score"}, + "activity": {"polarity": "higher_is_better", "description": "Activity-likeness score"}, + "safety": {"polarity": "higher_is_better", "description": "Safety score"}, + "synthesis": {"polarity": "higher_is_better", "description": "Synthesis feasibility score"}, + "novelty": {"polarity": "higher_is_better", "description": "Novelty score"}, "auroc": {"polarity": "higher_is_better", "description": "Area under ROC curve"}, "auprc": {"polarity": "higher_is_better", "description": "Area under precision-recall curve"}, "recall_at_10": {"polarity": "higher_is_better", "description": "Recall at top 10"}, @@ -21,7 +52,6 @@ "uncertainty": {"polarity": "lower_is_better", "description": "Simulation uncertainty score"} }, "safety_metrics": { - "safety": {"polarity": "higher_is_better", "description": "Safety score"}, "rich_selectivity": {"polarity": "higher_is_better", "description": "Selectivity composite"}, "hemolysis_risk": {"polarity": "lower_is_better", "description": "Predicted hemolysis risk"}, "toxicity_count": {"polarity": "lower_is_better", "description": "Number of toxic candidates"} diff --git a/scripts/check_metric_naming.py b/scripts/check_metric_naming.py new file mode 100644 index 00000000..5752d0fc --- /dev/null +++ b/scripts/check_metric_naming.py @@ -0,0 +1,104 @@ +"""Check metric names against the canonical registry.""" + +from __future__ import annotations + +import json +import re +import sys +from pathlib import Path + + +def load_registry(path: str = "configs/metric_polarity_registry.json") -> dict: + return json.loads(Path(path).read_text()) + + +def get_all_canonical(registry: dict) -> set[str]: + names = set() + for key in ["pipeline_metrics", "simulation_metrics", "safety_metrics"]: + names.update(registry.get(key, {}).keys()) + names.update(registry.get("canonical_names", {}).keys()) + return names + + +def get_aliases(registry: dict) -> dict[str, str]: + return registry.get("aliases", {}) + + +def check_metric_name(name: str, registry: dict) -> dict: + canonicals = get_all_canonical(registry) + aliases = get_aliases(registry) + + if name in canonicals: + return {"name": name, "status": "canonical", "canonical_name": name} + if name in aliases: + return {"name": name, "status": "alias", "canonical_name": aliases[name]} + return {"name": name, "status": "unknown", "canonical_name": None} + + +def scan_reports(docs_dir: str = "docs") -> list[dict]: + registry = load_registry() + canonicals = get_all_canonical(registry) + aliases = get_aliases(registry) + all_known = canonicals | set(aliases.keys()) + findings = [] + + pattern = re.compile(r"\b[a-z_]+(?:_at_\d+)?\b") + for md in Path(docs_dir).rglob("*.md"): + for match in pattern.finditer(md.read_text()): + word = match.group() + if word in all_known: + continue + # Skip common non-metric words + if word in {"docs", "make", "path", "file", "true", "false", "null", "none", "src", + "tests", "scripts", "configs", "outputs", "examples", "data", "models", + "schemas", "github", "html", "json", "yaml", "csv", "txt", "png"}: + continue + # Check if it looks like a metric name (snake_case with known endings) + if word.endswith(("_score", "_rate", "_count", "_auroc", "_auc", "_at", "_bias", + "_risk", "_weight", "_ratio", "_flag", "_binding", "_index")): + findings.append({ + "file": str(md), + "word": word, + "suggestion": aliases.get(word, "unknown — check spelling or add to registry"), + }) + + return findings + + +def check_names_in_file(file_path: str) -> dict: + registry = load_registry() + text = Path(file_path).read_text() + words = set(re.findall(r"\b[a-z_]+(?:_at_\d+)?\b", text)) + results = [] + for word in sorted(words): + result = check_metric_name(word, registry) + if result["status"] != "canonical": + results.append(result) + return {"file": file_path, "metrics_found": len(words), "non_canonical": results} + + +def main() -> int: + import argparse + parser = argparse.ArgumentParser(description="Check metric naming consistency") + parser.add_argument("--file", help="Check a specific file") + parser.add_argument("--scan-docs", action="store_true", help="Scan docs for unknown metric names") + args = parser.parse_args() + + if args.file: + result = check_names_in_file(args.file) + print(json.dumps(result, indent=2)) + return 3 if result["non_canonical"] else 0 + + if args.scan_docs: + findings = scan_reports() + print(f"Found {len(findings)} potentially unknown metric names in docs:") + for f in findings[:20]: + print(f" ❌ {f['file']}: '{f['word']}' → {f['suggestion']}") + return 3 if findings else 0 + + print("Specify --file or --scan-docs") + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_check_metric_naming.py b/tests/test_check_metric_naming.py new file mode 100644 index 00000000..06ed2f7f --- /dev/null +++ b/tests/test_check_metric_naming.py @@ -0,0 +1,52 @@ +"""Tests for metric naming consistency checker.""" +import json +import subprocess +import sys +from pathlib import Path +import pytest + +from scripts.check_metric_naming import check_metric_name, load_registry + + +@pytest.fixture +def registry(): + return load_registry() + + +def test_known_canonical(registry): + r = check_metric_name("auroc", registry) + assert r["status"] == "canonical" + + +def test_known_alias(registry): + r = check_metric_name("roc_auc", registry) + assert r["status"] == "alias" + assert r["canonical_name"] == "auroc" + + +def test_unknown_name(registry): + r = check_metric_name("nonexistent_metric", registry) + assert r["status"] == "unknown" + + +def test_cli_known(tmp_path): + f = tmp_path / "test.json" + f.write_text(json.dumps({"auroc": "ensemble"})) + r = subprocess.run([sys.executable, "scripts/check_metric_naming.py", "--file", str(f)], + capture_output=True, text=True, env={"PYTHONPATH": "src"}) + assert r.returncode == 0 + + +def test_registry_has_canonical_names(registry): + assert "canonical_names" in registry + assert len(registry["canonical_names"]) >= 5 + + +def test_registry_has_aliases(registry): + assert "aliases" in registry + assert len(registry["aliases"]) >= 3 + + +def test_aliases_resolve_to_canonical(registry): + for alias, canonical in registry["aliases"].items(): + assert canonical in registry["canonical_names"], f"{alias} -> {canonical} not in canonical_names" diff --git a/tests/test_metric_polarity_registry.py b/tests/test_metric_polarity_registry.py index c206e66f..775fcc4f 100644 --- a/tests/test_metric_polarity_registry.py +++ b/tests/test_metric_polarity_registry.py @@ -30,7 +30,10 @@ def test_disagreement_is_lower_better(): def test_all_metrics_have_description(): data = _registry() - for category in data.values(): + metric_cats = {"pipeline_metrics", "simulation_metrics", "safety_metrics"} + for key, category in data.items(): + if key not in metric_cats: + continue if isinstance(category, dict): for name, info in category.items(): assert "description" in info, f"{name} missing description" @@ -40,7 +43,10 @@ def test_all_metrics_have_description(): def test_polarity_values(): data = _registry() valid = {"higher_is_better", "lower_is_better", "depends"} - for category in data.values(): + metric_cats = {"pipeline_metrics", "simulation_metrics", "safety_metrics"} + for key, category in data.items(): + if key not in metric_cats: + continue if isinstance(category, dict): for name, info in category.items(): assert info["polarity"] in valid, f"{name} has invalid polarity: {info['polarity']}"