feat: Phase G G4 active-learning strategy comparison report

Makefile

@@ -1,4 +1,4 @@
-.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example
+.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example
 
 PYTHON := $(shell [ -f .venv/bin/python ] && echo .venv/bin/python || echo python3)
 PYTEST  := $(shell [ -f .venv/bin/pytest ] && echo .venv/bin/pytest || echo pytest)
@@ -55,6 +55,7 @@ help:
 	@echo "  make bench-simulation-ablation  Simulation ablation on AMP-vs-decoy benchmark"
 	@echo "  make bench-simulation-ablation-within-amp  Simulation ablation on hemolytic vs selective AMPs"
 	@echo "  make simulation-gate        Fail-closed gate for weighted simulation mode"
+	@echo "  make bench-strategy-compare Compare active-learning strategies (exploit/explore/diverse/combined/random)"
 	@echo "  make bench-gate               Benchmark regression gate (AUROC drift check)"
 	@echo "  make regenerate-all           Run all pipeline + benchmarks and verify determinism"
 	@echo "  make bench-hidden-active      Hidden-positive recovery on mixed benchmark set"
@@ -584,6 +585,11 @@ simulation-gate:
 		--within-amp-json outputs/simulation_ablation_within_amp.json \
 		--out outputs/simulation_gate_verdict.json
 
+bench-strategy-compare:
+	PYTHONPATH=src $(PYTHON) -m openamp_foundry.cli bench strategy-compare \
+		--out-json outputs/strategy_comparison.json \
+		--out-md outputs/strategy_comparison.md
+
 bench-cross-dataset:
 	PYTHONPATH=src $(PYTHON) scripts/benchmarks/benchmark_cross_dataset.py \
 		--out outputs/cross_dataset_benchmark.json

docs/evidence/METRICS_CURRENT.md

@@ -5,7 +5,8 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak
 > **Purpose:** One authoritative table of current pipeline metrics. If any doc disagrees
 > with this file, this file wins. Updated whenever benchmark/benchmark config changes.
 >
-> **Last updated:** 2026-07-09 (Phase G G3 — calibration pipeline consistency audit — v0.5.81)
+> **Last updated:** 2026-07-09 (Phase G G4 — active-learning strategy comparison report — v0.5.82)
+> **New in v0.5.82:** Active-learning strategy comparison report (G4) — CLI (`openamp-foundry bench strategy-compare`) that compares 5 selection strategies (exploitation, exploration, diversity, combined, random) on the same synthetic pool with identical hidden active candidates. Each strategy runs multi-round recovery of hidden actives; the report ranks strategies by recall, compares the production selector vs pure strategies and random baseline, and produces structured JSON + Markdown output with caveats. Prevents one-selector bias by making strategy performance transparent. Schema (`schemas/active_learning_strategy_comparison.schema.json`). `make bench-strategy-compare` target. 18 tests. 2955 total.
 > **New in v0.5.81:** Calibration pipeline consistency audit (G3) — CLI (`openamp-foundry calibration-audit`) that checks consistency across calibration pipeline artifacts (intake report, gate verdict, engine proposal, recalibration report). Checks intake↔gate count matching, engine↔gate verdict agreement, engine L1 budget compliance, report↔gate verdict consistency, report↔engine proposal consistency, timestamp sanity, and cohort-metrics warnings. Schema (`schemas/calibration_audit.schema.json`). `make calibration-audit` and `make calibration-audit-example` targets. 18 tests. 2937 total.
 > **New in v0.5.80:** Negative-result archive completeness checker (F10) — CLI (`scripts/check_negative_archive_completeness.py`) that reads a JSON archive of negative-result entries and checks each entry against completeness criteria: all required fields present, no duplicate candidate_ids, each entry has at least one content field (assay_result, score_safety, reviewer_notes, or reason_detail), date format valid YYYY-MM-DD, and optional intake_report_id references are well-formed. Outputs a structured completeness report as JSON and Markdown with summary, per-check results, and per-entry pass/fail. Schema (`schemas/negative_result_archive_completeness.schema.json`). Example file (`examples/negative_result_archive_example.json`) with 4 toy entries. 35 tests. 2919 total.
 >
@@ -73,7 +74,7 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak
 > **New in v0.5.31:** Added dipeptide-order features for sequence-order awareness. `dipeptide_order_score` achieves AUROC 0.7861 on AMP-vs-scrambled discrimination — the strongest order-dependent feature in the pipeline. Only 7/31 features survive scrambling (amphipathicity/helix-wheel + dipeptide). All composition features are purely position-independent (exactly 0.5000 AUROC on scrambled test).
 > **New in v0.5.30:** Easy baseline benchmark added — charge density alone (AUROC 0.8166) outperforms the full pipeline ensemble (0.7792) on AMP-vs-Swiss-Prot-decoy discrimination. Honest finding documented: expected because pipeline optimizes for safety, not raw discrimination.
 > **New in v0.5.29:** Expanded benchmark to 500 AMPs + 500 composition-matched decoys (n=1000). AUROC 0.7792 (CI₉₅: 0.7505–0.8065) confirms signal generalizes. Cluster-aware CI: 0.746–0.8102. Representative AUROC: 0.778. Standard benchmark (n=191) retained for backward comparison.
-> **Pipeline version:** v0.5.81 (G3)
+> **Pipeline version:** v0.5.82 (G4)
 > **Branch:** main
 
 ---

docs/research/ROADMAP.md

@@ -1,5 +1,61 @@
 # Roadmap
 
+## v0.5.82 — Loop 82: Phase G G4 — Active-Learning Strategy Comparison Report ✓ (2026-07-09)
+
+CLI (`openamp-foundry bench strategy-compare`) that compares 5 selection strategies
+(exploitation, exploration, diversity, combined, random) on the same synthetic pool
+with identical hidden active candidates. Each strategy runs multi-round recovery of
+hidden actives using the same batch-2 selector with different weights. The report
+ranks strategies by recall, compares the production selector vs pure strategies and
+random baseline, and produces structured JSON + Markdown output with caveats.
+Prevents one-selector bias by making strategy performance transparent.
+
+Changes:
+- `src/openamp_foundry/active_learning/strategy_comparison.py` (G4) —
+  `run_strategy_comparison()` generates a synthetic pool, hides N active candidates,
+  runs 5 strategies (exploitation, exploration, diversity, combined, random) via
+  multi-round selection, and produces a `StrategyComparisonReport` with per-strategy
+  recovery metrics, ranking by recall, best strategy identification, and production
+  selector comparison (vs random, exploitation, exploration, diversity).
+  `STRATEGY_WEIGHTS` dict defines the weight tuples for each strategy.
+  `write_comparison_json()` and `write_comparison_markdown()` produce structured output.
+- `schemas/active_learning_strategy_comparison.schema.json` (G4) — JSON Schema
+  Draft 2020-12 for the strategy comparison report. Validates all required fields
+  including per-strategy results, ranking, production comparisons, and notes.
+- `src/openamp_foundry/active_learning/__init__.py` — Exports `STRATEGY_WEIGHTS`,
+  `StrategyComparisonReport`, `StrategyResult`, `run_strategy_comparison`,
+  `write_comparison_json`, `write_comparison_markdown`.
+- `src/openamp_foundry/cli/commands/benchmark.py` — Added
+  `_run_active_learning_strategy_compare()` CLI handler with `--n-total`, `--n-active`,
+  `--n-hidden`, `--batch-size`, `--max-rounds`, `--rng-seed`, `--out-json`, `--out-md` flags.
+- `src/openamp_foundry/cli/main.py` — Registered `strategy-compare` subcommand under
+  `bench` with all argument flags and dispatch to handler.
+- `Makefile` — Added `bench-strategy-compare` target with default params writing to
+  `outputs/strategy_comparison.json` and `outputs/strategy_comparison.md`.
+- `tests/active_learning/test_strategy_comparison.py` — 18 tests covering: all 5
+  strategies present, required fields (top-level + per-result), valid recall range,
+  ranking order, production strategy name, best strategy not null, exploitation
+  recovers actives, random baseline notes, production vs comparison fields, notes
+  presence, JSON and Markdown output writing, CLI exit 0, CLI writes files, random
+  baseline ranges, JSON Schema conformance, and production_outperforms_random type.
+- `docs/evidence/METRICS_CURRENT.md` — v0.5.82 G4 changelog. Test count: 2955.
+- `tests/test_test_count_regression.py` — baseline updated to 2955.
+
+Honest boundaries:
+- This report uses **synthetic data** with known active/inactive labels. Results
+  reflect code-path integrity, not biological performance.
+- All strategies run with safety/selectivity gates disabled to isolate strategy
+  effects. Real performance may differ with gates enabled.
+- Random baseline is averaged over 20 Monte Carlo trials; deterministic strategies
+  use a single run per config.
+- The production selector optimizes for multiple objectives (activity, safety,
+  diversity) that this recall-based benchmark does not fully measure.
+- A strategy that ranks highest on recall may not be the best choice for real
+  candidate selection — domain-specific constraints, safety requirements, and
+  material constraints matter.
+- This comparison is informational and requires qualified human review before
+  influencing selection decisions.
+
 ## v0.5.81 — Loop 81: Phase G G3 — Calibration Pipeline Consistency Audit ✓ (2026-07-09)
 
 CLI (`openamp-foundry calibration-audit`) that checks consistency across the

schemas/active_learning_strategy_comparison.schema.json

@@ -0,0 +1,129 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://openamp.ai/schemas/active_learning_strategy_comparison.schema.json",
+  "title": "Active-Learning Strategy Comparison Report",
+  "description": "Report comparing multiple active-learning selection strategies on a synthetic pool. All fields are computational results — not biological evidence.",
+  "type": "object",
+  "required": [
+    "version",
+    "pool_size",
+    "n_active_total",
+    "n_hidden_actives",
+    "batch_size",
+    "max_rounds",
+    "results",
+    "ranked_by_recall",
+    "best_strategy",
+    "production_strategy",
+    "production_outperforms_random",
+    "notes"
+  ],
+  "properties": {
+    "version": {
+      "type": "string",
+      "description": "Schema/report version."
+    },
+    "pool_size": {
+      "type": "integer",
+      "minimum": 1,
+      "description": "Total number of candidates in the synthetic pool."
+    },
+    "n_active_total": {
+      "type": "integer",
+      "minimum": 0,
+      "description": "Total number of active (label=1) candidates."
+    },
+    "n_hidden_actives": {
+      "type": "integer",
+      "minimum": 1,
+      "description": "Number of active candidates hidden during selection."
+    },
+    "batch_size": {
+      "type": "integer",
+      "minimum": 1,
+      "description": "Number of candidates selected per round."
+    },
+    "max_rounds": {
+      "type": "integer",
+      "minimum": 1,
+      "description": "Maximum number of selection rounds."
+    },
+    "results": {
+      "type": "array",
+      "description": "Per-strategy recovery results.",
+      "items": {
+        "type": "object",
+        "required": [
+          "strategy_name",
+          "rounds_to_first_recovery",
+          "final_recall",
+          "n_hidden_recovered",
+          "n_hidden_actives",
+          "total_rounds_run",
+          "notes"
+        ],
+        "properties": {
+          "strategy_name": { "type": "string" },
+          "rounds_to_first_recovery": {
+            "oneOf": [
+              { "type": "number", "minimum": 1 },
+              { "type": "null" }
+            ]
+          },
+          "final_recall": { "type": "number", "minimum": 0.0, "maximum": 1.0 },
+          "n_hidden_recovered": { "type": "integer", "minimum": 0 },
+          "n_hidden_actives": { "type": "integer", "minimum": 1 },
+          "total_rounds_run": { "type": "integer", "minimum": 0 },
+          "notes": { "type": "array", "items": { "type": "string" } }
+        }
+      }
+    },
+    "ranked_by_recall": {
+      "type": "array",
+      "description": "Strategy names ranked by final recall descending.",
+      "items": { "type": "string" }
+    },
+    "best_strategy": {
+      "oneOf": [
+        { "type": "string" },
+        { "type": "null" }
+      ],
+      "description": "Name of the strategy with highest recall."
+    },
+    "production_strategy": {
+      "type": "string",
+      "description": "Name of the production selector strategy."
+    },
+    "production_outperforms_random": {
+      "oneOf": [
+        { "type": "boolean" },
+        { "type": "null" }
+      ],
+      "description": "Whether the production selector outperforms random baseline on recall."
+    },
+    "production_vs_exploitation": {
+      "oneOf": [
+        { "type": "string", "enum": ["better", "worse", "similar"] },
+        { "type": "null" }
+      ]
+    },
+    "production_vs_exploration": {
+      "oneOf": [
+        { "type": "string", "enum": ["better", "worse", "similar"] },
+        { "type": "null" }
+      ]
+    },
+    "production_vs_diversity": {
+      "oneOf": [
+        { "type": "string", "enum": ["better", "worse", "similar"] },
+        { "type": "null" }
+      ]
+    },
+    "notes": {
+      "type": "array",
+      "description": "Human-readable notes and caveats.",
+      "items": { "type": "string" }
+    }
+  },
+  "_caveat": "This report is a computational comparison of selection strategies on synthetic data. It is not biological evidence of sampling efficiency, candidate quality, or real-world selection performance."
+}

src/openamp_foundry/active_learning/__init__.py

@@ -12,6 +12,10 @@
 
 The benchmark module validates whether the selector can recover known active
 candidates faster than random baseline — a code-path integrity check.
+
+The strategy-comparison report runs multiple selection strategies (exploitation,
+exploration, diversity, combined, random) on the same synthetic pool and ranks
+them by recovery recall, preventing one-selector bias.
 """
 
 from openamp_foundry.active_learning.benchmark import (
@@ -25,13 +29,27 @@
     BatchSelection,
     select_batch_2,
 )
+from openamp_foundry.active_learning.strategy_comparison import (
+    STRATEGY_WEIGHTS,
+    StrategyComparisonReport,
+    StrategyResult,
+    run_strategy_comparison,
+    write_comparison_json,
+    write_comparison_markdown,
+)
 
 __all__ = [
     "ActiveLearningBenchmarkResult",
     "BatchSelection",
     "RecoveryRound",
+    "STRATEGY_WEIGHTS",
+    "StrategyComparisonReport",
+    "StrategyResult",
     "generate_benchmark_pool",
     "run_active_learning_benchmark",
+    "run_strategy_comparison",
     "select_batch_2",
     "write_benchmark_pool",
+    "write_comparison_json",
+    "write_comparison_markdown",
 ]