From e83e148576abd4562097ecc77b4e0bb0ce35aaa7 Mon Sep 17 00:00:00 2001 From: OpenCode Date: Thu, 9 Jul 2026 12:23:29 +0700 Subject: [PATCH] feat: Phase G G4 active-learning strategy comparison report -- prevents one-selector bias --- Makefile | 8 +- docs/evidence/METRICS_CURRENT.md | 5 +- docs/research/ROADMAP.md | 56 ++ ...e_learning_strategy_comparison.schema.json | 129 ++++ .../active_learning/__init__.py | 18 + .../active_learning/strategy_comparison.py | 621 ++++++++++++++++++ src/openamp_foundry/cli/commands/benchmark.py | 31 + src/openamp_foundry/cli/main.py | 59 +- .../test_strategy_comparison.py | 362 ++++++++++ tests/test_test_count_regression.py | 4 +- 10 files changed, 1287 insertions(+), 6 deletions(-) create mode 100644 schemas/active_learning_strategy_comparison.schema.json create mode 100644 src/openamp_foundry/active_learning/strategy_comparison.py create mode 100644 tests/active_learning/test_strategy_comparison.py diff --git a/Makefile b/Makefile index 2c6fb6ee..b273db10 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example +.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example PYTHON := $(shell [ -f .venv/bin/python ] && echo .venv/bin/python || echo python3) PYTEST := $(shell [ -f .venv/bin/pytest ] && echo .venv/bin/pytest || echo pytest) @@ -55,6 +55,7 @@ help: @echo " make bench-simulation-ablation Simulation ablation on AMP-vs-decoy benchmark" @echo " make bench-simulation-ablation-within-amp Simulation ablation on hemolytic vs selective AMPs" @echo " make simulation-gate Fail-closed gate for weighted simulation mode" + @echo " make bench-strategy-compare Compare active-learning strategies (exploit/explore/diverse/combined/random)" @echo " make bench-gate Benchmark regression gate (AUROC drift check)" @echo " make regenerate-all Run all pipeline + benchmarks and verify determinism" @echo " make bench-hidden-active Hidden-positive recovery on mixed benchmark set" @@ -584,6 +585,11 @@ simulation-gate: --within-amp-json outputs/simulation_ablation_within_amp.json \ --out outputs/simulation_gate_verdict.json +bench-strategy-compare: + PYTHONPATH=src $(PYTHON) -m openamp_foundry.cli bench strategy-compare \ + --out-json outputs/strategy_comparison.json \ + --out-md outputs/strategy_comparison.md + bench-cross-dataset: PYTHONPATH=src $(PYTHON) scripts/benchmarks/benchmark_cross_dataset.py \ --out outputs/cross_dataset_benchmark.json diff --git a/docs/evidence/METRICS_CURRENT.md b/docs/evidence/METRICS_CURRENT.md index a49c6556..881725d0 100644 --- a/docs/evidence/METRICS_CURRENT.md +++ b/docs/evidence/METRICS_CURRENT.md @@ -5,7 +5,8 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **Purpose:** One authoritative table of current pipeline metrics. If any doc disagrees > with this file, this file wins. Updated whenever benchmark/benchmark config changes. > -> **Last updated:** 2026-07-09 (Phase G G3 — calibration pipeline consistency audit — v0.5.81) +> **Last updated:** 2026-07-09 (Phase G G4 — active-learning strategy comparison report — v0.5.82) +> **New in v0.5.82:** Active-learning strategy comparison report (G4) — CLI (`openamp-foundry bench strategy-compare`) that compares 5 selection strategies (exploitation, exploration, diversity, combined, random) on the same synthetic pool with identical hidden active candidates. Each strategy runs multi-round recovery of hidden actives; the report ranks strategies by recall, compares the production selector vs pure strategies and random baseline, and produces structured JSON + Markdown output with caveats. Prevents one-selector bias by making strategy performance transparent. Schema (`schemas/active_learning_strategy_comparison.schema.json`). `make bench-strategy-compare` target. 18 tests. 2955 total. > **New in v0.5.81:** Calibration pipeline consistency audit (G3) — CLI (`openamp-foundry calibration-audit`) that checks consistency across calibration pipeline artifacts (intake report, gate verdict, engine proposal, recalibration report). Checks intake↔gate count matching, engine↔gate verdict agreement, engine L1 budget compliance, report↔gate verdict consistency, report↔engine proposal consistency, timestamp sanity, and cohort-metrics warnings. Schema (`schemas/calibration_audit.schema.json`). `make calibration-audit` and `make calibration-audit-example` targets. 18 tests. 2937 total. > **New in v0.5.80:** Negative-result archive completeness checker (F10) — CLI (`scripts/check_negative_archive_completeness.py`) that reads a JSON archive of negative-result entries and checks each entry against completeness criteria: all required fields present, no duplicate candidate_ids, each entry has at least one content field (assay_result, score_safety, reviewer_notes, or reason_detail), date format valid YYYY-MM-DD, and optional intake_report_id references are well-formed. Outputs a structured completeness report as JSON and Markdown with summary, per-check results, and per-entry pass/fail. Schema (`schemas/negative_result_archive_completeness.schema.json`). Example file (`examples/negative_result_archive_example.json`) with 4 toy entries. 35 tests. 2919 total. > @@ -73,7 +74,7 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **New in v0.5.31:** Added dipeptide-order features for sequence-order awareness. `dipeptide_order_score` achieves AUROC 0.7861 on AMP-vs-scrambled discrimination — the strongest order-dependent feature in the pipeline. Only 7/31 features survive scrambling (amphipathicity/helix-wheel + dipeptide). All composition features are purely position-independent (exactly 0.5000 AUROC on scrambled test). > **New in v0.5.30:** Easy baseline benchmark added — charge density alone (AUROC 0.8166) outperforms the full pipeline ensemble (0.7792) on AMP-vs-Swiss-Prot-decoy discrimination. Honest finding documented: expected because pipeline optimizes for safety, not raw discrimination. > **New in v0.5.29:** Expanded benchmark to 500 AMPs + 500 composition-matched decoys (n=1000). AUROC 0.7792 (CI₉₅: 0.7505–0.8065) confirms signal generalizes. Cluster-aware CI: 0.746–0.8102. Representative AUROC: 0.778. Standard benchmark (n=191) retained for backward comparison. -> **Pipeline version:** v0.5.81 (G3) +> **Pipeline version:** v0.5.82 (G4) > **Branch:** main --- diff --git a/docs/research/ROADMAP.md b/docs/research/ROADMAP.md index 38c09e78..fd5513bf 100644 --- a/docs/research/ROADMAP.md +++ b/docs/research/ROADMAP.md @@ -1,5 +1,61 @@ # Roadmap +## v0.5.82 — Loop 82: Phase G G4 — Active-Learning Strategy Comparison Report ✓ (2026-07-09) + +CLI (`openamp-foundry bench strategy-compare`) that compares 5 selection strategies +(exploitation, exploration, diversity, combined, random) on the same synthetic pool +with identical hidden active candidates. Each strategy runs multi-round recovery of +hidden actives using the same batch-2 selector with different weights. The report +ranks strategies by recall, compares the production selector vs pure strategies and +random baseline, and produces structured JSON + Markdown output with caveats. +Prevents one-selector bias by making strategy performance transparent. + +Changes: +- `src/openamp_foundry/active_learning/strategy_comparison.py` (G4) — + `run_strategy_comparison()` generates a synthetic pool, hides N active candidates, + runs 5 strategies (exploitation, exploration, diversity, combined, random) via + multi-round selection, and produces a `StrategyComparisonReport` with per-strategy + recovery metrics, ranking by recall, best strategy identification, and production + selector comparison (vs random, exploitation, exploration, diversity). + `STRATEGY_WEIGHTS` dict defines the weight tuples for each strategy. + `write_comparison_json()` and `write_comparison_markdown()` produce structured output. +- `schemas/active_learning_strategy_comparison.schema.json` (G4) — JSON Schema + Draft 2020-12 for the strategy comparison report. Validates all required fields + including per-strategy results, ranking, production comparisons, and notes. +- `src/openamp_foundry/active_learning/__init__.py` — Exports `STRATEGY_WEIGHTS`, + `StrategyComparisonReport`, `StrategyResult`, `run_strategy_comparison`, + `write_comparison_json`, `write_comparison_markdown`. +- `src/openamp_foundry/cli/commands/benchmark.py` — Added + `_run_active_learning_strategy_compare()` CLI handler with `--n-total`, `--n-active`, + `--n-hidden`, `--batch-size`, `--max-rounds`, `--rng-seed`, `--out-json`, `--out-md` flags. +- `src/openamp_foundry/cli/main.py` — Registered `strategy-compare` subcommand under + `bench` with all argument flags and dispatch to handler. +- `Makefile` — Added `bench-strategy-compare` target with default params writing to + `outputs/strategy_comparison.json` and `outputs/strategy_comparison.md`. +- `tests/active_learning/test_strategy_comparison.py` — 18 tests covering: all 5 + strategies present, required fields (top-level + per-result), valid recall range, + ranking order, production strategy name, best strategy not null, exploitation + recovers actives, random baseline notes, production vs comparison fields, notes + presence, JSON and Markdown output writing, CLI exit 0, CLI writes files, random + baseline ranges, JSON Schema conformance, and production_outperforms_random type. +- `docs/evidence/METRICS_CURRENT.md` — v0.5.82 G4 changelog. Test count: 2955. +- `tests/test_test_count_regression.py` — baseline updated to 2955. + +Honest boundaries: +- This report uses **synthetic data** with known active/inactive labels. Results + reflect code-path integrity, not biological performance. +- All strategies run with safety/selectivity gates disabled to isolate strategy + effects. Real performance may differ with gates enabled. +- Random baseline is averaged over 20 Monte Carlo trials; deterministic strategies + use a single run per config. +- The production selector optimizes for multiple objectives (activity, safety, + diversity) that this recall-based benchmark does not fully measure. +- A strategy that ranks highest on recall may not be the best choice for real + candidate selection — domain-specific constraints, safety requirements, and + material constraints matter. +- This comparison is informational and requires qualified human review before + influencing selection decisions. + ## v0.5.81 — Loop 81: Phase G G3 — Calibration Pipeline Consistency Audit ✓ (2026-07-09) CLI (`openamp-foundry calibration-audit`) that checks consistency across the diff --git a/schemas/active_learning_strategy_comparison.schema.json b/schemas/active_learning_strategy_comparison.schema.json new file mode 100644 index 00000000..8172a243 --- /dev/null +++ b/schemas/active_learning_strategy_comparison.schema.json @@ -0,0 +1,129 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://openamp.ai/schemas/active_learning_strategy_comparison.schema.json", + "title": "Active-Learning Strategy Comparison Report", + "description": "Report comparing multiple active-learning selection strategies on a synthetic pool. All fields are computational results — not biological evidence.", + "type": "object", + "required": [ + "version", + "pool_size", + "n_active_total", + "n_hidden_actives", + "batch_size", + "max_rounds", + "results", + "ranked_by_recall", + "best_strategy", + "production_strategy", + "production_outperforms_random", + "notes" + ], + "properties": { + "version": { + "type": "string", + "description": "Schema/report version." + }, + "pool_size": { + "type": "integer", + "minimum": 1, + "description": "Total number of candidates in the synthetic pool." + }, + "n_active_total": { + "type": "integer", + "minimum": 0, + "description": "Total number of active (label=1) candidates." + }, + "n_hidden_actives": { + "type": "integer", + "minimum": 1, + "description": "Number of active candidates hidden during selection." + }, + "batch_size": { + "type": "integer", + "minimum": 1, + "description": "Number of candidates selected per round." + }, + "max_rounds": { + "type": "integer", + "minimum": 1, + "description": "Maximum number of selection rounds." + }, + "results": { + "type": "array", + "description": "Per-strategy recovery results.", + "items": { + "type": "object", + "required": [ + "strategy_name", + "rounds_to_first_recovery", + "final_recall", + "n_hidden_recovered", + "n_hidden_actives", + "total_rounds_run", + "notes" + ], + "properties": { + "strategy_name": { "type": "string" }, + "rounds_to_first_recovery": { + "oneOf": [ + { "type": "number", "minimum": 1 }, + { "type": "null" } + ] + }, + "final_recall": { "type": "number", "minimum": 0.0, "maximum": 1.0 }, + "n_hidden_recovered": { "type": "integer", "minimum": 0 }, + "n_hidden_actives": { "type": "integer", "minimum": 1 }, + "total_rounds_run": { "type": "integer", "minimum": 0 }, + "notes": { "type": "array", "items": { "type": "string" } } + } + } + }, + "ranked_by_recall": { + "type": "array", + "description": "Strategy names ranked by final recall descending.", + "items": { "type": "string" } + }, + "best_strategy": { + "oneOf": [ + { "type": "string" }, + { "type": "null" } + ], + "description": "Name of the strategy with highest recall." + }, + "production_strategy": { + "type": "string", + "description": "Name of the production selector strategy." + }, + "production_outperforms_random": { + "oneOf": [ + { "type": "boolean" }, + { "type": "null" } + ], + "description": "Whether the production selector outperforms random baseline on recall." + }, + "production_vs_exploitation": { + "oneOf": [ + { "type": "string", "enum": ["better", "worse", "similar"] }, + { "type": "null" } + ] + }, + "production_vs_exploration": { + "oneOf": [ + { "type": "string", "enum": ["better", "worse", "similar"] }, + { "type": "null" } + ] + }, + "production_vs_diversity": { + "oneOf": [ + { "type": "string", "enum": ["better", "worse", "similar"] }, + { "type": "null" } + ] + }, + "notes": { + "type": "array", + "description": "Human-readable notes and caveats.", + "items": { "type": "string" } + } + }, + "_caveat": "This report is a computational comparison of selection strategies on synthetic data. It is not biological evidence of sampling efficiency, candidate quality, or real-world selection performance." +} diff --git a/src/openamp_foundry/active_learning/__init__.py b/src/openamp_foundry/active_learning/__init__.py index 078fc1c0..de047862 100644 --- a/src/openamp_foundry/active_learning/__init__.py +++ b/src/openamp_foundry/active_learning/__init__.py @@ -12,6 +12,10 @@ The benchmark module validates whether the selector can recover known active candidates faster than random baseline — a code-path integrity check. + +The strategy-comparison report runs multiple selection strategies (exploitation, +exploration, diversity, combined, random) on the same synthetic pool and ranks +them by recovery recall, preventing one-selector bias. """ from openamp_foundry.active_learning.benchmark import ( @@ -25,13 +29,27 @@ BatchSelection, select_batch_2, ) +from openamp_foundry.active_learning.strategy_comparison import ( + STRATEGY_WEIGHTS, + StrategyComparisonReport, + StrategyResult, + run_strategy_comparison, + write_comparison_json, + write_comparison_markdown, +) __all__ = [ "ActiveLearningBenchmarkResult", "BatchSelection", "RecoveryRound", + "STRATEGY_WEIGHTS", + "StrategyComparisonReport", + "StrategyResult", "generate_benchmark_pool", "run_active_learning_benchmark", + "run_strategy_comparison", "select_batch_2", "write_benchmark_pool", + "write_comparison_json", + "write_comparison_markdown", ] diff --git a/src/openamp_foundry/active_learning/strategy_comparison.py b/src/openamp_foundry/active_learning/strategy_comparison.py new file mode 100644 index 00000000..3ac06fab --- /dev/null +++ b/src/openamp_foundry/active_learning/strategy_comparison.py @@ -0,0 +1,621 @@ +"""Active-learning strategy comparison report. + +Compares multiple selection strategies (exploitation, exploration, diversity, +combined, random) on the same synthetic pool with the same hidden active +candidates, producing a structured report that makes selection bias transparent. + +Each strategy is a wrapper around ``select_batch_2`` with different weights: + +- **exploitation**: ensemble_weight=1.0, others=0 — picks highest-scoring + candidates regardless of uncertainty or novelty. +- **exploration**: uncertainty_weight=1.0 — picks candidates where the model + is most unsure. +- **diversity**: diversity_weight=1.0 — picks candidates most dissimilar to + batch 1, exploring new sequence space. +- **combined**: default weights (ensemble=0.40, uncertainty=0.30, + diversity=0.30) — the production selector. +- **random**: random selection — cheap baseline for comparison. + +The report records per-strategy recovery metrics, ranks strategies by recall, +and identifies whether the production selector is meaningfully different from +simpler alternatives. +""" + +from __future__ import annotations + +import csv +import logging +import random +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +_log = logging.getLogger(__name__) + +_VERSION = "0.1.0" + +# Strategy definitions: human-readable label -> (ensemble, uncertainty, diversity) +STRATEGY_WEIGHTS: dict[str, tuple[float, float, float]] = { + "exploitation": (1.0, 0.0, 0.0), + "exploration": (0.0, 1.0, 0.0), + "diversity": (0.0, 0.0, 1.0), + "combined": (0.40, 0.30, 0.30), + "random": (0.0, 0.0, 0.0), # random uses no weights — handled separately +} + + +@dataclass(frozen=True) +class StrategyResult: + """Recovery metrics for a single strategy.""" + + strategy_name: str + rounds_to_first_recovery: int | None + final_recall: float + n_hidden_recovered: int + n_hidden_actives: int + total_rounds_run: int + notes: tuple[str, ...] + + def to_dict(self) -> dict[str, Any]: + return { + "strategy_name": self.strategy_name, + "rounds_to_first_recovery": self.rounds_to_first_recovery, + "final_recall": self.final_recall, + "n_hidden_recovered": self.n_hidden_recovered, + "n_hidden_actives": self.n_hidden_actives, + "total_rounds_run": self.total_rounds_run, + "notes": list(self.notes), + } + + +@dataclass(frozen=True) +class StrategyComparisonReport: + """Report comparing all active-learning strategies.""" + + version: str + pool_size: int + n_active_total: int + n_hidden_actives: int + batch_size: int + max_rounds: int + results: tuple[StrategyResult, ...] + ranked_by_recall: tuple[str, ...] + best_strategy: str | None + production_strategy: str + production_outperforms_random: bool | None + production_vs_exploitation: str | None + production_vs_exploration: str | None + production_vs_diversity: str | None + notes: tuple[str, ...] + + def to_dict(self) -> dict[str, Any]: + return { + "version": self.version, + "pool_size": self.pool_size, + "n_active_total": self.n_active_total, + "n_hidden_actives": self.n_hidden_actives, + "batch_size": self.batch_size, + "max_rounds": self.max_rounds, + "results": [r.to_dict() for r in self.results], + "ranked_by_recall": list(self.ranked_by_recall), + "best_strategy": self.best_strategy, + "production_strategy": self.production_strategy, + "production_outperforms_random": self.production_outperforms_random, + "production_vs_exploitation": self.production_vs_exploitation, + "production_vs_exploration": self.production_vs_exploration, + "production_vs_diversity": self.production_vs_diversity, + "notes": list(self.notes), + } + + +def _generate_pool( + *, + n_total: int = 50, + n_active: int = 10, + rng_seed: int = 42, +) -> list[dict]: + """Generate a synthetic candidate pool for strategy comparison. + + Active candidates get higher ensemble scores and lower disagreement. + """ + rng = random.Random(rng_seed) + pool: list[dict] = [] + for i in range(n_total): + is_active = i < n_active + cid = f"BENCH-CAND-{i:04d}" + if is_active: + ensemble = rng.uniform(0.75, 0.95) + disagreement = rng.uniform(0.05, 0.20) + else: + ensemble = rng.uniform(0.30, 0.55) + disagreement = rng.uniform(0.20, 0.50) + pool.append({ + "pilot_rank": str(i + 1), + "candidate_id": cid, + "sequence": f"PEPTIDE{i:04d}AKLF", + "length": 12, + "seed": "BENCH-SEED", + "ensemble": round(ensemble, 4), + "activity": round(ensemble, 4), + "boman_activity": "0.50", + "disagreement": round(disagreement, 4), + "safety": "0.90", + "synthesis": "0.85", + "novelty": "0.70", + "serum_stability": "0.65", + "selectivity_proxy": "0.60", + "rich_selectivity": "0.75", + "pilot_priority": "0.75", + "amphipathic_score": "1.5", + "charge_ph74": "4.0", + "label": 1 if is_active else 0, + }) + return pool + + +def _write_pool(pool: list[dict], path: Path) -> Path: + fieldnames = list(pool[0].keys()) if pool else [] + with open(path, "w", newline="") as f: + writer = csv.DictWriter(f, fieldnames=fieldnames) + writer.writeheader() + for row in pool: + writer.writerow(row) + return path + + +def _simulate_random_selection( + hidden_ids: set[str], + remaining_ids: list[str], + batch_size: int, + max_rounds: int, + rng: random.Random, +) -> tuple[int | None, float, int]: + """Simulate random batch selection. + + Returns (rounds_to_first_recovery, final_recall, n_recovered). + """ + available = list(remaining_ids) + recovered: set[str] = set() + rounds_to_first: int | None = None + rounds_run = 0 + + for round_n in range(1, max_rounds + 1): + if not available: + break + rounds_run = round_n + n_pick = min(batch_size, len(available)) + batch = rng.sample(available, n_pick) + just_recovered = set(batch) & hidden_ids + recovered.update(just_recovered) + if just_recovered and rounds_to_first is None: + rounds_to_first = round_n + available = [cid for cid in available if cid not in set(batch)] + + final_recall = len(recovered) / len(hidden_ids) if hidden_ids else 0.0 + return rounds_to_first, final_recall, len(recovered) + + +def _run_strategy( + strategy_name: str, + ensemble_w: float, + uncertainty_w: float, + diversity_w: float, + pool_csv: Path, + hidden_ids: set[str], + all_candidate_ids: list[str], + batch_size: int, + max_rounds: int, + rng: random.Random, +) -> StrategyResult: + """Run a single strategy and return its recovery metrics.""" + if strategy_name == "random": + result_data = _simulate_random_selection( + hidden_ids, + all_candidate_ids, + batch_size, + max_rounds, + rng, + ) + rounds_to_first, final_recall, n_recovered = result_data + notes: list[str] = [] + return StrategyResult( + strategy_name=strategy_name, + rounds_to_first_recovery=rounds_to_first, + final_recall=round(final_recall, 4), + n_hidden_recovered=n_recovered, + n_hidden_actives=len(hidden_ids), + total_rounds_run=max_rounds if rounds_to_first is None else ( + rounds_to_first if final_recall >= 1.0 else max_rounds + ), + notes=tuple(notes), + ) + + from openamp_foundry.active_learning.selector import select_batch_2 + + selected_so_far_ids: list[str] = [] + recovered: set[str] = set() + rounds_to_first: int | None = None + notes = [] + rounds_run = 0 + + for round_n in range(1, max_rounds + 1): + remaining = [ + cid for cid in all_candidate_ids + if cid not in set(selected_so_far_ids) + ] + if not remaining: + notes.append(f"Round {round_n}: pool exhausted") + break + rounds_run = round_n + + temp_csv = pool_csv.parent / f"_strategy_{strategy_name}_round_{round_n}.csv" + _write_remaining_pool(pool_csv, remaining, temp_csv) + + try: + selection = select_batch_2( + candidates_csv=temp_csv, + batch_1_ids=[], + n=batch_size, + safety_threshold=0.0, + selectivity_threshold=0.0, + ensemble_weight=ensemble_w, + uncertainty_weight=uncertainty_w, + diversity_weight=diversity_w, + min_uncertainty_probes=0, + ) + finally: + temp_csv.unlink(missing_ok=True) + + selected_ids = [str(c.get("candidate_id", "")) for c in selection.selected] + selected_so_far_ids.extend(selected_ids) + + just_recovered = set(selected_ids) & hidden_ids + recovered.update(just_recovered) + + if just_recovered and rounds_to_first is None: + rounds_to_first = round_n + + final_recall = len(recovered) / len(hidden_ids) if hidden_ids else 0.0 + + return StrategyResult( + strategy_name=strategy_name, + rounds_to_first_recovery=rounds_to_first, + final_recall=round(final_recall, 4), + n_hidden_recovered=len(recovered), + n_hidden_actives=len(hidden_ids), + total_rounds_run=rounds_run, + notes=tuple(notes), + ) + + +def _write_remaining_pool( + original_csv: Path, + remaining_ids: list[str], + output_csv: Path, +) -> None: + """Write a CSV containing only the remaining candidate IDs.""" + id_set = set(remaining_ids) + rows: list[dict] = [] + with open(original_csv, newline="") as f: + reader = csv.DictReader(f) + fieldnames = reader.fieldnames + for row in reader: + if row.get("candidate_id") in id_set: + rows.append(row) + + with open(output_csv, "w", newline="") as f: + writer = csv.DictWriter(f, fieldnames=fieldnames) + writer.writeheader() + for row in rows: + writer.writerow(row) + + +def _compare_recall( + production_recall: float, + other_recall: float, +) -> str | None: + """Compare production recall to another strategy's recall.""" + delta = production_recall - other_recall + if delta > 0.01: + return "better" + if delta < -0.01: + return "worse" + return "similar" + + +def run_strategy_comparison( + *, + rng_seed: int = 42, + n_total: int = 50, + n_active: int = 10, + n_hidden_actives: int = 3, + batch_size: int = 5, + max_rounds: int = 5, +) -> StrategyComparisonReport: + """Run the active-learning strategy comparison report. + + All strategies share the same synthetic pool and hidden active candidates. + Each strategy is evaluated on multi-round recovery of hidden actives. + + Returns: + ``StrategyComparisonReport`` with per-strategy results and rankings. + """ + rng = random.Random(rng_seed) + + pool = _generate_pool( + n_total=n_total, + n_active=n_active, + rng_seed=rng_seed, + ) + + active_ids = [ + str(c["candidate_id"]) for c in pool if int(c.get("label", 0)) == 1 + ] + if not active_ids: + raise ValueError("No active candidates in generated pool") + + all_candidate_ids = [str(c["candidate_id"]) for c in pool] + n_hide = min(n_hidden_actives, len(active_ids)) + hidden_ids = set(rng.sample(active_ids, n_hide)) + + notes: list[str] = [] + notes.append( + f"Pool: {len(pool)} total, {len(active_ids)} active, " + f"{len(pool) - len(active_ids)} inactive" + ) + notes.append( + f"Hiding {n_hide}/{len(active_ids)} active candidates" + ) + + import tempfile + pool_csv = Path(tempfile.mktemp(suffix=".csv")) + try: + _write_pool(pool, pool_csv) + + strategy_names = list(STRATEGY_WEIGHTS.keys()) + results: list[StrategyResult] = [] + + for sname in strategy_names: + if sname == "random": + # Average over 20 random trials + random_trials = [ + _run_strategy( + "random", 0.0, 0.0, 0.0, + pool_csv, hidden_ids, all_candidate_ids, + batch_size, max_rounds, rng, + ) + for _ in range(20) + ] + avg_recall = sum( + r.final_recall for r in random_trials + ) / len(random_trials) + valid_rounds = [ + r.rounds_to_first_recovery + for r in random_trials + if r.rounds_to_first_recovery is not None + ] + avg_rounds = ( + sum(valid_rounds) / len(valid_rounds) + if valid_rounds else None + ) + avg_recovered = sum( + r.n_hidden_recovered for r in random_trials + ) // len(random_trials) + results.append(StrategyResult( + strategy_name="random", + rounds_to_first_recovery=( + round(avg_rounds, 2) if avg_rounds is not None else None + ), + final_recall=round(avg_recall, 4), + n_hidden_recovered=avg_recovered, + n_hidden_actives=len(hidden_ids), + total_rounds_run=max_rounds, + notes=("Averaged over 20 random trials",), + )) + else: + ew, uw, dw = STRATEGY_WEIGHTS[sname] + result = _run_strategy( + sname, ew, uw, dw, + pool_csv, hidden_ids, all_candidate_ids, + batch_size, max_rounds, rng, + ) + results.append(result) + finally: + pool_csv.unlink(missing_ok=True) + + # Rank by recall descending + ranked = sorted(results, key=lambda r: r.final_recall, reverse=True) + ranked_names = tuple(r.strategy_name for r in ranked) + + # Best strategy + best = ranked[0].strategy_name if ranked else None + + # Comparison between production ("combined") and others + combined_result = next( + (r for r in results if r.strategy_name == "combined"), None + ) + random_result = next( + (r for r in results if r.strategy_name == "random"), None + ) + exploitation_result = next( + (r for r in results if r.strategy_name == "exploitation"), None + ) + exploration_result = next( + (r for r in results if r.strategy_name == "exploration"), None + ) + diversity_result = next( + (r for r in results if r.strategy_name == "diversity"), None + ) + + prod_beats_random: bool | None = None + if combined_result is not None and random_result is not None: + prod_beats_random = ( + combined_result.final_recall > random_result.final_recall + ) + + return StrategyComparisonReport( + version=_VERSION, + pool_size=len(pool), + n_active_total=len(active_ids), + n_hidden_actives=n_hide, + batch_size=batch_size, + max_rounds=max_rounds, + results=tuple(results), + ranked_by_recall=ranked_names, + best_strategy=best, + production_strategy="combined", + production_outperforms_random=prod_beats_random, + production_vs_exploitation=( + _compare_recall( + combined_result.final_recall if combined_result else 0.0, + exploitation_result.final_recall if exploitation_result else 0.0, + ) if combined_result and exploitation_result else None + ), + production_vs_exploration=( + _compare_recall( + combined_result.final_recall if combined_result else 0.0, + exploration_result.final_recall if exploration_result else 0.0, + ) if combined_result and exploration_result else None + ), + production_vs_diversity=( + _compare_recall( + combined_result.final_recall if combined_result else 0.0, + diversity_result.final_recall if diversity_result else 0.0, + ) if combined_result and diversity_result else None + ), + notes=tuple(notes), + ) + + +def write_comparison_json( + report: StrategyComparisonReport, + path: Path, +) -> Path: + """Write the comparison report to a JSON file.""" + import json + path.write_text(json.dumps(report.to_dict(), indent=2) + "\n") + return path + + +def write_comparison_markdown( + report: StrategyComparisonReport, + path: Path, +) -> Path: + """Write a human-readable Markdown summary.""" + lines: list[str] = [] + lines.append("# Active-Learning Strategy Comparison Report") + lines.append("") + lines.append(f"> **Version:** {report.version}") + lines.append(f"> **Generated:** auto") + lines.append("") + lines.append("## Overview") + lines.append("") + lines.append( + f"Compared {len(report.results)} strategies on a synthetic pool of " + f"**{report.pool_size}** candidates " + f"(**{report.n_active_total}** active, " + f"**{report.n_hidden_actives}** hidden)." + ) + lines.append(f"Batch size: **{report.batch_size}**, Max rounds: **{report.max_rounds}**.") + lines.append("") + + # Strategy descriptions + lines.append("## Strategies") + lines.append("") + lines.append("| Strategy | Ensemble weight | Uncertainty weight | Diversity weight | Description |") + lines.append("|----------|:---------------:|:-----------------:|:----------------:|-------------|") + for sname in STRATEGY_WEIGHTS: + ew, uw, dw = STRATEGY_WEIGHTS[sname] + if sname == "random": + desc = "Random selection (20-trial average)" + elif sname == "combined": + desc = "Production selector with default weights" + elif sname == "exploitation": + desc = "Highest ensemble score only" + elif sname == "exploration": + desc = "Highest uncertainty only" + elif sname == "diversity": + desc = "Most diverse vs batch 1 only" + else: + desc = "" + lines.append(f"| {sname} | {ew} | {uw} | {dw} | {desc} |") + lines.append("") + + # Per-strategy results + lines.append("## Per-Strategy Recovery Results") + lines.append("") + lines.append("| Strategy | Rounds to first recovery | Final recall | Hidden recovered | Notes |") + lines.append("|----------|:------------------------:|:------------:|:----------------:|-------|") + for r in report.results: + rtf = ( + f"{r.rounds_to_first_recovery}" + if r.rounds_to_first_recovery is not None + else "N/A" + ) + note = r.notes[0] if r.notes else "" + lines.append( + f"| {r.strategy_name} | {rtf} | {r.final_recall} | " + f"{r.n_hidden_recovered}/{r.n_hidden_actives} | {note} |" + ) + lines.append("") + + # Ranking + lines.append("## Ranking by Recall") + lines.append("") + lines.append(f"Best strategy: **{report.best_strategy}**") + lines.append("") + for i, sname in enumerate(report.ranked_by_recall, 1): + r = next(r for r in report.results if r.strategy_name == sname) + marker = " ← production" if sname == "combined" else "" + lines.append(f"{i}. **{sname}**: recall={r.final_recall}{marker}") + lines.append("") + + # Production comparison + lines.append("## Production Selector Comparison") + lines.append("") + lines.append( + f"The production strategy (**{report.production_strategy}**) " + f"{'outperforms' if report.production_outperforms_random else 'does NOT outperform'} " + f"random selection." + ) + lines.append("") + comparisons = [ + ("vs exploitation", report.production_vs_exploitation), + ("vs exploration", report.production_vs_exploration), + ("vs diversity", report.production_vs_diversity), + ] + for label, value in comparisons: + if value is not None: + lines.append(f"- **{label}**: {value}") + lines.append("") + + # Caveat + lines.append("## Caveats") + lines.append("") + lines.append( + "- This report uses **synthetic data** with known active/inactive labels. " + "Results reflect code-path integrity, not biological performance." + ) + lines.append( + "- All strategies were run with safety/selectivity gates disabled " + "(threshold=0.0) to isolate strategy effects." + ) + lines.append( + "- Random baseline is averaged over 20 Monte Carlo trials; " + "deterministic strategies use a single run." + ) + lines.append( + "- The production selector optimizes for multiple objectives " + "(activity, safety, diversity) that this benchmark does not fully measure." + ) + lines.append( + "- A strategy that ranks highest on recall may not be the best choice " + "for real candidate selection — domain-specific constraints matter." + ) + lines.append( + "- This comparison is informational and requires qualified human review " + "before influencing selection decisions." + ) + + path.write_text("\n".join(lines) + "\n") + return path diff --git a/src/openamp_foundry/cli/commands/benchmark.py b/src/openamp_foundry/cli/commands/benchmark.py index d3cf5130..92571aa7 100644 --- a/src/openamp_foundry/cli/commands/benchmark.py +++ b/src/openamp_foundry/cli/commands/benchmark.py @@ -323,6 +323,37 @@ def _run_active_learning_bench(args: argparse.Namespace) -> int: return 0 +def _run_active_learning_strategy_compare(args: argparse.Namespace) -> int: + """Run the active-learning strategy comparison report.""" + import json + from pathlib import Path + from openamp_foundry.active_learning.strategy_comparison import ( + run_strategy_comparison, + write_comparison_json, + write_comparison_markdown, + ) + + report = run_strategy_comparison( + rng_seed=args.rng_seed, + n_total=args.n_total, + n_active=args.n_active, + n_hidden_actives=args.n_hidden, + batch_size=args.batch_size, + max_rounds=args.max_rounds, + ) + + out_dict = report.to_dict() + if args.out_json: + write_comparison_json(report, Path(args.out_json)) + print(f"JSON report written to {args.out_json}") + if args.out_md: + write_comparison_markdown(report, Path(args.out_md)) + print(f"Markdown report written to {args.out_md}") + if not args.out_json and not args.out_md: + print(json.dumps(out_dict, indent=2)) + return 0 + + def _run_simulation_gate(args: argparse.Namespace) -> int: """Validate whether virtual-assay scores may affect ranking.""" from openamp_foundry.simulation.gate import evaluate_simulation_gate diff --git a/src/openamp_foundry/cli/main.py b/src/openamp_foundry/cli/main.py index 5c481655..3be2c834 100644 --- a/src/openamp_foundry/cli/main.py +++ b/src/openamp_foundry/cli/main.py @@ -1,7 +1,7 @@ from __future__ import annotations from openamp_foundry.cli.commands.core import _run_generate_batch -from openamp_foundry.cli.commands.benchmark import _run_bench, _run_validate_scoring, _run_cluster_split_bench, _run_expert_ablation_bench, _run_selectivity_bench, _run_triage, _run_metrics_snapshot, _run_feature_decomp, _run_active_learning_bench, _run_simulation_gate +from openamp_foundry.cli.commands.benchmark import _run_bench, _run_validate_scoring, _run_cluster_split_bench, _run_expert_ablation_bench, _run_selectivity_bench, _run_triage, _run_metrics_snapshot, _run_feature_decomp, _run_active_learning_bench, _run_active_learning_strategy_compare, _run_simulation_gate from openamp_foundry.cli.commands.selection import _run_pilot_panel, _run_pilot_confident, _run_diversity_check, _run_select_batch from openamp_foundry.cli.commands.external import _run_external_predict, _run_external_consensus from openamp_foundry.cli.commands.qc import _run_synthesis_order, _run_presynth_qc @@ -330,6 +330,61 @@ def build_parser() -> argparse.ArgumentParser: help="Random seed for reproducibility (default: 42).", ) + strategy_compare = bench_sub.add_parser( + "strategy-compare", + help=( + "Compare active-learning selection strategies (exploitation, exploration, " + "diversity, combined, random) on the same synthetic pool. Prevents " + "one-selector bias by making strategy comparison transparent." + ), + ) + strategy_compare.add_argument( + "--n-total", + type=int, + default=50, + help="Total candidate pool size (default: 50).", + ) + strategy_compare.add_argument( + "--n-active", + type=int, + default=10, + help="Number of active candidates in pool (default: 10).", + ) + strategy_compare.add_argument( + "--n-hidden", + type=int, + default=3, + help="Number of active candidates to hide (default: 3).", + ) + strategy_compare.add_argument( + "--batch-size", + type=int, + default=5, + help="Batch size per selection round (default: 5).", + ) + strategy_compare.add_argument( + "--max-rounds", + type=int, + default=5, + help="Maximum selection rounds (default: 5).", + ) + strategy_compare.add_argument( + "--rng-seed", + type=int, + default=42, + help="Random seed for reproducibility (default: 42).", + ) + strategy_compare.add_argument( + "--out-json", + default=None, + help="Optional JSON output path.", + ) + strategy_compare.add_argument( + "--out-md", + default=None, + help="Optional Markdown output path.", + ) + simulation_gate = bench_sub.add_parser( "simulation-gate", help=( @@ -1160,6 +1215,8 @@ def main(argv: list[str] | None = None) -> int: return _run_metrics_snapshot(args) if args.bench_command == "active-learning": return _run_active_learning_bench(args) + if args.bench_command == "strategy-compare": + return _run_active_learning_strategy_compare(args) if args.bench_command == "simulation-gate": return _run_simulation_gate(args) return _run_bench(args) diff --git a/tests/active_learning/test_strategy_comparison.py b/tests/active_learning/test_strategy_comparison.py new file mode 100644 index 00000000..5181f24f --- /dev/null +++ b/tests/active_learning/test_strategy_comparison.py @@ -0,0 +1,362 @@ +"""Tests for the active-learning strategy comparison report. + +All data is synthetic. Tests verify code-path integrity, structure, and +honest comparison mechanics — not biological sampling efficiency. +""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +import pytest + + +# ── Helpers ─────────────────────────────────────────────────────────── + + +def _check_report_shape(report: dict) -> None: + """Assert all required top-level fields exist.""" + required = [ + "version", "pool_size", "n_active_total", "n_hidden_actives", + "batch_size", "max_rounds", "results", "ranked_by_recall", + "best_strategy", "production_strategy", + "production_outperforms_random", "notes", + ] + for field in required: + assert field in report, f"Missing field: {field}" + + +def _check_result_shape(result: dict) -> None: + """Assert all required per-strategy result fields exist.""" + required = [ + "strategy_name", "rounds_to_first_recovery", "final_recall", + "n_hidden_recovered", "n_hidden_actives", "total_rounds_run", "notes", + ] + for field in required: + assert field in result, f"Missing result field: {field}" + + +# ── Tests ───────────────────────────────────────────────────────────── + + +class TestStrategyComparisonReport: + """Tests for the strategy comparison report generation.""" + + def test_report_has_all_strategies(self): + """Report contains results for all 5 strategies.""" + from openamp_foundry.active_learning.strategy_comparison import ( + STRATEGY_WEIGHTS, + run_strategy_comparison, + ) + + report = run_strategy_comparison( + n_total=30, n_active=5, n_hidden_actives=2, + batch_size=3, max_rounds=3, + ) + d = report.to_dict() + strategy_names = {r["strategy_name"] for r in d["results"]} + expected = set(STRATEGY_WEIGHTS.keys()) + assert strategy_names == expected, ( + f"Missing strategies: {expected - strategy_names}" + ) + + def test_report_required_fields(self): + """Report has all required top-level fields.""" + from openamp_foundry.active_learning.strategy_comparison import ( + run_strategy_comparison, + ) + + report = run_strategy_comparison( + n_total=30, n_active=5, n_hidden_actives=2, + batch_size=3, max_rounds=3, + ) + _check_report_shape(report.to_dict()) + + def test_per_strategy_required_fields(self): + """Each strategy result has all required fields.""" + from openamp_foundry.active_learning.strategy_comparison import ( + run_strategy_comparison, + ) + + report = run_strategy_comparison( + n_total=30, n_active=5, n_hidden_actives=2, + batch_size=3, max_rounds=3, + ) + for r in report.to_dict()["results"]: + _check_result_shape(r) + + def test_recall_in_valid_range(self): + """Final recall is between 0 and 1 for all strategies.""" + from openamp_foundry.active_learning.strategy_comparison import ( + run_strategy_comparison, + ) + + report = run_strategy_comparison( + n_total=30, n_active=5, n_hidden_actives=2, + batch_size=3, max_rounds=3, + ) + for r in report.to_dict()["results"]: + assert 0.0 <= r["final_recall"] <= 1.0, ( + f"Recall {r['final_recall']} out of range for {r['strategy_name']}" + ) + + def test_ranked_by_recall_matches_results(self): + """Ranked strategy list matches sorted results by recall descending.""" + from openamp_foundry.active_learning.strategy_comparison import ( + run_strategy_comparison, + ) + + report = run_strategy_comparison( + n_total=30, n_active=5, n_hidden_actives=2, + batch_size=3, max_rounds=3, + ) + d = report.to_dict() + results = {r["strategy_name"]: r["final_recall"] for r in d["results"]} + ranked = d["ranked_by_recall"] + assert len(ranked) == len(results) + for i in range(len(ranked) - 1): + assert results[ranked[i]] >= results[ranked[i + 1]], ( + f"Ranking order wrong: {ranked[i]} ({results[ranked[i]]}) " + f"should be >= {ranked[i + 1]} ({results[ranked[i + 1]]})" + ) + + def test_production_strategy_is_combined(self): + """The production strategy is 'combined'.""" + from openamp_foundry.active_learning.strategy_comparison import ( + run_strategy_comparison, + ) + + report = run_strategy_comparison( + n_total=30, n_active=5, n_hidden_actives=2, + batch_size=3, max_rounds=3, + ) + assert report.production_strategy == "combined" + + def test_best_strategy_not_null(self): + """Best strategy is identified.""" + from openamp_foundry.active_learning.strategy_comparison import ( + run_strategy_comparison, + ) + + report = run_strategy_comparison( + n_total=30, n_active=5, n_hidden_actives=2, + batch_size=3, max_rounds=3, + ) + assert report.best_strategy is not None + assert report.best_strategy in [r.strategy_name for r in report.results] + + def test_exploitation_recovers_actives_fast(self): + """Exploitation should recover hidden actives quickly because they + have higher ensemble scores.""" + from openamp_foundry.active_learning.strategy_comparison import ( + run_strategy_comparison, + ) + + report = run_strategy_comparison( + n_total=50, n_active=10, n_hidden_actives=3, + batch_size=5, max_rounds=5, + ) + exploitation = next( + r for r in report.results if r.strategy_name == "exploitation" + ) + assert exploitation.final_recall > 0.0, ( + f"Exploitation failed to recover any actives: {exploitation.notes}" + ) + + def test_random_baseline_has_notes(self): + """Random strategy result includes averaging note.""" + from openamp_foundry.active_learning.strategy_comparison import ( + run_strategy_comparison, + ) + + report = run_strategy_comparison( + n_total=30, n_active=5, n_hidden_actives=2, + batch_size=3, max_rounds=3, + ) + random_result = next( + r for r in report.results if r.strategy_name == "random" + ) + assert len(random_result.notes) > 0 + + def test_production_vs_comparison_fields(self): + """Production vs other strategy comparison fields are present.""" + from openamp_foundry.active_learning.strategy_comparison import ( + run_strategy_comparison, + ) + + report = run_strategy_comparison( + n_total=30, n_active=5, n_hidden_actives=2, + batch_size=3, max_rounds=3, + ) + d = report.to_dict() + for field in [ + "production_vs_exploitation", + "production_vs_exploration", + "production_vs_diversity", + ]: + assert field in d, f"Missing comparison field: {field}" + assert d[field] in ("better", "worse", "similar", None), ( + f"Unexpected value for {field}: {d[field]}" + ) + + def test_notes_are_present(self): + """Report has human-readable notes.""" + from openamp_foundry.active_learning.strategy_comparison import ( + run_strategy_comparison, + ) + + report = run_strategy_comparison( + n_total=30, n_active=5, n_hidden_actives=2, + batch_size=3, max_rounds=3, + ) + assert len(report.notes) > 0 + assert any("Pool" in n for n in report.notes) + + def test_write_json(self, tmp_path): + """Writing to JSON produces valid file with expected fields.""" + from openamp_foundry.active_learning.strategy_comparison import ( + run_strategy_comparison, + write_comparison_json, + ) + + report = run_strategy_comparison( + n_total=30, n_active=5, n_hidden_actives=2, + batch_size=3, max_rounds=3, + ) + out = tmp_path / "report.json" + write_comparison_json(report, out) + assert out.exists() + data = json.loads(out.read_text()) + _check_report_shape(data) + + def test_write_markdown(self, tmp_path): + """Writing to Markdown produces expected sections.""" + from openamp_foundry.active_learning.strategy_comparison import ( + run_strategy_comparison, + write_comparison_markdown, + ) + + report = run_strategy_comparison( + n_total=30, n_active=5, n_hidden_actives=2, + batch_size=3, max_rounds=3, + ) + out = tmp_path / "report.md" + write_comparison_markdown(report, out) + assert out.exists() + content = out.read_text() + assert "# Active-Learning Strategy Comparison Report" in content + assert "## Overview" in content + assert "## Strategies" in content + assert "## Per-Strategy Recovery Results" in content + assert "## Ranking by Recall" in content + assert "## Production Selector Comparison" in content + assert "## Caveats" in content + + def test_cli_exits_zero(self): + """CLI bench strategy-compare exits 0 and produces valid output.""" + result = subprocess.run( + [ + sys.executable, "-m", "openamp_foundry.cli", "bench", + "strategy-compare", + "--n-total", "20", + "--n-active", "4", + "--n-hidden", "2", + "--batch-size", "3", + "--max-rounds", "3", + "--rng-seed", "42", + ], + capture_output=True, text=True, timeout=60, + ) + assert result.returncode == 0, f"CLI failed: {result.stderr[:500]}" + data = json.loads(result.stdout) + assert "version" in data + assert "results" in data + assert "ranked_by_recall" in data + assert len(data["results"]) == 5 + + def test_cli_writes_json_and_md(self, tmp_path): + """CLI with --out-json and --out-md writes both files.""" + out_json = tmp_path / "report.json" + out_md = tmp_path / "report.md" + result = subprocess.run( + [ + sys.executable, "-m", "openamp_foundry.cli", "bench", + "strategy-compare", + "--n-total", "20", + "--n-active", "4", + "--n-hidden", "2", + "--batch-size", "3", + "--max-rounds", "3", + "--out-json", str(out_json), + "--out-md", str(out_md), + ], + capture_output=True, text=True, timeout=60, + ) + assert result.returncode == 0, f"CLI failed: {result.stderr[:500]}" + assert out_json.exists() + assert out_md.exists() + data = json.loads(out_json.read_text()) + _check_report_shape(data) + md_content = out_md.read_text() + assert "# Active-Learning Strategy Comparison Report" in md_content + + def test_random_has_expected_ranges(self): + """Random baseline recall should be within expected bounds + on a small pool with several hidden actives.""" + from openamp_foundry.active_learning.strategy_comparison import ( + run_strategy_comparison, + ) + + report = run_strategy_comparison( + n_total=20, n_active=5, n_hidden_actives=3, + batch_size=5, max_rounds=3, + ) + random_result = next( + r for r in report.results if r.strategy_name == "random" + ) + assert 0.0 <= random_result.final_recall <= 1.0 + # With 3 hidden out of 20, batch size 5, 3 rounds, expectation is + # non-trivial recovery probability + assert random_result.n_hidden_recovered >= 0 + + def test_schema_conformance(self, tmp_path): + """JSON output validates against JSON Schema.""" + import jsonschema + + schema_path = ( + Path(__file__).resolve().parents[2] + / "schemas" + / "active_learning_strategy_comparison.schema.json" + ) + schema = json.loads(schema_path.read_text()) + + from openamp_foundry.active_learning.strategy_comparison import ( + run_strategy_comparison, + write_comparison_json, + ) + + report = run_strategy_comparison( + n_total=30, n_active=5, n_hidden_actives=2, + batch_size=3, max_rounds=3, + ) + out = tmp_path / "report.json" + write_comparison_json(report, out) + data = json.loads(out.read_text()) + + jsonschema.validate(instance=data, schema=schema) + + def test_production_outperforms_random_type(self): + """production_outperforms_random is bool or None.""" + from openamp_foundry.active_learning.strategy_comparison import ( + run_strategy_comparison, + ) + + report = run_strategy_comparison( + n_total=30, n_active=5, n_hidden_actives=2, + batch_size=3, max_rounds=3, + ) + val = report.production_outperforms_random + assert val is True or val is False or val is None diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index f8dfc2e1..ece981fb 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -2,9 +2,9 @@ import subprocess import sys -# Baseline: 2919 tests as of Loop 80 (2026-07-09) -- Phase F F10 (negative-result archive completeness checker) +# Baseline: 2955 tests as of Loop 82 (2026-07-09) -- Phase G G4 (active-learning strategy comparison report) # Raise threshold if legitimate tests are removed; lower threshold is okay. -BASELINE = 2937 +BASELINE = 2955 TOLERANCE = 0.05 # allow 5% fluctuation