From ad20807a681ab89d7a3459803a75f6eef2a21602 Mon Sep 17 00:00:00 2001 From: OpenCode Date: Thu, 9 Jul 2026 12:48:36 +0700 Subject: [PATCH] feat: Phase G G7 result-quality flag propagation -- low-quality outcomes cannot drive calibration updates --- Makefile | 9 +- docs/evidence/METRICS_CURRENT.md | 5 +- docs/research/ROADMAP.md | 50 +++ schemas/result_quality_report.schema.json | 63 ++++ src/openamp_foundry/calibration/__init__.py | 16 + .../calibration/result_quality.py | 313 ++++++++++++++++++ src/openamp_foundry/cli/commands/reports.py | 50 +++ src/openamp_foundry/cli/main.py | 30 +- tests/calibration/test_result_quality.py | 189 +++++++++++ tests/test_test_count_regression.py | 4 +- 10 files changed, 723 insertions(+), 6 deletions(-) create mode 100644 schemas/result_quality_report.schema.json create mode 100644 src/openamp_foundry/calibration/result_quality.py create mode 100644 tests/calibration/test_result_quality.py diff --git a/Makefile b/Makefile index 5d4cf25a..66412671 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check +.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter PYTHON := $(shell [ -f .venv/bin/python ] && echo .venv/bin/python || echo python3) PYTEST := $(shell [ -f .venv/bin/pytest ] && echo .venv/bin/pytest || echo pytest) @@ -678,3 +678,10 @@ calibration-overfit-check: --cohort-sizes 8,25,50 --model-params 10 --n-features 5 \ --out-json /tmp/overfit_check.json --out-md /tmp/overfit_check.md @echo "Overfit check complete. See /tmp/overfit_check.json" + +result-quality-filter: + echo '[{"candidate_id":"AMP001","flags":[]},{"candidate_id":"AMP002","flags":["contamination"]},{"candidate_id":"AMP003","flags":["replicate_disagreement","borderline_threshold"]}]' > /tmp/rq_input.json + .venv/bin/openamp-foundry result-quality-filter \ + --results-json /tmp/rq_input.json \ + --out-json /tmp/rq_output.json --out-md /tmp/rq_output.md + @echo "Result quality filter done. See /tmp/rq_output.json" diff --git a/docs/evidence/METRICS_CURRENT.md b/docs/evidence/METRICS_CURRENT.md index 4b93867d..7b2c0385 100644 --- a/docs/evidence/METRICS_CURRENT.md +++ b/docs/evidence/METRICS_CURRENT.md @@ -5,7 +5,8 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **Purpose:** One authoritative table of current pipeline metrics. If any doc disagrees > with this file, this file wins. Updated whenever benchmark/benchmark config changes. > -> **Last updated:** 2026-07-09 (Phase G G6 — calibration-overfit warning — v0.5.84) +> **Last updated:** 2026-07-09 (Phase G G7 — result-quality flag propagation — v0.5.85) +> **New in v0.5.85:** Result-quality flag propagation (G7) — `assess_result_quality()` and `filter_results_for_calibration()` propagate result-quality flags into the calibration engine. Candidates with contamination or assay interference are excluded; candidates with 2+ quality flags are excluded; candidates with 1 flag are included with caution; clean candidates are fully included. `QUALITY_FLAGS` dictionary (8 flags). `ResultQualityReport` dataclass. CLI (`openamp-foundry result-quality-filter`) accepts `--results-json`, `--out-json`, `--out-md`. JSON + Markdown output. Schema (`schemas/result_quality_report.schema.json`). `make result-quality-filter` target. 27 tests. 3022 total. > **New in v0.5.84:** Calibration-overfit warning (G6) — `check_cohort_overfit_risk()` and `run_overfit_check()` flag when a calibration cohort is too small relative to model parameters. Warns at three severity levels (critical/warning/caution/none). CLI (`openamp-foundry calibration-overfit-check`) accepts comma-separated cohort sizes, model params, n features. JSON + Markdown output. Schema (`schemas/calibration_overfit_check.schema.json`). `make calibration-overfit-check` target. 21 tests. 2995 total. > **New in v0.5.83:** Batch-2 selection rationale report (G5) — CLI (`openamp-foundry batch-rationale`) that generates a synthetic candidate pool, runs the batch-2 selector with configurable weights, and produces a per-candidate rationale report classifying each selected candidate into exploit / explore / diversity / combined roles. Reports weight configuration, role breakdown summary, per-candidate contributions (ensemble×weight, uncertainty×weight, diversity×weight), safety gate impact, and caveats. Schema (`schemas/batch_rationale_report.schema.json`). `make batch-rationale` target. 19 tests. 2974 total. > **New in v0.5.82:** Active-learning strategy comparison report (G4) — CLI (`openamp-foundry bench strategy-compare`) that compares 5 selection strategies (exploitation, exploration, diversity, combined, random) on the same synthetic pool with identical hidden active candidates. Each strategy runs multi-round recovery of hidden actives; the report ranks strategies by recall, compares the production selector vs pure strategies and random baseline, and produces structured JSON + Markdown output with caveats. Prevents one-selector bias by making strategy performance transparent. Schema (`schemas/active_learning_strategy_comparison.schema.json`). `make bench-strategy-compare` target. 18 tests. 2955 total. @@ -76,7 +77,7 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **New in v0.5.31:** Added dipeptide-order features for sequence-order awareness. `dipeptide_order_score` achieves AUROC 0.7861 on AMP-vs-scrambled discrimination — the strongest order-dependent feature in the pipeline. Only 7/31 features survive scrambling (amphipathicity/helix-wheel + dipeptide). All composition features are purely position-independent (exactly 0.5000 AUROC on scrambled test). > **New in v0.5.30:** Easy baseline benchmark added — charge density alone (AUROC 0.8166) outperforms the full pipeline ensemble (0.7792) on AMP-vs-Swiss-Prot-decoy discrimination. Honest finding documented: expected because pipeline optimizes for safety, not raw discrimination. > **New in v0.5.29:** Expanded benchmark to 500 AMPs + 500 composition-matched decoys (n=1000). AUROC 0.7792 (CI₉₅: 0.7505–0.8065) confirms signal generalizes. Cluster-aware CI: 0.746–0.8102. Representative AUROC: 0.778. Standard benchmark (n=191) retained for backward comparison. -> **Pipeline version:** v0.5.84 (G6) +> **Pipeline version:** v0.5.85 (G7) > **Branch:** main --- diff --git a/docs/research/ROADMAP.md b/docs/research/ROADMAP.md index 7e91c2fe..b35503df 100644 --- a/docs/research/ROADMAP.md +++ b/docs/research/ROADMAP.md @@ -1,5 +1,55 @@ # Roadmap +## v0.5.85 — Loop 85: Phase G G7 — Result-Quality Flag Propagation into Calibration Engine ✓ (2026-07-09) + +`assess_result_quality()` and `filter_results_for_calibration()` propagate +result-quality flags into the calibration engine. Low-quality outcomes cannot +drive updates — garbage results must not update the scoring model. + +Changes: +- `src/openamp_foundry/calibration/result_quality.py` (G7) — Core module with + `QUALITY_FLAGS` dictionary (8 standard flags), `EXCLUDED_FLAGS` set + (contamination, assay_interference), `ResultQualityReport` dataclass (7 fields: + candidate_id, flags, quality_level, can_drive_update, propagation_action, + explanation, dry_lab_only), `assess_result_quality()` applying multi-tier rules + (excluded flags → excluded, 2+ minor flags → low/excluded, 1 flag → acceptable, + 0 flags → high), `filter_results_for_calibration()` grouping results into + included/included_with_caution/excluded with summary counts, + `write_result_quality_json()` and `write_result_quality_markdown()` for output. +- `schemas/result_quality_report.schema.json` (G7) — JSON Schema Draft 07 for + per-candidate or aggregate result quality reports. Validates all 7 required + fields including flag enum constraint and dry_lab_only const=true. +- `src/openamp_foundry/calibration/__init__.py` — Exports all result-quality + symbols. +- `src/openamp_foundry/cli/commands/reports.py` — Added `_run_result_quality_filter()` + CLI handler with `--results-json`, `--out-json`, `--out-md` flags. +- `src/openamp_foundry/cli/main.py` — Registered `result-quality-filter` subcommand + with all argument flags and dispatch to handler. +- `Makefile` — Added `result-quality-filter` target with default example data writing + to `/tmp/rq_output.json` and `/tmp/rq_output.md`. Added to `.PHONY`. +- `tests/calibration/test_result_quality.py` — 27 tests covering: high quality (no + flags), contamination excluded, assay_interference excluded, 2 minor flags excluded + (low quality), 1 minor flag include_with_caution, can_drive_update True/False for + all quality levels, dry_lab_only always True, unknown flag raises ValueError, + explanation non-empty, to_dict output, contamination+other flags still excluded, + assay_interference+other flags still excluded, filter_results_for_calibration + empty/summary counts/per-action grouping/can_drive_update_count, missing flags + handling, EXCLUDED_FLAGS set, QUALITY_FLAGS descriptions and count. +- `docs/evidence/METRICS_CURRENT.md` — v0.5.85 G7 changelog. Test count: 3022. +- `tests/test_test_count_regression.py` — baseline updated to 3022. + +Honest boundaries: +- Quality assessment is a computational filter on structural and metadata criteria. + A "high quality" result does not confirm biological activity. +- Flag-based classification uses a pre-defined rule set. Edge cases (e.g., + borderline_threshold with ambiguous_activity) are treated the same as any + other 2-flag combination — excluded for caution. +- Excluded results may still contain useful scientific information and should + remain available for expert review. +- The `dry_lab_only=True` constraint is an attestation, not a technical proof. +- All calibration decisions require qualified human review regardless of + result quality assessment. + ## v0.5.84 — Loop 84: Phase G G6 — Calibration-Overfit Warning for Small Cohorts ✓ (2026-07-09) `check_cohort_overfit_risk()` and `run_overfit_check()` flag when a calibration diff --git a/schemas/result_quality_report.schema.json b/schemas/result_quality_report.schema.json new file mode 100644 index 00000000..44c56bdc --- /dev/null +++ b/schemas/result_quality_report.schema.json @@ -0,0 +1,63 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://openamp.ai/schemas/result_quality_report.schema.json", + "title": "Result Quality Report", + "description": "Schema for a result quality assessment report. Dry-lab only — does not measure biological activity.", + "type": "object", + "properties": { + "candidate_id": { + "type": "string", + "description": "Identifier for the candidate peptide." + }, + "flags": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "low_sample_quality", + "assay_interference", + "ambiguous_activity", + "contamination", + "protocol_deviation", + "borderline_threshold", + "replicate_disagreement", + "missing_negative_control" + ] + }, + "description": "Quality flags assigned to this result." + }, + "quality_level": { + "type": "string", + "enum": ["high", "acceptable", "low", "excluded"], + "description": "Overall quality assessment level." + }, + "can_drive_update": { + "type": "boolean", + "description": "Whether this result may drive a calibration update." + }, + "propagation_action": { + "type": "string", + "enum": ["include", "include_with_caution", "exclude"], + "description": "Propagation action for the calibration engine." + }, + "explanation": { + "type": "string", + "description": "Human-readable explanation of the quality assessment." + }, + "dry_lab_only": { + "type": "boolean", + "description": "Always true. This is a computational assessment, not biological proof.", + "const": true + } + }, + "required": [ + "candidate_id", + "flags", + "quality_level", + "can_drive_update", + "propagation_action", + "explanation", + "dry_lab_only" + ], + "additionalProperties": false +} diff --git a/src/openamp_foundry/calibration/__init__.py b/src/openamp_foundry/calibration/__init__.py index 64a04493..1de029d7 100644 --- a/src/openamp_foundry/calibration/__init__.py +++ b/src/openamp_foundry/calibration/__init__.py @@ -74,6 +74,15 @@ auto_increment_version, ) +from openamp_foundry.calibration.result_quality import ( + QUALITY_FLAGS, + ResultQualityReport, + assess_result_quality, + filter_results_for_calibration, + write_result_quality_json, + write_result_quality_markdown, +) + __all__ = [ # Constants "MIN_COHORT_SIZE", @@ -120,4 +129,11 @@ "run_overfit_check", "write_overfit_check_json", "write_overfit_check_markdown", + # Result quality + "QUALITY_FLAGS", + "ResultQualityReport", + "assess_result_quality", + "filter_results_for_calibration", + "write_result_quality_json", + "write_result_quality_markdown", ] diff --git a/src/openamp_foundry/calibration/result_quality.py b/src/openamp_foundry/calibration/result_quality.py new file mode 100644 index 00000000..24ad8f52 --- /dev/null +++ b/src/openamp_foundry/calibration/result_quality.py @@ -0,0 +1,313 @@ +"""Result-quality flag propagation into the calibration engine. + +Low-quality outcomes cannot drive updates — garbage results must not +update the scoring model. This module provides quality assessment of +lab results before they enter the calibration pipeline. + +Dry-lab only. Does not measure biological activity. +""" +from __future__ import annotations + +import json +from dataclasses import dataclass, field, asdict +from pathlib import Path +from typing import Any + +QUALITY_FLAGS: dict[str, str] = { + "low_sample_quality": "Sample quality below acceptable threshold", + "assay_interference": "Assay interference detected (e.g., compound precipitation, quenching)", + "ambiguous_activity": "Activity borderline or unclear from dose-response", + "contamination": "Sample contamination detected", + "protocol_deviation": "Protocol deviation recorded for this result", + "borderline_threshold": "Result at borderline of assay threshold", + "replicate_disagreement": "Replicates show high variability", + "missing_negative_control": "Negative control missing or failed", +} + +EXCLUDED_FLAGS: set[str] = {"contamination", "assay_interference"} + + +@dataclass +class ResultQualityReport: + candidate_id: str + flags: list[str] = field(default_factory=list) + quality_level: str = "high" + can_drive_update: bool = True + propagation_action: str = "include" + explanation: str = "" + dry_lab_only: bool = True + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +def assess_result_quality( + candidate_id: str, + flags: list[str], +) -> ResultQualityReport: + """Assess the quality of a single result and determine whether it + can drive a calibration update. + + Parameters + ---------- + candidate_id : str + Identifier for the candidate. + flags : list of str + Quality flags from QUALITY_FLAGS keys. + + Returns + ------- + ResultQualityReport with quality_level, can_drive_update, + propagation_action, and explanation. + """ + unknown = [f for f in flags if f not in QUALITY_FLAGS] + if unknown: + raise ValueError( + f"Unknown quality flags for {candidate_id}: {unknown}. " + f"Valid flags: {list(QUALITY_FLAGS.keys())}" + ) + + has_excluded = any(f in EXCLUDED_FLAGS for f in flags) + + if has_excluded: + return ResultQualityReport( + candidate_id=candidate_id, + flags=flags, + quality_level="excluded", + can_drive_update=False, + propagation_action="exclude", + explanation=( + f"Candidate {candidate_id} excluded due to " + f"irrecoverable quality flags: {flags}. " + f"Contamination or assay interference present." + ), + dry_lab_only=True, + ) + + if len(flags) >= 2: + return ResultQualityReport( + candidate_id=candidate_id, + flags=flags, + quality_level="low", + can_drive_update=False, + propagation_action="exclude", + explanation=( + f"Candidate {candidate_id} has {len(flags)} quality flags " + f"({flags}). Multiple quality issues prevent reliable " + f"calibration update." + ), + dry_lab_only=True, + ) + + if len(flags) == 1: + return ResultQualityReport( + candidate_id=candidate_id, + flags=flags, + quality_level="acceptable", + can_drive_update=True, + propagation_action="include_with_caution", + explanation=( + f"Candidate {candidate_id} has 1 quality flag " + f"({flags[0]}). Result is acceptable but should be " + f"used with caution in calibration." + ), + dry_lab_only=True, + ) + + return ResultQualityReport( + candidate_id=candidate_id, + flags=flags, + quality_level="high", + can_drive_update=True, + propagation_action="include", + explanation=( + f"Candidate {candidate_id} has no quality flags. " + f"Result is suitable for driving calibration updates." + ), + dry_lab_only=True, + ) + + +def filter_results_for_calibration( + results: list[dict], +) -> dict: + """Filter a list of result dicts into quality-based categories. + + Each result dict must have: + candidate_id (str) + flags (list[str]) + + Returns a dict with 'included', 'included_with_caution', 'excluded' + lists (each containing dict representations of ResultQualityReport), + a 'summary' with counts, 'can_drive_update_count', and + 'dry_lab_only' flag. + """ + included: list[dict] = [] + included_with_caution: list[dict] = [] + excluded: list[dict] = [] + + for r in results: + raw_flags = r.get("flags") or [] + report = assess_result_quality( + candidate_id=r["candidate_id"], + flags=raw_flags, + ) + d = report.to_dict() + if report.propagation_action == "include": + included.append(d) + elif report.propagation_action == "include_with_caution": + included_with_caution.append(d) + else: + excluded.append(d) + + total = len(results) + n_included = len(included) + n_included_with_caution = len(included_with_caution) + n_excluded = len(excluded) + can_drive_update_count = n_included + n_included_with_caution + + return { + "included": included, + "included_with_caution": included_with_caution, + "excluded": excluded, + "summary": { + "total": total, + "included": n_included, + "included_with_caution": n_included_with_caution, + "excluded": n_excluded, + }, + "can_drive_update_count": can_drive_update_count, + "dry_lab_only": True, + } + + +def write_result_quality_json( + report: ResultQualityReport | dict, + path: str | Path, +) -> None: + if isinstance(report, ResultQualityReport): + data = report.to_dict() + else: + data = report + with open(path, "w") as f: + json.dump(data, f, indent=2) + + +def write_result_quality_markdown( + report: ResultQualityReport | dict, + path: str | Path, +) -> None: + if isinstance(report, ResultQualityReport): + data = report.to_dict() + else: + data = report + + if "included" in data: + _write_filter_markdown(data, path) + else: + _write_single_report_markdown(data, path) + + +def _write_single_report_markdown(data: dict, path: str | Path) -> None: + lines = [ + "# Result Quality Report", + "", + "> **Dry-lab only.** This report is a computational assessment.", + "", + "## Summary", + "", + f"| Key | Value |", + f"|-----|-------|", + f"| Candidate ID | {data.get('candidate_id', '?')} |", + f"| Quality level | {data.get('quality_level', '?')} |", + f"| Can drive update | {data.get('can_drive_update', '?')} |", + f"| Propagation action | {data.get('propagation_action', '?')} |", + f"| Dry-lab only | {data.get('dry_lab_only', '?')} |", + "", + "## Flags", + "", + ] + flags = data.get("flags", []) + if flags: + for f in flags: + desc = QUALITY_FLAGS.get(f, "Unknown flag") + lines.append(f"- **{f}**: {desc}") + else: + lines.append("No quality flags.") + + lines.extend([ + "", + "## Explanation", + "", + data.get("explanation", ""), + "", + "## Caveats", + "", + "- This report is a computational assessment of result quality.", + "- A high-quality assessment does not confirm biological activity.", + "- All calibration decisions require qualified human review.", + ]) + with open(path, "w") as f: + f.write("\n".join(lines) + "\n") + + +def _write_filter_markdown(data: dict, path: str | Path) -> None: + summary = data.get("summary", {}) + lines = [ + "# Calibration Result Quality Filter", + "", + "> **Dry-lab only.** This report is a computational filter of", + "> lab result quality for calibration eligibility.", + "", + "## Summary", + "", + f"| Metric | Value |", + f"|--------|-------|", + f"| Total results | {summary.get('total', 0)} |", + f"| Included | {summary.get('included', 0)} |", + f"| Included with caution | {summary.get('included_with_caution', 0)} |", + f"| Excluded | {summary.get('excluded', 0)} |", + f"| Can drive update count | {data.get('can_drive_update_count', 0)} |", + f"| Dry-lab only | {data.get('dry_lab_only', True)} |", + "", + "## Included Results", + "", + ] + for r in data.get("included", []): + lines.append( + f"- **{r['candidate_id']}**: {r['quality_level']} " + f"({r['explanation']})" + ) + lines.extend([ + "", + "## Included with Caution", + "", + ]) + for r in data.get("included_with_caution", []): + flags_str = ", ".join(r.get("flags", [])) + lines.append( + f"- **{r['candidate_id']}**: flags=[{flags_str}] " + f"({r['explanation']})" + ) + lines.extend([ + "", + "## Excluded Results", + "", + ]) + for r in data.get("excluded", []): + flags_str = ", ".join(r.get("flags", [])) + lines.append( + f"- **{r['candidate_id']}**: flags=[{flags_str}] " + f"({r['explanation']})" + ) + lines.extend([ + "", + "## Caveats", + "", + "- Quality filtering is a computational assessment only.", + "- Excluded results may still contain useful information for review.", + "- Inclusion does not confirm biological activity or safety.", + "- All calibration decisions require qualified human review.", + ]) + with open(path, "w") as f: + f.write("\n".join(lines) + "\n") diff --git a/src/openamp_foundry/cli/commands/reports.py b/src/openamp_foundry/cli/commands/reports.py index 10f3877b..75e7eea3 100644 --- a/src/openamp_foundry/cli/commands/reports.py +++ b/src/openamp_foundry/cli/commands/reports.py @@ -1030,3 +1030,53 @@ def _run_calibration_overfit_check(args: argparse.Namespace) -> int: "out_md": args.out_md, }, indent=2)) return 0 + + +def _run_result_quality_filter(args: argparse.Namespace) -> int: + """Filter lab results by quality flags for calibration eligibility.""" + import json + from pathlib import Path + + from openamp_foundry.calibration.result_quality import ( + filter_results_for_calibration, + write_result_quality_json, + write_result_quality_markdown, + ) + + results_path = Path(args.results_json) + if not results_path.exists(): + print(json.dumps({ + "status": "error", + "error": f"results JSON not found: {results_path}", + })) + return 2 + + with results_path.open() as f: + results = json.load(f) + + if not isinstance(results, list): + print(json.dumps({ + "status": "error", + "error": "results JSON must be a list of {candidate_id, flags} objects", + })) + return 2 + + filtered = filter_results_for_calibration(results) + + if args.out_json: + write_result_quality_json(filtered, args.out_json) + if args.out_md: + write_result_quality_markdown(filtered, args.out_md) + + print(json.dumps({ + "status": "ok", + "total": filtered["summary"]["total"], + "included": filtered["summary"]["included"], + "included_with_caution": filtered["summary"]["included_with_caution"], + "excluded": filtered["summary"]["excluded"], + "can_drive_update_count": filtered["can_drive_update_count"], + "dry_lab_only": filtered["dry_lab_only"], + "out_json": args.out_json, + "out_md": args.out_md, + }, indent=2)) + return 0 diff --git a/src/openamp_foundry/cli/main.py b/src/openamp_foundry/cli/main.py index c5b21046..29ed10e4 100644 --- a/src/openamp_foundry/cli/main.py +++ b/src/openamp_foundry/cli/main.py @@ -5,7 +5,7 @@ from openamp_foundry.cli.commands.selection import _run_pilot_panel, _run_pilot_confident, _run_diversity_check, _run_select_batch, _run_batch_rationale from openamp_foundry.cli.commands.external import _run_external_predict, _run_external_consensus from openamp_foundry.cli.commands.qc import _run_synthesis_order, _run_presynth_qc -from openamp_foundry.cli.commands.reports import _run_reviewer_questionnaire, _run_ip_report, _run_batch_pack, _run_gold_standard, _run_novelty_check_broad, _run_lab_result_report, _run_calibration_intake, _run_recalibration_gate, _run_recalibration_engine, _run_validate_policy_version, _run_calibration_audit, _run_calibration_overfit_check +from openamp_foundry.cli.commands.reports import _run_reviewer_questionnaire, _run_ip_report, _run_batch_pack, _run_gold_standard, _run_novelty_check_broad, _run_lab_result_report, _run_calibration_intake, _run_recalibration_gate, _run_recalibration_engine, _run_validate_policy_version, _run_calibration_audit, _run_calibration_overfit_check, _run_result_quality_filter from openamp_foundry.cli.commands.gates import _run_gate_check import argparse @@ -1255,6 +1255,31 @@ def build_parser() -> argparse.ArgumentParser: help="Output path for the batch-2 manifest JSON.", ) + # ── Result quality filter ───────────────────────────────────── + result_quality = sub.add_parser( + "result-quality-filter", + help=( + "Filter lab results by quality flags. Low-quality outcomes " + "cannot drive calibration updates — garbage results must not " + "update the scoring model. Dry-lab only." + ), + ) + result_quality.add_argument( + "--results-json", + required=True, + help="Path to JSON file with list of {candidate_id, flags} objects.", + ) + result_quality.add_argument( + "--out-json", + default=None, + help="Optional output path for filtered results JSON.", + ) + result_quality.add_argument( + "--out-md", + default=None, + help="Optional output path for markdown report.", + ) + return parser @@ -1398,6 +1423,9 @@ def main(argv: list[str] | None = None) -> int: if args.command == "calibration-overfit-check": return _run_calibration_overfit_check(args) + if args.command == "result-quality-filter": + return _run_result_quality_filter(args) + parser.error("unknown command") return 2 diff --git a/tests/calibration/test_result_quality.py b/tests/calibration/test_result_quality.py new file mode 100644 index 00000000..0c274454 --- /dev/null +++ b/tests/calibration/test_result_quality.py @@ -0,0 +1,189 @@ +"""Tests for result-quality flag propagation into calibration engine.""" +from __future__ import annotations + +import pytest +from openamp_foundry.calibration.result_quality import ( + QUALITY_FLAGS, + EXCLUDED_FLAGS, + ResultQualityReport, + assess_result_quality, + filter_results_for_calibration, +) + + +class TestAssessResultQuality: + def test_high_quality_no_flags(self): + report = assess_result_quality("AMP001", []) + assert report.quality_level == "high" + assert report.can_drive_update is True + assert report.propagation_action == "include" + assert report.explanation != "" + + def test_contamination_excluded(self): + report = assess_result_quality("AMP002", ["contamination"]) + assert report.quality_level == "excluded" + assert report.can_drive_update is False + assert report.propagation_action == "exclude" + assert report.explanation != "" + + def test_assay_interference_excluded(self): + report = assess_result_quality("AMP003", ["assay_interference"]) + assert report.quality_level == "excluded" + assert report.can_drive_update is False + assert report.propagation_action == "exclude" + assert report.explanation != "" + + def test_two_minor_flags_excluded_low(self): + report = assess_result_quality("AMP004", ["replicate_disagreement", "borderline_threshold"]) + assert report.quality_level == "low" + assert report.can_drive_update is False + assert report.propagation_action == "exclude" + assert report.explanation != "" + + def test_one_minor_flag_acceptable(self): + report = assess_result_quality("AMP005", ["protocol_deviation"]) + assert report.quality_level == "acceptable" + assert report.can_drive_update is True + assert report.propagation_action == "include_with_caution" + assert report.explanation != "" + + def test_can_drive_update_true_high(self): + report = assess_result_quality("AMP006", []) + assert report.can_drive_update is True + + def test_can_drive_update_true_acceptable(self): + report = assess_result_quality("AMP007", ["ambiguous_activity"]) + assert report.can_drive_update is True + + def test_can_drive_update_false_excluded(self): + report = assess_result_quality("AMP008", ["contamination"]) + assert report.can_drive_update is False + + def test_can_drive_update_false_low(self): + report = assess_result_quality("AMP009", ["low_sample_quality", "protocol_deviation"]) + assert report.can_drive_update is False + + def test_dry_lab_only_always_true(self): + for flags in ([], ["contamination"], ["protocol_deviation"], + ["replicate_disagreement", "borderline_threshold"]): + report = assess_result_quality("AMP010", flags) + assert report.dry_lab_only is True + + def test_unknown_flag_raises(self): + with pytest.raises(ValueError, match="Unknown quality flags"): + assess_result_quality("AMP011", ["nonexistent_flag"]) + + def test_explanation_non_empty(self): + report = assess_result_quality("AMP012", []) + assert len(report.explanation) > 0 + + def test_explanation_mentions_candidate_id(self): + report = assess_result_quality("AMP013", ["contamination"]) + assert "AMP013" in report.explanation + + def test_to_dict_includes_all_fields(self): + report = assess_result_quality("AMP014", ["missing_negative_control"]) + d = report.to_dict() + assert d["candidate_id"] == "AMP014" + assert d["quality_level"] == "acceptable" + assert d["can_drive_update"] is True + assert d["propagation_action"] == "include_with_caution" + assert d["dry_lab_only"] is True + assert "explanation" in d + assert d["flags"] == ["missing_negative_control"] + + def test_contamination_with_other_flags_still_excluded(self): + report = assess_result_quality("AMP015", ["contamination", "protocol_deviation"]) + assert report.quality_level == "excluded" + assert report.can_drive_update is False + + def test_assay_interference_with_other_flags_still_excluded(self): + report = assess_result_quality("AMP016", ["assay_interference", "ambiguous_activity"]) + assert report.quality_level == "excluded" + assert report.can_drive_update is False + + +class TestFilterResultsForCalibration: + def test_empty_results(self): + result = filter_results_for_calibration([]) + assert result["summary"]["total"] == 0 + assert result["can_drive_update_count"] == 0 + assert result["dry_lab_only"] is True + + def test_summary_counts(self): + results = [ + {"candidate_id": "A", "flags": []}, + {"candidate_id": "B", "flags": ["contamination"]}, + {"candidate_id": "C", "flags": ["protocol_deviation"]}, + {"candidate_id": "D", "flags": ["low_sample_quality", "replicate_disagreement"]}, + ] + result = filter_results_for_calibration(results) + assert result["summary"]["total"] == 4 + assert result["summary"]["included"] == 1 + assert result["summary"]["included_with_caution"] == 1 + assert result["summary"]["excluded"] == 2 + assert result["can_drive_update_count"] == 2 + + def test_included_has_correct_action(self): + results = [ + {"candidate_id": "X", "flags": []}, + ] + result = filter_results_for_calibration(results) + assert len(result["included"]) == 1 + assert result["included"][0]["candidate_id"] == "X" + assert result["included"][0]["propagation_action"] == "include" + + def test_included_with_caution_has_correct_action(self): + results = [ + {"candidate_id": "Y", "flags": ["ambiguous_activity"]}, + ] + result = filter_results_for_calibration(results) + assert len(result["included_with_caution"]) == 1 + assert result["included_with_caution"][0]["candidate_id"] == "Y" + assert result["included_with_caution"][0]["propagation_action"] == "include_with_caution" + + def test_excluded_has_correct_action(self): + results = [ + {"candidate_id": "Z", "flags": ["contamination"]}, + ] + result = filter_results_for_calibration(results) + assert len(result["excluded"]) == 1 + assert result["excluded"][0]["candidate_id"] == "Z" + assert result["excluded"][0]["propagation_action"] == "exclude" + + def test_can_drive_update_count_matches(self): + results = [ + {"candidate_id": "A", "flags": []}, + {"candidate_id": "B", "flags": ["protocol_deviation"]}, + {"candidate_id": "C", "flags": ["contamination"]}, + {"candidate_id": "D", "flags": ["low_sample_quality", "replicate_disagreement"]}, + ] + result = filter_results_for_calibration(results) + assert result["can_drive_update_count"] == 2 + + def test_missing_flags_defaults_to_empty(self): + results = [ + {"candidate_id": "M", "flags": None}, + ] + result = filter_results_for_calibration(results) + assert result["summary"]["included"] == 1 + + +class TestExcludedFlags: + def test_excluded_flags_set(self): + assert "contamination" in EXCLUDED_FLAGS + assert "assay_interference" in EXCLUDED_FLAGS + + def test_excluded_flags_size(self): + assert len(EXCLUDED_FLAGS) == 2 + + +class TestQualityFlags: + def test_all_flags_have_descriptions(self): + for flag, desc in QUALITY_FLAGS.items(): + assert isinstance(flag, str) + assert isinstance(desc, str) + assert len(desc) > 0 + + def test_known_flags_count(self): + assert len(QUALITY_FLAGS) == 8 diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index 85afa355..80c07da4 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -2,9 +2,9 @@ import subprocess import sys -# Baseline: 2995 tests as of Loop 84 (2026-07-09) -- Phase G G6 (calibration-overfit warning) +# Baseline: 3022 tests as of Loop 85 (2026-07-09) -- Phase G G7 (result-quality flag propagation) # Raise threshold if legitimate tests are removed; lower threshold is okay. -BASELINE = 2995 +BASELINE = 3022 TOLERANCE = 0.05 # allow 5% fluctuation