From 75878cdadd5b0153c96581f979fe07158d047281 Mon Sep 17 00:00:00 2001 From: OpenCode Date: Thu, 9 Jul 2026 09:46:02 +0700 Subject: [PATCH] feat: rejection reason taxonomy schema (Phase F F3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Machine-readable rejection reason taxonomy for OpenAMP candidate failures. Covers 21 reasons across 6 pipeline stages with severity, evidence impact, and stage classification. - schemas/rejection_taxonomy.schema.json — JSON Schema for taxonomy entries - examples/rejection_taxonomy_example.json — 21-entry complete taxonomy - docs/evidence/REJECTION_TAXONOMY.md — comprehensive reference doc - tests/evidence/test_rejection_taxonomy_schema.py — 19 tests - docs/evidence/METRICS_CURRENT.md — v0.5.74 entry (2698 tests) - docs/research/ROADMAP.md — v0.5.74 entry - tests/test_test_count_regression.py — baseline 2698 --- docs/evidence/METRICS_CURRENT.md | 6 +- docs/evidence/REJECTION_TAXONOMY.md | 128 +++++++-- docs/research/ROADMAP.md | 40 +++ examples/rejection_taxonomy_example.json | 244 ++++++++++++++++++ schemas/rejection_taxonomy.schema.json | 119 +++++++++ .../test_rejection_taxonomy_schema.py | 157 +++++++++++ tests/test_test_count_regression.py | 4 +- 7 files changed, 673 insertions(+), 25 deletions(-) create mode 100644 examples/rejection_taxonomy_example.json create mode 100644 schemas/rejection_taxonomy.schema.json create mode 100644 tests/evidence/test_rejection_taxonomy_schema.py diff --git a/docs/evidence/METRICS_CURRENT.md b/docs/evidence/METRICS_CURRENT.md index a63983a3..5b52290f 100644 --- a/docs/evidence/METRICS_CURRENT.md +++ b/docs/evidence/METRICS_CURRENT.md @@ -5,7 +5,9 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **Purpose:** One authoritative table of current pipeline metrics. If any doc disagrees > with this file, this file wins. Updated whenever benchmark/benchmark config changes. > -> **Last updated:** 2026-07-09 (Phase E safety release, preregistration, packet generator — v0.5.73) +> **Last updated:** 2026-07-09 (Phase F rejection taxonomy — v0.5.74) +> **New in v0.5.74:** Rejection reason taxonomy schema (F3) — machine-readable taxonomy with severity, evidence_impact, and applies_at_stage fields. Comprehensive REJECTION_TAXONOMY.md doc. 19 tests. 2698 total. +> > **New in v0.5.73:** Safety-release decision schema (E4), pilot pre-registration schema (E5), review packet generator CLI (E6). 40 tests. 2679 total. > > **New in v0.5.72:** External review packet schema (E1), example packet (E2), reviewer questionnaire schema (E3). 27 tests. 2666 total. @@ -56,7 +58,7 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **New in v0.5.31:** Added dipeptide-order features for sequence-order awareness. `dipeptide_order_score` achieves AUROC 0.7861 on AMP-vs-scrambled discrimination — the strongest order-dependent feature in the pipeline. Only 7/31 features survive scrambling (amphipathicity/helix-wheel + dipeptide). All composition features are purely position-independent (exactly 0.5000 AUROC on scrambled test). > **New in v0.5.30:** Easy baseline benchmark added — charge density alone (AUROC 0.8166) outperforms the full pipeline ensemble (0.7792) on AMP-vs-Swiss-Prot-decoy discrimination. Honest finding documented: expected because pipeline optimizes for safety, not raw discrimination. > **New in v0.5.29:** Expanded benchmark to 500 AMPs + 500 composition-matched decoys (n=1000). AUROC 0.7792 (CI₉₅: 0.7505–0.8065) confirms signal generalizes. Cluster-aware CI: 0.746–0.8102. Representative AUROC: 0.778. Standard benchmark (n=191) retained for backward comparison. -> **Pipeline version:** v0.5.73 +> **Pipeline version:** v0.5.74 > **Branch:** main --- diff --git a/docs/evidence/REJECTION_TAXONOMY.md b/docs/evidence/REJECTION_TAXONOMY.md index 6662c989..b9cf1302 100644 --- a/docs/evidence/REJECTION_TAXONOMY.md +++ b/docs/evidence/REJECTION_TAXONOMY.md @@ -2,32 +2,118 @@ ## Purpose -Standardized categories for why candidates are rejected by the pipeline. Enables analysis of failure patterns across batches. +Standardized, machine-readable categories for why candidates are rejected by +the OpenAMP pipeline. Enables analysis of failure patterns across batches, +consistent archive entries, and evidence-aware claim boundaries for rejected +candidates. -## Categories +## Schema -| Category | Description | Example | -|----------|-------------|---------| -| `sequence_invalid` | Sequence contains non-canonical amino acids or unparseable symbols. | "Sequence contains 'X' at position 5." | -| `length_out_of_range` | Sequence length outside configured min/max filter bounds. | "Length 6 outside filter range [8, 35]." | -| `novelty_below_threshold` | Similarity to a known reference exceeds the novelty cutoff. | "Nearest reference similarity 0.85 exceeds max 0.70." | -| `safety_risk_threshold` | Safety score below minimum (or risk score above maximum). | "Safety score 0.31 below minimum 0.50." | -| `synthesis_feasibility_low` | Synthesis feasibility score below viability threshold. | "Synthesis score 0.20 — multiple Trp residues with oxidation risk." | -| `disagreement_too_high` | Scorer disagreement exceeds max tolerance. | "Disagreement 0.85 exceeds max 0.50." | -| `diversity_filter_prune` | Removed by diversity selection to avoid near-duplicates. | "Similarity 0.82 to already-selected candidate." | -| `simulation_uncertainty_high` | Simulation proxy uncertainty exceeds 0.5 threshold. | "Max simulation uncertainty 0.92 exceeds 0.5." | -| `toxicity_risk_flagged` | Hemolysis or cytotoxicity risk above acceptable level. | "Hemolysis risk 0.89 flagged as HIGH." | +A machine-readable JSON Schema defines the taxonomy structure: -## Usage +- **Source:** `schemas/rejection_taxonomy.schema.json` +- **Example:** `examples/rejection_taxonomy_example.json` -Rejection categories are recorded in: -- Pipeline output JSONL (`known_failure_modes` field) -- Failed-candidate report output (if enabled) -- Negative-result archive entries +Every taxonomy entry includes: `code`, `category`, `label`, `description`, +`severity`, `evidence_impact`, `applies_at_stage`, and `related_reason_category` +(mapping to `schemas/negative_result_entry.schema.json`). + +## Taxonomy hierarchy + +Rejection reasons are grouped by pipeline stage. Each entry has a severity +and an evidence impact on the proof ladder. + +### Pre-selection (before scoring) + +| Code | Severity | Evidence impact | Description | +|------|:--------:|:---------------:|-------------| +| `PRE_SEQ_INVALID` | hard | claim_unsupported | Non-canonical amino acids or unparseable symbols | +| `PRE_LENGTH_OUT` | hard | claim_unsupported | Length outside configured range | +| `PRE_PATENT_NEAR` | soft | downgrade_by_1 | Similarity to a known reference exceeds novelty cutoff | + +### Pipeline scoring + +| Code | Severity | Evidence impact | Description | +|------|:--------:|:---------------:|-------------| +| `PIPE_ACTIVITY_LOW` | soft | downgrade_by_1 | Activity score below minimum gate threshold | +| `PIPE_SAFETY_LOW` | soft | downgrade_by_1 | Safety score below minimum threshold | +| `PIPE_NOVELTY_LOW` | soft | downgrade_by_1 | Novelty score below minimum threshold | +| `PIPE_SYNTHESIS_LOW` | soft | downgrade_by_1 | Synthesis feasibility below viability threshold | +| `PIPE_DISAGREEMENT_HI` | informational | no_impact | Scorer disagreement exceeds maximum tolerance | +| `PIPE_ENSEMBLE_LOW` | soft | downgrade_by_1 | Composite ensemble score below synthesis gate threshold | +| `PIPE_SIM_UNCERTAINTY` | informational | no_impact | Simulation proxy uncertainty exceeds 0.5 threshold | + +### Diversity and selection + +| Code | Severity | Evidence impact | Description | +|------|:--------:|:---------------:|-------------| +| `DIV_TOO_SIMILAR` | soft | no_impact | Near-duplicate of already-selected candidate | +| `DIV_FAMILY_CAP` | soft | no_impact | Per-family panel cap reached | +| `DIV_BUDGET` | informational | no_impact | Budget or scope limitation | + +### Reviewer rejection + +| Code | Severity | Evidence impact | Description | +|------|:--------:|:---------------:|-------------| +| `REV_EXPERT_REJECT` | hard | downgrade_by_2 | Qualified domain expert recommended against progression | +| `REV_SAFETY_CONCERN` | hard | downgrade_by_2 | Human safety reviewer flagged structural or dual-use concern | +| `REV_INSUFFICIENT` | soft | downgrade_by_1 | Evidence package does not meet minimum reviewer standards | + +### Lab result + +| Code | Severity | Evidence impact | Description | +|------|:--------:|:---------------:|-------------| +| `LAB_INACTIVE` | hard | downgrade_by_2 | Lab-confirmed inactive (MIC above cutoff) | +| `LAB_TOXIC` | hard | downgrade_by_2 | Lab-confirmed hemolytic or cytotoxic | +| `LAB_CONTROL_FAIL` | hard | claim_unsupported | Assay controls failed — batch invalidated | +| `LAB_SYNTHESIS_FAIL` | hard | downgrade_by_2 | Could not be synthesized at sufficient purity | + +### Lifecycle + +| Code | Severity | Evidence impact | Description | +|------|:--------:|:---------------:|-------------| +| `LIFECYCLE_SUPERSEDED` | informational | no_impact | Replaced by a higher-scoring or better-characterised candidate | +| `LIFECYCLE_WITHDRAWN` | informational | no_impact | Withdrawn for non-technical reasons (scope, budget, partner feedback) | + +## Severity categories + +| Severity | Meaning | +|----------|---------| +| `hard` | Final rejection. The candidate cannot proceed regardless of threshold adjustment. Requires new evidence to overturn. | +| `soft` | Conditional on current thresholds. A policy or threshold change could convert this to a pass. The rejection record should document the specific threshold. | +| `informational` | Context-dependent. The candidate was rejected for reasons external to its own merits (budget, diversity, uncertainty). May still be valid in a different context. | + +## Evidence impact + +| Impact | Meaning | +|--------|---------| +| `claim_unsupported` | No claim about this candidate can be made at any proof-ladder level. The input itself is invalid or the assay failed. | +| `downgrade_by_2` | The highest supportable proof-ladder level drops by 2. For example, from Level 4 (multi-signal candidate evidence) to Level 2 (baseline-triaged). | +| `downgrade_by_1` | The highest supportable proof-ladder level drops by 1. | +| `no_impact` | The rejection does not affect the candidate's claim level. The candidate remains valid at its current evidence level (e.g., diversity pruning). | + +## Usage in tools + +| Artifact | How the taxonomy is used | +|----------|--------------------------| +| `schemas/negative_result_entry.schema.json` | `reason_category` field maps to the taxonomy's top-level categories | +| Pipeline output JSONL | `known_failure_modes` field records the applying rejection codes | +| Failed-candidate report generator | Reads taxonomy to produce structured rejection summaries | +| Negative-result archive validator | Checks `reason_category` against allowed values | +| External review packet | Rejection codes and rationales are included for reviewed candidates | + +## Cross-references + +- `docs/evidence/TAXONOMY_EXCLUSION_REASONS.md` — Lightweight exclusion-reason table for selection workflows +- `docs/evidence/NEGATIVE_RESULT_ARCHIVE.md` — Archive format that stores rejection records +- `schemas/negative_result_entry.schema.json` — Rejection entry schema with reason_category enum +- `schemas/rejection_taxonomy.schema.json` — Machine-readable taxonomy schema ## Non-goals This taxonomy does not cover: -- Wet-lab assay failures (e.g., synthesis failure, control failure) — see NEGATIVE_RESULT_ARCHIVE.md -- Human review rejections — see EXTERNAL_REVIEW_PACKET.md -- Safety or dual-use review rejections — see DUAL_USE_REVIEW.md + +- Wet-lab assay design decisions — see `docs/review/WET_LAB_HANDOFF.md` +- Human review deliberation format — see `docs/review/EXTERNAL_REVIEW_PACKET.md` +- Safety or dual-use review policy — see `docs/trust/SAFETY_DOC_AUDIT.md` +- Recalibration policy decisions — see `docs/evidence/CALIBRATION_POLICY.md` diff --git a/docs/research/ROADMAP.md b/docs/research/ROADMAP.md index f5a9e088..9c941ed0 100644 --- a/docs/research/ROADMAP.md +++ b/docs/research/ROADMAP.md @@ -1,5 +1,45 @@ # Roadmap +## v0.5.74 — Loop 73: Phase F F3 — Rejection Reason Taxonomy Schema ✓ (2026-07-09) + +Negative-result infrastructure begins: machine-readable rejection reason taxonomy +with severity, evidence impact, and stage classification. Foundation for F4 (report +generator) and F5 (publication filter). + +Changes: +- `schemas/rejection_taxonomy.schema.json` (F3) — JSON Schema Draft 2020-12 defining + rejection taxonomy entries. Required fields: code (pattern `^[A-Z]+_[A-Za-z_]+$`), + category (pre_selection/pipeline/diversity/reviewer/lab/lifecycle), label, + description (min 10 chars), severity (hard/soft/informational), evidence_impact + (claim_unsupported/downgrade_by_1/downgrade_by_2/no_impact), applies_at_stage + (pre_selection/after_scoring/after_diversity/after_review/after_lab/any), + related_reason_category mapping to negative_result_entry schema. Optional: example. +- `examples/rejection_taxonomy_example.json` (F3) — Complete taxonomy with 21 entries + covering all 6 categories: PRE_SEQ_INVALID, PRE_LENGTH_OUT, PRE_PATENT_NEAR, + PIPE_ACTIVITY_LOW, PIPE_SAFETY_LOW, PIPE_NOVELTY_LOW, PIPE_SYNTHESIS_LOW, + PIPE_DISAGREEMENT_HI, PIPE_ENSEMBLE_LOW, PIPE_SIM_UNCERTAINTY, DIV_TOO_SIMILAR, + DIV_FAMILY_CAP, DIV_BUDGET, REV_EXPERT_REJECT, REV_SAFETY_CONCERN, REV_INSUFFICIENT, + LAB_INACTIVE, LAB_TOXIC, LAB_CONTROL_FAIL, LAB_SYNTHESIS_FAIL, LIFECYCLE_SUPERSEDED, + LIFECYCLE_WITHDRAWN. +- `docs/evidence/REJECTION_TAXONOMY.md` — Comprehensive reference doc with hierarchy + tables, severity definitions, evidence impact definitions, usage in tools, and + cross-references to related schemas/docs. +- `tests/evidence/test_rejection_taxonomy_schema.py` — 19 tests covering valid entry, + missing required fields, invalid enums (category, severity, evidence_impact, + applies_at_stage, related_reason_category), code pattern validation, description + length, multiple entries, optional fields, example file validation, code uniqueness, + and stage coverage completeness. + +Honest boundaries: +- The taxonomy defines valid rejection reasons but does not enforce that tools + actually use them — adoption requires downstream tool updates (F4, F5). +- Hard rejections require new evidence to overturn, but the taxonomy does not + define what counts as sufficient new evidence. +- The `evidence_impact` field describes claim-level effects; it does not guarantee + that reviewers will apply the specified downgrade in practice. +- Severity is a guideline; final rejection authority rests with human reviewers + for class C/D decisions. + ## v0.5.73 — Loop 72: Phase E E4-E6 — Safety Release, Preregistration, Packet CLI ✓ (2026-07-09) External review infrastructure: human-gated release decisions, pre-registered pilot diff --git a/examples/rejection_taxonomy_example.json b/examples/rejection_taxonomy_example.json new file mode 100644 index 00000000..70d08221 --- /dev/null +++ b/examples/rejection_taxonomy_example.json @@ -0,0 +1,244 @@ +[ + { + "code": "PRE_SEQ_INVALID", + "category": "pre_selection", + "label": "Invalid sequence", + "description": "Sequence contains non-canonical amino-acid symbols or unparseable characters that prevent pipeline scoring.", + "severity": "hard", + "evidence_impact": "claim_unsupported", + "applies_at_stage": "pre_selection", + "related_reason_category": "pre_selection_reject", + "example": "Sequence contains 'X' at position 5 and 'B' at position 12." + }, + { + "code": "PRE_LENGTH_OUT", + "category": "pre_selection", + "label": "Length outside range", + "description": "Sequence length falls below minimum or above maximum configured filter bounds.", + "severity": "hard", + "evidence_impact": "claim_unsupported", + "applies_at_stage": "pre_selection", + "related_reason_category": "pre_selection_reject", + "example": "Length 6 outside filter range [8, 50]." + }, + { + "code": "PRE_PATENT_NEAR", + "category": "pre_selection", + "label": "Patent proximity", + "description": "Sequence similarity to a patented or known AMP reference exceeds the configured novelty cutoff.", + "severity": "soft", + "evidence_impact": "downgrade_by_1", + "applies_at_stage": "pre_selection", + "related_reason_category": "pre_selection_reject", + "example": "Nearest reference similarity 0.91 exceeds novelty cutoff 0.70." + }, + { + "code": "PIPE_ACTIVITY_LOW", + "category": "pipeline", + "label": "Activity score below threshold", + "description": "The pipeline activity score falls below the configured minimum gate threshold, indicating low predicted antimicrobial potential.", + "severity": "soft", + "evidence_impact": "downgrade_by_1", + "applies_at_stage": "after_scoring", + "related_reason_category": "pre_selection_reject", + "example": "Activity score 0.42 below minimum threshold 0.50." + }, + { + "code": "PIPE_SAFETY_LOW", + "category": "pipeline", + "label": "Safety score below threshold", + "description": "The pipeline safety score falls below the configured minimum, indicating elevated predicted hemolysis or cytotoxicity risk.", + "severity": "soft", + "evidence_impact": "downgrade_by_1", + "applies_at_stage": "after_scoring", + "related_reason_category": "pre_selection_reject", + "example": "Safety score 0.31 below minimum threshold 0.50 (driven by high hemolysis proxy 0.89)." + }, + { + "code": "PIPE_NOVELTY_LOW", + "category": "pipeline", + "label": "Novelty score below threshold", + "description": "The novelty score falls below the configured minimum, indicating the candidate is too similar to known sequences in the novelty reference database.", + "severity": "soft", + "evidence_impact": "downgrade_by_1", + "applies_at_stage": "after_scoring", + "related_reason_category": "pre_selection_reject", + "example": "Novelty score 0.35 below minimum threshold 0.40 (near-duplicate of known AMP)." + }, + { + "code": "PIPE_SYNTHESIS_LOW", + "category": "pipeline", + "label": "Synthesis feasibility low", + "description": "The synthesis feasibility score falls below the configured viability threshold, indicating challenging solid-phase peptide synthesis.", + "severity": "soft", + "evidence_impact": "downgrade_by_1", + "applies_at_stage": "after_scoring", + "related_reason_category": "pre_selection_reject", + "example": "Synthesis score 0.20 — multiple Trp residues with oxidation risk, aggregation propensity above threshold." + }, + { + "code": "PIPE_DISAGREEMENT_HI", + "category": "pipeline", + "label": "Scorer disagreement high", + "description": "Disagreement between independent scorers exceeds the configured maximum tolerance, indicating uncertainty about the candidate profile.", + "severity": "informational", + "evidence_impact": "no_impact", + "applies_at_stage": "after_scoring", + "related_reason_category": "pre_selection_reject", + "example": "Scorer disagreement 0.72 exceeds maximum tolerance 0.50." + }, + { + "code": "PIPE_ENSEMBLE_LOW", + "category": "pipeline", + "label": "Ensemble gate below threshold", + "description": "The composite ensemble score falls below the configured synthesis gate threshold, combining multiple signal dimensions.", + "severity": "soft", + "evidence_impact": "downgrade_by_1", + "applies_at_stage": "after_scoring", + "related_reason_category": "pre_selection_reject", + "example": "Ensemble score 0.55 below synthesis gate threshold 0.65." + }, + { + "code": "PIPE_SIM_UNCERTAINTY", + "category": "pipeline", + "label": "Simulation uncertainty high", + "description": "Simulation proxy uncertainty exceeds the configured threshold (0.5), preventing simulation outputs from affecting selection.", + "severity": "informational", + "evidence_impact": "no_impact", + "applies_at_stage": "after_scoring", + "related_reason_category": "pre_selection_reject", + "example": "Max simulation uncertainty 0.92 exceeds 0.5 threshold — simulation outputs are informational only." + }, + { + "code": "DIV_TOO_SIMILAR", + "category": "diversity", + "label": "Near-duplicate of selected candidate", + "description": "Candidate was removed during diversity selection because it exceeds the similarity threshold to an already-selected candidate.", + "severity": "soft", + "evidence_impact": "no_impact", + "applies_at_stage": "after_diversity", + "related_reason_category": "pre_selection_reject", + "example": "Sequence similarity 0.82 to already-selected candidate WAVE0.5-003 exceeds diversity threshold 0.70." + }, + { + "code": "DIV_FAMILY_CAP", + "category": "diversity", + "label": "Per-family cap reached", + "description": "Candidate belongs to a scaffold family that has already reached its maximum panel allocation.", + "severity": "soft", + "evidence_impact": "no_impact", + "applies_at_stage": "after_diversity", + "related_reason_category": "pre_selection_reject", + "example": "Family SEED-012 has reached its cap of 2 panel slots; candidate SEED-012_VAR_005 excluded." + }, + { + "code": "DIV_BUDGET", + "category": "diversity", + "label": "Budget or scope limitation", + "description": "Candidate excluded because panel size or budget constraints prevent testing all qualifying candidates.", + "severity": "informational", + "evidence_impact": "no_impact", + "applies_at_stage": "after_diversity", + "related_reason_category": "selected_untested", + "example": "Panel budget limited to 24 candidates; lowest-ranked qualifying candidate excluded." + }, + { + "code": "REV_EXPERT_REJECT", + "category": "reviewer", + "label": "Expert reviewer rejected", + "description": "A qualified domain expert reviewed the candidate evidence package and recommended against proceeding.", + "severity": "hard", + "evidence_impact": "downgrade_by_2", + "applies_at_stage": "after_review", + "related_reason_category": "selected_untested", + "example": "Expert reviewer: 'Candidate shows no mechanism novelty — insufficient evidence to warrant synthesis.'" + }, + { + "code": "REV_SAFETY_CONCERN", + "category": "reviewer", + "label": "Safety review concern", + "description": "A human safety reviewer flagged the candidate based on structural alerts, dual-use potential, or toxicity profile not captured by computational safety scoring.", + "severity": "hard", + "evidence_impact": "downgrade_by_2", + "applies_at_stage": "after_review", + "related_reason_category": "selected_untested", + "example": "Safety reviewer: 'Candidate contains a predicted hemolytic motif that the safety scorer is known to miss (melittin-class blind spot).'" + }, + { + "code": "REV_INSUFFICIENT", + "category": "reviewer", + "label": "Insufficient evidence", + "description": "The candidate evidence package does not meet the reviewer's minimum requirements for progression — missing data, weak signal, or unresolved uncertainty.", + "severity": "soft", + "evidence_impact": "downgrade_by_1", + "applies_at_stage": "after_review", + "related_reason_category": "selected_untested", + "example": "Reviewer: 'No hemolysis data; safety score is based on proxy features only. Recommend hold until selectivity benchmark is available.'" + }, + { + "code": "LAB_INACTIVE", + "category": "lab", + "label": "Lab-confirmed inactive", + "description": "Candidate was tested in an approved assay and confirmed inactive (MIC above cutoff or no observable antimicrobial activity).", + "severity": "hard", + "evidence_impact": "downgrade_by_2", + "applies_at_stage": "after_lab", + "related_reason_category": "lab_inactive", + "example": "MIC > 128 ug/mL against E. coli ATCC 25922, S. aureus ATCC 29213, and P. aeruginosa ATCC 27853." + }, + { + "code": "LAB_TOXIC", + "category": "lab", + "label": "Lab-confirmed toxic", + "description": "Candidate passed computational safety gates but showed hemolysis or cytotoxicity in assay.", + "severity": "hard", + "evidence_impact": "downgrade_by_2", + "applies_at_stage": "after_lab", + "related_reason_category": "lab_toxic", + "example": "100% hemolysis at 50 uM (RBC assay) — computational safety score was 0.85 (false negative)." + }, + { + "code": "LAB_CONTROL_FAIL", + "category": "lab", + "label": "Assay control failure", + "description": "Assay batch failed positive or negative controls, invalidating all results. Archived for transparency but flagged as non-interpretable.", + "severity": "hard", + "evidence_impact": "claim_unsupported", + "applies_at_stage": "after_lab", + "related_reason_category": "control_failure", + "example": "Positive control (melittin) showed only 12% hemolysis at 50 uM — batch invalidated." + }, + { + "code": "LAB_SYNTHESIS_FAIL", + "category": "lab", + "label": "Synthesis failure", + "description": "Candidate could not be synthesized at sufficient purity or yield for assay testing.", + "severity": "hard", + "evidence_impact": "downgrade_by_2", + "applies_at_stage": "after_lab", + "related_reason_category": "synthesis_failure", + "example": "SPPS failed at residue 7 (Trp aggregation); maximum achievable purity 65% after 3 attempts." + }, + { + "code": "LIFECYCLE_SUPERSEDED", + "category": "lifecycle", + "label": "Superseded by better candidate", + "description": "Candidate was replaced by a later candidate that scores higher or addresses a limitation of the original selection.", + "severity": "informational", + "evidence_impact": "no_impact", + "applies_at_stage": "any", + "related_reason_category": "selected_untested", + "example": "Superseded by WAVE0.5-031 which achieves 0.88 ensemble score (vs 0.76) with improved safety profile." + }, + { + "code": "LIFECYCLE_WITHDRAWN", + "category": "lifecycle", + "label": "Withdrawn by team", + "description": "Candidate was withdrawn from consideration by the research team for non-technical reasons (scope change, strategic reprioritisation, partner feedback).", + "severity": "informational", + "evidence_impact": "no_impact", + "applies_at_stage": "any", + "related_reason_category": "selected_untested", + "example": "Withdrawn after partner requested focus on lower-hemolysis families only." + } +] diff --git a/schemas/rejection_taxonomy.schema.json b/schemas/rejection_taxonomy.schema.json new file mode 100644 index 00000000..1110e274 --- /dev/null +++ b/schemas/rejection_taxonomy.schema.json @@ -0,0 +1,119 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://openamp-foundry.org/schemas/rejection_taxonomy.schema.json", + "title": "OpenAMP Rejection Reason Taxonomy", + "description": "Machine-readable rejection reason taxonomy for OpenAMP candidate failures. Each entry defines a valid rejection reason code with its category, severity, evidence impact, and the stage at which it applies. Tools read this schema as the single source of truth for valid rejection reasons across the pipeline.", + "type": "array", + "items": { + "$ref": "#/$defs/taxonomy_entry" + }, + "minItems": 1, + "uniqueItems": true, + "$defs": { + "taxonomy_entry": { + "type": "object", + "required": [ + "code", + "category", + "label", + "description", + "severity", + "evidence_impact", + "applies_at_stage" + ], + "properties": { + "code": { + "type": "string", + "pattern": "^[A-Z]+_[A-Za-z_]+$", + "description": "Machine-readable unique code for this rejection reason (e.g. PIPE_ACTIVITY_LOW)." + }, + "category": { + "type": "string", + "enum": [ + "pre_selection", + "pipeline", + "diversity", + "reviewer", + "lab", + "lifecycle" + ], + "description": "Top-level phase when this rejection typically occurs." + }, + "label": { + "type": "string", + "minLength": 3, + "description": "Short human-readable label for reports and summaries." + }, + "description": { + "type": "string", + "minLength": 10, + "description": "Detailed explanation of what this rejection means." + }, + "severity": { + "type": "string", + "enum": ["hard", "soft", "informational"], + "description": "How definitive or irreversible this rejection is. 'hard' = final (e.g. sequence invalid), 'soft' = conditional on current thresholds (e.g. safety score below gate), 'informational' = context-dependent (e.g. budget limit)." + }, + "evidence_impact": { + "type": "string", + "enum": [ + "claim_unsupported", + "downgrade_by_1", + "downgrade_by_2", + "no_impact" + ], + "description": "Effect on the proof-ladder level if this reason applies. 'claim_unsupported' = claim cannot be made at any level, 'downgrade_by_N' = lower the claim level by N, 'no_impact' = claims at current level remain valid." + }, + "applies_at_stage": { + "type": "string", + "enum": [ + "pre_selection", + "after_scoring", + "after_diversity", + "after_review", + "after_lab", + "any" + ], + "description": "The pipeline stage at which this rejection reason first becomes applicable." + }, + "related_reason_category": { + "type": "string", + "enum": [ + "pre_selection_reject", + "selected_untested", + "lab_inactive", + "lab_toxic", + "control_failure", + "synthesis_failure" + ], + "description": "Maps to the reason_category in schemas/negative_result_entry.schema.json for archive entries." + }, + "example": { + "type": "string", + "minLength": 5, + "description": "Brief example illustrating when this rejection would apply." + } + }, + "examples": [ + { + "code": "PIPE_ACTIVITY_LOW", + "category": "pipeline", + "label": "Activity score below threshold", + "description": "The pipeline activity score falls below the configured minimum gate threshold, indicating low predicted antimicrobial potential.", + "severity": "soft", + "evidence_impact": "downgrade_by_1", + "applies_at_stage": "after_scoring", + "related_reason_category": "pre_selection_reject", + "example": "Activity score 0.42 below minimum threshold 0.50." + } + ] + }, + "all_reasons": { + "type": "array", + "description": "The complete set of valid rejection reason codes registered in this schema version.", + "items": { + "type": "string" + } + } + } +} diff --git a/tests/evidence/test_rejection_taxonomy_schema.py b/tests/evidence/test_rejection_taxonomy_schema.py new file mode 100644 index 00000000..0f79c5fe --- /dev/null +++ b/tests/evidence/test_rejection_taxonomy_schema.py @@ -0,0 +1,157 @@ +"""Tests for rejection_taxonomy.schema.json.""" +from __future__ import annotations + +import json +from pathlib import Path + +import jsonschema +import pytest + +from openamp_foundry.evidence.schemas import validate_json_schema +from openamp_foundry.utils.io import read_json + +_SCHEMA_DIR = Path(__file__).parents[2] / "schemas" +_EXAMPLES_DIR = Path(__file__).parents[2] / "examples" + +TAXONOMY_SCHEMA = _SCHEMA_DIR / "rejection_taxonomy.schema.json" +TAXONOMY_EXAMPLE = _EXAMPLES_DIR / "rejection_taxonomy_example.json" + + +def _valid_entry() -> dict: + return { + "code": "PIPE_ACTIVITY_LOW", + "category": "pipeline", + "label": "Activity score below threshold", + "description": "The pipeline activity score falls below the configured minimum gate threshold.", + "severity": "soft", + "evidence_impact": "downgrade_by_1", + "applies_at_stage": "after_scoring", + "related_reason_category": "pre_selection_reject", + "example": "Activity score 0.42 below minimum threshold 0.50.", + } + + +def _valid_taxonomy() -> list[dict]: + return [_valid_entry()] + + +class TestRejectionTaxonomySchema: + def test_schema_file_exists(self): + assert TAXONOMY_SCHEMA.exists() + + def test_valid_entry_passes(self): + validate_json_schema([_valid_entry()], TAXONOMY_SCHEMA) + + def test_missing_code_fails(self): + entry = _valid_entry() + del entry["code"] + with pytest.raises(jsonschema.ValidationError): + validate_json_schema([entry], TAXONOMY_SCHEMA) + + def test_missing_required_field_fails(self): + entry = {"code": "TEST_CODE"} + with pytest.raises(jsonschema.ValidationError): + validate_json_schema([entry], TAXONOMY_SCHEMA) + + def test_invalid_category_fails(self): + entry = _valid_entry() + entry["category"] = "invalid_phase" + with pytest.raises(jsonschema.ValidationError): + validate_json_schema([entry], TAXONOMY_SCHEMA) + + def test_invalid_severity_fails(self): + entry = _valid_entry() + entry["severity"] = "critical" + with pytest.raises(jsonschema.ValidationError): + validate_json_schema([entry], TAXONOMY_SCHEMA) + + def test_invalid_evidence_impact_fails(self): + entry = _valid_entry() + entry["evidence_impact"] = "invalid_impact" + with pytest.raises(jsonschema.ValidationError): + validate_json_schema([entry], TAXONOMY_SCHEMA) + + def test_invalid_applies_at_stage_fails(self): + entry = _valid_entry() + entry["applies_at_stage"] = "never" + with pytest.raises(jsonschema.ValidationError): + validate_json_schema([entry], TAXONOMY_SCHEMA) + + def test_invalid_reason_category_fails(self): + entry = _valid_entry() + entry["related_reason_category"] = "not_a_valid_category" + with pytest.raises(jsonschema.ValidationError): + validate_json_schema([entry], TAXONOMY_SCHEMA) + + def test_code_pattern_validation(self): + entry = _valid_entry() + entry["code"] = "invalid code" + with pytest.raises(jsonschema.ValidationError): + validate_json_schema([entry], TAXONOMY_SCHEMA) + + def test_min_description_length(self): + entry = _valid_entry() + entry["description"] = "Short" + with pytest.raises(jsonschema.ValidationError): + validate_json_schema([entry], TAXONOMY_SCHEMA) + + def test_multiple_valid_entries(self): + entries = [ + _valid_entry(), + { + "code": "PRE_SEQ_INVALID", + "category": "pre_selection", + "label": "Invalid sequence", + "description": "Sequence contains non-canonical amino-acid symbols.", + "severity": "hard", + "evidence_impact": "claim_unsupported", + "applies_at_stage": "pre_selection", + "related_reason_category": "pre_selection_reject", + "example": "Sequence contains 'X' at position 5.", + }, + ] + validate_json_schema(entries, TAXONOMY_SCHEMA) + + def test_optional_fields_accepted(self): + entry = _valid_entry() + entry["example"] = "Optional example text." + validate_json_schema([entry], TAXONOMY_SCHEMA) + + def test_all_categories_covered(self): + taxonomy = read_json(TAXONOMY_EXAMPLE) + categories = {e["category"] for e in taxonomy} + expected = {"pre_selection", "pipeline", "diversity", "reviewer", "lab", "lifecycle"} + assert categories == expected, ( + f"Missing categories: {expected - categories}" + ) + + def test_all_severities_covered(self): + taxonomy = read_json(TAXONOMY_EXAMPLE) + severities = {e["severity"] for e in taxonomy} + expected = {"hard", "soft", "informational"} + assert severities == expected, ( + f"Missing severities: {expected - severities}" + ) + + def test_all_evidence_impacts_covered(self): + taxonomy = read_json(TAXONOMY_EXAMPLE) + impacts = {e["evidence_impact"] for e in taxonomy} + expected = {"claim_unsupported", "downgrade_by_1", "downgrade_by_2", "no_impact"} + assert impacts == expected, ( + f"Missing evidence impacts: {expected - impacts}" + ) + + def test_example_file_validates_against_schema(self): + validate_json_schema(read_json(TAXONOMY_EXAMPLE), TAXONOMY_SCHEMA) + + def test_example_has_all_unique_codes(self): + taxonomy = read_json(TAXONOMY_EXAMPLE) + codes = [e["code"] for e in taxonomy] + assert len(codes) == len(set(codes)), "Duplicate codes found in example" + + def test_each_code_has_a_category(self): + taxonomy = read_json(TAXONOMY_EXAMPLE) + for entry in taxonomy: + assert entry["category"] in { + "pre_selection", "pipeline", "diversity", "reviewer", "lab", "lifecycle" + }, f"Entry {entry['code']} has invalid category" diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index 38645ef3..9d5420e7 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -2,9 +2,9 @@ import subprocess import sys -# Baseline: 2679 tests as of Loop 72 (2026-07-09) — Phase E E4-E6 (safety release, preregistration, CLI) +# Baseline: 2698 tests as of Loop 73 (2026-07-09) — Phase F F3 (rejection taxonomy schema) # Raise threshold if legitimate tests are removed; lower threshold is okay. -BASELINE = 2679 +BASELINE = 2698 TOLERANCE = 0.05 # allow 5% fluctuation