diff --git a/Makefile b/Makefile index 74cb38fb..6bba3763 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter +.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness PYTHON := $(shell [ -f .venv/bin/python ] && echo .venv/bin/python || echo python3) PYTEST := $(shell [ -f .venv/bin/pytest ] && echo .venv/bin/pytest || echo pytest) @@ -603,6 +603,12 @@ failed-candidate-report: --out-md outputs/failed_candidate_report.md \ --validate-rejection-codes +classify-negative-informativeness: + PYTHONPATH=src $(PYTHON) scripts/classify_negative_result_informativeness.py \ + --input examples/negative_result_entry_example.json \ + --out-json outputs/negative_result_informativeness_result.json \ + --out-md outputs/negative_result_informativeness_result.md + safe-publication-filter: PYTHONPATH=src $(PYTHON) scripts/safe_publication_filter.py \ --input examples/safe_publication_filter_example_input.json \ diff --git a/docs/PROJECT_INDEX.md b/docs/PROJECT_INDEX.md index 910fe76d..be722ac3 100644 --- a/docs/PROJECT_INDEX.md +++ b/docs/PROJECT_INDEX.md @@ -236,6 +236,7 @@ First useful contribution: make the repo easier to maintain without lowering saf | [`docs/review/ASSAY_PREREGISTRATION.md`](review/ASSAY_PREREGISTRATION.md) | Candidate-selection pilot pre-registration template, not a protocol. | | [`docs/review/COLLABORATION_PLAYBOOK.md`](review/COLLABORATION_PLAYBOOK.md) | External collaboration modes and boundaries. | | [`docs/evidence/NEGATIVE_RESULT_ARCHIVE.md`](evidence/../evidence/NEGATIVE_RESULT_ARCHIVE.md) | Safe negative-result format. | +| [`docs/evidence/NEGATIVE_RESULT_INFORMATIVENESS_GUIDE.md`](evidence/../evidence/NEGATIVE_RESULT_INFORMATIVENESS_GUIDE.md) | 7-dimension framework for classifying negative-result entries as informative/neutral/non-informative. | ## Core safety docs diff --git a/docs/evidence/METRICS_CURRENT.md b/docs/evidence/METRICS_CURRENT.md index 3ef04f87..341e43e3 100644 --- a/docs/evidence/METRICS_CURRENT.md +++ b/docs/evidence/METRICS_CURRENT.md @@ -5,7 +5,9 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **Purpose:** One authoritative table of current pipeline metrics. If any doc disagrees > with this file, this file wins. Updated whenever benchmark/benchmark config changes. > -> **Last updated:** 2026-07-09 (Phase F safe-publication filter — v0.5.76) +> **Last updated:** 2026-07-09 (Phase F negative-result informativeness — v0.5.77) +> **New in v0.5.77:** Negative-result informativeness guide (F6) — comprehensive documentation defining 7-dimension informativeness framework for negative-result entries, with 14 examples across all 6 reason categories, before/after transformation pairs, and quick-reference checklist. CLI classifier (`scripts/classify_negative_result_informativeness.py`) that scores entries on 7 dimensions and produces informative/neutral/non-informative classification. 37 tests. 2796 total. +> > **New in v0.5.76:** Safe-publication filter (F5) — CLI that reads candidate panel data with safety metadata and checks each candidate against publication safety constraints before external release. FAIL-CLOSED: defaults to reject unless all checks pass. Checks: dry_lab_only (must be true), proof_ladder_level (≤4), toxicity_screened, hemolysis_screened, dual_use_reviewed (all must be true). Produces structured JSON filter result and human-readable Markdown summary. 33 tests. 2758 total. > > **New in v0.5.75:** Failed-candidate report generator (F4) — CLI that reads candidate rejection data + taxonomy, produces structured JSON report with summary aggregation (by category, severity, stage, evidence impact) and human-readable Markdown summary. 26 tests. 2724 total. @@ -62,7 +64,7 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **New in v0.5.31:** Added dipeptide-order features for sequence-order awareness. `dipeptide_order_score` achieves AUROC 0.7861 on AMP-vs-scrambled discrimination — the strongest order-dependent feature in the pipeline. Only 7/31 features survive scrambling (amphipathicity/helix-wheel + dipeptide). All composition features are purely position-independent (exactly 0.5000 AUROC on scrambled test). > **New in v0.5.30:** Easy baseline benchmark added — charge density alone (AUROC 0.8166) outperforms the full pipeline ensemble (0.7792) on AMP-vs-Swiss-Prot-decoy discrimination. Honest finding documented: expected because pipeline optimizes for safety, not raw discrimination. > **New in v0.5.29:** Expanded benchmark to 500 AMPs + 500 composition-matched decoys (n=1000). AUROC 0.7792 (CI₉₅: 0.7505–0.8065) confirms signal generalizes. Cluster-aware CI: 0.746–0.8102. Representative AUROC: 0.778. Standard benchmark (n=191) retained for backward comparison. -> **Pipeline version:** v0.5.76 +> **Pipeline version:** v0.5.77 > **Branch:** main --- diff --git a/docs/evidence/NEGATIVE_RESULT_INFORMATIVENESS_GUIDE.md b/docs/evidence/NEGATIVE_RESULT_INFORMATIVENESS_GUIDE.md new file mode 100644 index 00000000..e14344e8 --- /dev/null +++ b/docs/evidence/NEGATIVE_RESULT_INFORMATIVENESS_GUIDE.md @@ -0,0 +1,465 @@ +# Negative-Result Informativeness Guide + +## Purpose + +Not all negative results are equally useful. A negative result that says "failed" +with no context tells future reviewers nothing. A negative result that documents +the specific threshold, the driving features, the assay conditions, and a learning +hypothesis is a calibration asset. + +This guide defines what makes a negative-result entry **informative**, gives +concrete examples across all six reason categories, and provides a quick-reference +checklist for human and agent contributors. + +## The Seven Dimensions of Informativeness + +| # | Dimension | What it measures | Why it matters | +|---|-----------|---|---| +| 1 | **Identity** | Is the candidate uniquely identified (candidate_id, sequence)? | Without identity, the entry cannot be linked to pipeline outputs or other batches. | +| 2 | **Context** | Are pipeline_version, source_batch, and date recorded? | Without context, you cannot know which rules or scores were in effect. | +| 3 | **Specificity** | Does reason_detail include concrete values, thresholds, or driving features? | Vague reasons cannot be acted on. Specific reasons enable threshold analysis. | +| 4 | **Actionability** | Does the entry enable a specific action or learning (tighten a gate, flag a scorer weakness, redesign a candidate class)? | Entries that only record failure without analysis cannot improve the pipeline. | +| 5 | **Verifiability** | Are scores and/or assay results included so the claim could be reproduced or checked? | Unverifiable entries cannot be distinguished from errors. | +| 6 | **Structured metadata** | Are category-appropriate conditional fields populated (assay_type/result/unit for lab entries, superseded_by for corrections)? | Structured fields enable machine analysis across entries. | +| 7 | **Interpretation** | Do reviewer_notes include analysis, caveats, or learning hypotheses? | Interpretation turns a data point into actionable knowledge. | + +### Classification + +| Class | Score range | Meaning | +|-------|-----------|---------| +| **INFORMATIVE** | 6.0–7.0 | Full documentation with analysis. Calibration asset. | +| **NEUTRAL** | 3.5–5.5 | Basic information present but lacks analysis or specificity. | +| **NON_INFORMATIVE** | 0.0–3.0 | Missing critical fields. Cannot be used for learning. | + +--- + +## Examples by Category + +### 1. Pre-selection reject + +#### Non-informative + +``` +entry_id: 101 +candidate_id: WAVE0.5-101 +sequence: ACDEFGHIKLMNPQRSTVWY +reason_category: pre_selection_reject +reason_detail: "Failed safety check." +pipeline_version: v0.5.76 +source_batch: wave0.5 +date: 2026-07-09 +``` + +Why it is non-informative: The reason_detail says nothing about which safety +check failed, what the threshold was, what score the candidate received, or +whether this was a systematic or candidate-specific issue. No reviewer_notes. + +#### Informative + +``` +entry_id: 201 +candidate_id: WAVE0.5-201 +sequence: KKKKLLLLKKKKLLLLKKKK +reason_category: pre_selection_reject +reason_detail: "Safety score 0.31 below minimum threshold 0.50. Driving factor: high hemolysis proxy (0.89) from elevated hydrophobicity (GRAVY=1.2). Candidate is a designed amphipathic helix with alternating K/L pattern typical of membrane-active peptides." +pipeline_version: v0.5.76 +source_batch: wave0.5 +date: 2026-07-09 +score_activity: 0.82 +score_safety: 0.31 +score_novelty: 0.91 +score_ensemble: 0.68 +reviewer_notes: "Rejected at Gate W0.5-3. High GRAVY is structural — the alternating K/L pattern is the intended mechanism, not an assay risk artifact. Consider whether the safety gate should distinguish between designed membrane activity and non-specific toxicity via a two-tier flag." +``` + +Why it is informative: Documents the specific threshold (0.50), the driving +feature (GRAVY=1.2 with hemolysis proxy 0.89), all relevant scores, and +captures a hypothesis about whether the rule needs adjustment. The +reviewer_notes provide context that a future reviewer or recalibration +engine can use. + +--- + +### 2. Pipeline reject (after scoring) + +#### Non-informative + +``` +entry_id: 102 +candidate_id: PHASE3-042 +sequence: AAAAAAKKKKKKAAAAAA +reason_category: pipeline_reject +reason_detail: "Low ensemble score." +pipeline_version: v0.5.76 +source_batch: phase3 +date: 2026-07-09 +``` + +Why it is non-informative: "Low" compared to what? No threshold, no scores, +no indication of which ensemble components drove the low result. + +#### Informative + +``` +entry_id: 202 +candidate_id: PHASE3-142 +sequence: GGGGKKKKGGGGKKKKGG +reason_category: pipeline_reject +reason_detail: "Ensemble score 0.42 below synthesis gate threshold 0.65. Activity score 0.38 (activity scorer penalised low hydrophobicity GRAVY=-0.8). Safety score 0.91 acceptable. Dominant failure mode: activity scorer assumes amphipathic helicity, but sequence is glycine-rich with no helical propensity." +pipeline_version: v0.5.76 +source_batch: phase3 +date: 2026-07-09 +score_activity: 0.38 +score_safety: 0.91 +score_novelty: 0.72 +score_ensemble: 0.42 +reviewer_notes: "This sequence class (glycine-rich, low hydrophobicity) is systematically undervalued by the activity scorer. Similar sequences in other phase3 batches show the same pattern. Recommend evaluating whether the activity scorer needs a non-helical fallback or whether glycine-rich candidates should be flagged for alternative scoring." +``` + +Why it is informative: Every score is documented, the specific threshold is +given, the failure mode is analysed (activity scorer assumptions vs actual +sequence properties), and the reviewer identifies a systematic issue that +warrants pipeline improvement. + +--- + +### 3. Diversity reject + +#### Non-informative + +``` +entry_id: 103 +candidate_id: SEED-015_VAR_009 +sequence: KLLKLLLKLLLKLLK +reason_category: diversity_reject +reason_detail: "Too similar to existing candidate." +pipeline_version: v0.5.76 +source_batch: wave0.5b +date: 2026-07-09 +``` + +Why it is non-informative: No similarity metric, no reference candidate, +no threshold — impossible to know whether the diversity filter is working +as intended or over-rejecting. + +#### Informative + +``` +entry_id: 203 +candidate_id: SEED-015_VAR_009 +sequence: KLLKLLLKLLLKLLK +reason_category: diversity_reject +reason_detail: "Edit-distance similarity 0.88 to SEED-015_VAR_005 (KLLKLLLKLLLKLL) exceeds diversity gate threshold 0.70. The single K→L substitution at position 7 does not provide sufficient sequence diversity for a separate panel slot. Both share the same scaffold (SEED-015, alternating K/L helix)." +pipeline_version: v0.5.76 +source_batch: wave0.5b +date: 2026-07-09 +score_activity: 0.81 +score_safety: 0.67 +score_novelty: 0.55 +score_ensemble: 0.68 +reviewer_notes: "Correct rejection. The candidate is a point mutant of an already-represented scaffold. If we want more SEED-015 diversity, we should design variants with non-conservative substitutions at multiple positions, not single-residue changes." +``` + +Why it is informative: Specifies the metric (edit-distance), the threshold +(0.70), the reference candidate, what the actual difference is, and includes +a design recommendation. + +--- + +### 4. Reviewer reject + +#### Non-informative + +``` +entry_id: 104 +candidate_id: SEED-008_VAR_003 +sequence: WFWFRWRWFWFRWR +reason_category: reviewer_reject +reason_detail: "Rejected by expert reviewer." +pipeline_version: v0.5.76 +source_batch: wave1 +date: 2026-07-09 +``` + +Why it is non-informative: No reviewer identity, no rationale, no reference +to the review criteria that were violated. + +#### Informative + +``` +entry_id: 204 +candidate_id: SEED-008_VAR_003 +sequence: WFWFRWRWFWFRWR +reason_category: reviewer_reject +reason_detail: "Rejected by Dr. A. Reviewer (Institution) on 2026-07-08. Safety concern: 8 Trp residues (Trp fraction 0.57) create dual-use concern per DUAL_USE_REVIEW_GUIDE v1.2 section 4.3. Reviewer notes the candidate has high membrane-permeation potential that could enable non-therapeutic applications." +pipeline_version: v0.5.76 +source_batch: wave1 +date: 2026-07-09 +score_activity: 0.79 +score_safety: 0.44 +score_novelty: 0.88 +score_ensemble: 0.70 +reviewer_notes: "Dual-use concern is valid. The candidate's Trp-rich pattern enables membrane translocation — a feature that is desirable for antimicrobial activity but also confers cell-penetrating capability. This candidate was correctly flagged for expert review per DUAL_USE_REVIEW_POLICY. The rejection should stand unless new evidence demonstrates selective antimicrobial mechanism without mammalian cell penetration." +``` + +Why it is informative: Names the reviewer (institutional accountability), +cites the specific policy violated (DUAL_USE_REVIEW_GUIDE v1.2 §4.3), +documents the structural feature causing the concern, and the reviewer_notes +explain why this rejection is important and what would be needed to overturn it. + +--- + +### 5. Lab inactive + +#### Non-informative + +``` +entry_id: 105 +candidate_id: WAVE0.5-105 +sequence: RRRRIIIIRRRRIIII +reason_category: lab_inactive +reason_detail: "Not active." +pipeline_version: v0.5.76 +source_batch: wave0.5 +date: 2026-07-09 +assay_type: MIC +``` + +Why it is non-informative: "Not active" — at what concentration? Against +which strain? Was the positive control working? Was the assay valid? No +assay_result or assay_unit. No reviewer_notes. + +#### Informative + +``` +entry_id: 205 +candidate_id: WAVE0.5-205 +sequence: RRIRIIRRIRIIRRI +reason_category: lab_inactive +reason_detail: "MIC > 128 ug/mL against E. coli ATCC 25922, S. aureus ATCC 29213, and P. aeruginosa ATCC 27853. No detectable activity up to 256 ug/mL in any of 3 biological replicates. Positive control (polymyxin B) showed expected MIC 1-2 ug/mL against E. coli, confirming assay validity." +pipeline_version: v0.5.76 +source_batch: wave0.5 +date: 2026-07-09 +assay_type: MIC +assay_result: ">128" +assay_unit: ug/mL +score_activity: 0.78 +score_safety: 0.85 +score_novelty: 0.62 +score_ensemble: 0.75 +reviewer_notes: "Activity score 0.78 was a false positive. Sequence is net-charge +7 with moderate amphipathicity — the activity scorer likely overvalued the charge component. This candidate class (Arg-Ile alternating pattern with no hydrophobic core) appears to be a recurring false-positive motif. Consider tracking this pattern for activity-scorer recalibration." +``` + +Why it is informative: Reports multi-strain testing with 3 biological +replicates, confirms assay validity (positive control data), documents the +exact concentration range tested, and identifies a recurring false-positive +pattern for pipeline improvement. + +--- + +### 6. Lab toxic + +#### Non-informative + +``` +entry_id: 106 +candidate_id: WAVE0.5-106 +sequence: LLLLKKKKLLLLKKKK +reason_category: lab_toxic +reason_detail: "Toxic in hemolysis assay." +pipeline_version: v0.5.76 +source_batch: wave0.5 +date: 2026-07-09 +assay_type: hemolysis_RBC +``` + +Why it is non-informative: "Toxic" — at what concentration? What was the +HC50? How many replicates? Was the negative control valid? Without assay +result and unit, the entry is a judgement call with no supporting data. + +#### Informative + +``` +entry_id: 206 +candidate_id: WAVE0.5-206 +sequence: LKKLLKKLLKKLKKL +reason_category: lab_toxic +reason_detail: "100% hemolysis at 50 uM (human RBC, pooled donors). HC50 = 6.2 uM. Dose-response curve shows steep transition between 2.5 uM (12% hemolysis) and 10 uM (78% hemolysis). Assay positive control (0.1% Triton X-100) showed 100% hemolysis. Negative control (PBS) showed 0.8% hemolysis. 3 technical replicates per concentration." +pipeline_version: v0.5.76 +source_batch: wave0.5 +date: 2026-07-09 +assay_type: hemolysis_RBC +assay_result: "100" +assay_unit: "%" +score_activity: 0.76 +score_safety: 0.54 +score_novelty: 0.91 +score_ensemble: 0.74 +reviewer_notes: "Computational safety score 0.54 was below the 0.50 threshold but only marginally. The steep dose-response (2.5 uM → 10 uM transition) suggests a cooperative membrane-disruption mechanism typical of amphipathic helices. Current safety scorer does not model dose-response steepness. Consider adding a cooperativity penalty for candidates with steep hemolysis transitions." +``` + +Why it is informative: Full dose-response data, HC50 value, control validity, +replicate count, and a specific improvement recommendation (cooperativity +penalty based on dose-response steepness). + +--- + +### 7. Synthesis failure + +#### Non-informative + +``` +entry_id: 107 +candidate_id: WAVE0.5-107 +sequence: CCCCGGGGCCCCGGGG +reason_category: synthesis_failure +reason_detail: "Could not synthesize." +pipeline_version: v0.5.76 +source_batch: wave0.5 +date: 2026-07-09 +``` + +Why it is non-informative: "Could not synthesize" is vague. Was it a purity +issue? A yield issue? An aggregation problem? Without details, the same +failure will recur if the candidate is reordered from a different vendor. + +#### Informative + +``` +entry_id: 207 +candidate_id: WAVE0.5-207 +sequence: CCCCCCCCCKKKKKKK +reason_category: synthesis_failure +reason_detail: "Synthesis failed at purification stage. Crude purity 24% by HPLC (target > 80%). Major byproducts: des-Cys truncations at positions 3 and 7 (confirmed by MS). The poly-Cys N-terminal region causes aggregation during Fmoc deprotection. Two vendor attempts (vendor A and vendor B) both failed with similar impurity profiles. Synthesis difficulty score was 0.31 (below 0.50 feasibility threshold)." +pipeline_version: v0.5.76 +source_batch: wave0.5 +date: 2026-07-09 +score_activity: 0.84 +score_safety: 0.72 +score_novelty: 0.67 +score_ensemble: 0.74 +reviewer_notes: "The synthesis score correctly predicted failure (0.31). The poly-Cys motif is known to cause aggregation during SPPS. Future designs should avoid >3 consecutive Cys residues. If this scaffold is needed, consider using pseudoproline or other difficult-sequence chemistries — but this would increase cost significantly." +``` + +Why it is informative: Documents the specific failure mode (poly-Cys +aggregation), provides quantitative data (24% purity, 2 vendor attempts), +confirms the pipeline's synthesis score was accurate, and gives a design +rule to prevent recurrence. + +--- + +## Before / After Pairs + +### Before (non-informative) + +``` +entry_id: 301 +reason_detail: "Failed safety check." +``` + +### After (informative) + +``` +entry_id: 301 +candidate_id: BATCH-X-042 +sequence: KKKLKKKLKKKLKKK +reason_category: pre_selection_reject +reason_detail: "Safety score 0.28 below threshold 0.50. Hemolysis proxy 0.92 from charge density +6.2 (net charge +8, length 14). All 4 high-charge candidates in batch BATCH-X show similar safety scores — batch-level pattern." +pipeline_version: v0.5.76 +source_batch: BATCH-X +date: 2026-07-09 +score_activity: 0.85 +score_safety: 0.28 +score_ensemble: 0.59 +reviewer_notes: "Batch-level pattern: all high-charge candidates failing safety. This is expected for short, highly cationic sequences. No individual candidate anomaly. Track whether this batch's failure rate is within expected distribution for high-charge designs." +``` + +### Before (non-informative) + +``` +entry_id: 302 +assay_result: ">64" +assay_unit: ug/mL +reason_detail: "MIC too high." +``` + +### After (informative) + +``` +entry_id: 302 +candidate_id: BATCH-Y-015 +sequence: RLLRLLRLLRLLRLL +reason_category: lab_inactive +reason_detail: "MIC > 64 ug/mL against E. coli ATCC 25922. Tested up to 128 ug/mL. No zone of inhibition in disk diffusion at 50 ug/disk. Positive control (gentamicin) MIC = 1 ug/mL. Negative control (DMSO 1%) showed no inhibition. n=3 biological replicates." +pipeline_version: v0.5.76 +source_batch: BATCH-Y +date: 2026-07-09 +assay_type: MIC +assay_result: ">64" +assay_unit: ug/mL +score_activity: 0.71 +score_safety: 0.88 +score_ensemble: 0.69 +reviewer_notes: "Borderline activity score (0.71) with no actual activity. The candidate is a low-charge (+3), moderate-GRAVY design — the activity score may have been inflated by moderate hydrophobicity rather than genuine antimicrobial features. This is consistent with the known weakness of the activity scorer on non-cationic candidates." +``` + +--- + +## Quick-Reference Checklist + +Use this checklist when writing or reviewing a negative-result entry. Each +"yes" earns 1 point toward the informativeness score. + +### Identity (1 point) +- [ ] candidate_id is non-empty and specific +- [ ] sequence is a valid amino-acid string + +### Context (1 point) +- [ ] pipeline_version is recorded +- [ ] source_batch is recorded +- [ ] date is in ISO 8601 format + +### Specificity (1 point) +- [ ] reason_detail includes a specific numeric threshold or value +- [ ] reason_detail identifies the driving feature or failure mode + +### Actionability (1 point) +- [ ] The entry enables a specific action (tighten gate, fix scorer, redesign class) +- [ ] Multiple similar entries would reveal a systematic issue + +### Verifiability (1 point) +- [ ] Scores are present (score_activity, score_safety, etc., where applicable) +- [ ] Assay results include numeric values and units (where applicable) + +### Structured metadata (1 point) +- [ ] Category-appropriate conditional fields are populated +- [ ] superseded_by is set if this entry corrects a prior entry + +### Interpretation (1 point) +- [ ] reviewer_notes contain analysis beyond restating the reason_detail +- [ ] reviewer_notes identify a learning hypothesis or pipeline improvement + +## Scoring + +Count ✓ marks: ____ / 14 possible half-points → ____ / 7 total points + +| Score | Classification | +|-------|---------------| +| 6.0–7.0 | **INFORMATIVE** — calibration asset | +| 3.5–5.5 | **NEUTRAL** — basic data present, analysis missing | +| 0.0–3.0 | **NON_INFORMATIVE** — cannot support learning | + +## Limitations + +- Informativeness scoring is heuristic. A score of 7.0 does not guarantee + that the entry is scientifically correct — only that it is well-documented. +- A non-informative entry may contain genuinely important negative data that + the submitter simply did not document well. Missing structure does not mean + missing signal. +- The framework assumes the entry is intended for machine analysis and + human review. Some entries may be intentionally sparse (e.g., privacy- + constrained dual-use rejections); those should be flagged as such rather + than classified as non-informative. +- Informativeness is not a review of scientific validity. An informative + entry about an incorrect conclusion is still well-documented — and more + useful than a non-informative entry about a correct one. diff --git a/docs/research/ROADMAP.md b/docs/research/ROADMAP.md index e6a89ea0..eadfaae1 100644 --- a/docs/research/ROADMAP.md +++ b/docs/research/ROADMAP.md @@ -1,5 +1,45 @@ # Roadmap +## v0.5.77 — Loop 76: Phase F F6 — Negative-Result Informativeness Guide ✓ (2026-07-09) + +Comprehensive guide defining informative vs non-informative negative results with +a 7-dimension framework, 14 examples across all 6 reason categories, before/after +transformation pairs, and a CLI classifier that scores entries. + +Changes: +- `docs/evidence/NEGATIVE_RESULT_INFORMATIVENESS_GUIDE.md` (F6) — Standalone guide + with 7-dimension informativeness framework (identity, context, specificity, + actionability, verifiability, structured metadata, interpretation). 14 annotated + examples across all 6 reason categories, 2 before/after transformation pairs, + quick-reference checklist with scoring rubric. Classification: INFORMATIVE + (6.0–7.0), NEUTRAL (3.5–5.5), NON_INFORMATIVE (0.0–3.0). +- `scripts/classify_negative_result_informativeness.py` (F6) — CLI that loads a + single negative-result entry (JSON string or file), scores it on 7 dimensions, + and produces a classification with per-dimension breakdown. Supports `--entry`, + `--input`, `--out-json`, and `--out-md` flags. Fail-closed on missing entries. +- `examples/negative_result_entry_example.json` (F6) — Toy example entry with + full informative documentation. Clearly marked EXAMPLE. +- `Makefile` — Added `classify-negative-informativeness` target. +- `tests/evidence/test_negative_result_informativeness.py` — 37 tests covering: + entry classification (informative/neutral/non-informative), each dimension + scorer (presence/absence thresholds), markdown/caveat generation, output + structure, entry loading (dict/JSON/file/error paths), CLI flags (entry, + input, missing flag, JSON output file, Markdown output file), and example + file validation. +- `docs/evidence/METRICS_CURRENT.md` — v0.5.77 changelog. Pipeline version bumped. + Test count: 2796. +- `tests/test_test_count_regression.py` — baseline updated to 2796. + +Honest boundaries: +- Informativeness scoring is heuristic. A score of 7.0 does not guarantee + scientific correctness — only thorough documentation. +- A NON_INFORMATIVE entry may contain genuinely important data that the + submitter simply did not document well. +- The framework assumes the entry is intended for machine analysis and human + review. Privacy-constrained entries (e.g., dual-use rejections) should be + flagged rather than classified as non-informative. +- The classifier validates documentation quality, not scientific validity. + ## v0.5.76 — Loop 75: Phase F F5 — Safe-Publication Filter ✓ (2026-07-09) Safe-publication filter: CLI that reads a candidate panel with safety metadata diff --git a/examples/negative_result_entry_example.json b/examples/negative_result_entry_example.json new file mode 100644 index 00000000..5a46e5e8 --- /dev/null +++ b/examples/negative_result_entry_example.json @@ -0,0 +1,20 @@ +{ + "schema": "EXAMPLE — NOT REAL DATA. For demonstration only.", + "entry_id": 205, + "date": "2026-07-09", + "candidate_id": "WAVE0.5-EXAMPLE-205", + "sequence": "RRIRIIRRIRIIRRI", + "reason_category": "lab_inactive", + "reason_detail": "MIC > 128 ug/mL against E. coli ATCC 25922, S. aureus ATCC 29213, and P. aeruginosa ATCC 27853. No detectable activity up to 256 ug/mL in any of 3 biological replicates. Positive control (polymyxin B) showed expected MIC 1-2 ug/mL against E. coli, confirming assay validity.", + "pipeline_version": "v0.5.76", + "source_batch": "wave0.5", + "assay_type": "MIC", + "assay_result": ">128", + "assay_unit": "ug/mL", + "score_activity": 0.78, + "score_safety": 0.85, + "score_novelty": 0.62, + "score_ensemble": 0.75, + "reviewer_notes": "Activity score 0.78 was a false positive. Sequence is net-charge +7 with moderate amphipathicity — the activity scorer likely overvalued the charge component. This candidate class (Arg-Ile alternating pattern with no hydrophobic core) appears to be a recurring false-positive motif. Consider tracking this pattern for activity-scorer recalibration.", + "recalibration_used": "no" +} diff --git a/scripts/classify_negative_result_informativeness.py b/scripts/classify_negative_result_informativeness.py new file mode 100644 index 00000000..069e6d10 --- /dev/null +++ b/scripts/classify_negative_result_informativeness.py @@ -0,0 +1,343 @@ +"""Classify a negative-result entry as informative, neutral, or non-informative. + +Evaluates entries against the 7-dimension informativeness framework defined in +docs/evidence/NEGATIVE_RESULT_INFORMATIVENESS_GUIDE.md. + +Each entry is scored 0.0-7.0 (14 half-points). Classification: + - INFORMATIVE: 6.0–7.0 — calibration asset + - NEUTRAL: 3.5–5.5 — basic data present, analysis missing + - NON_INFORMATIVE: 0.0–3.0 — cannot support learning + +Fail-closed on missing entries: any missing required field scores 0 for that dimension. + +Usage: + python scripts/classify_negative_result_informativeness.py \\ + --entry '{"candidate_id": "TEST-001", "sequence": "ACDEFGHIKLMNPQRSTVWY", ...}' \\ + [--out-json output.json] + + python scripts/classify_negative_result_informativeness.py \\ + --input examples/negative_result_entry_example.json \\ + [--out-json output.json] + +Exit codes: + 0 Success + 2 Input error (missing file, invalid JSON) +""" +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any + + +INFORMATIVE_SCORE_MIN = 6.0 +NEUTRAL_SCORE_MIN = 3.5 + + +def load_entry(source: str | Path | dict[str, Any]) -> dict[str, Any]: + """Load entry from JSON string, file path, or dict.""" + if isinstance(source, dict): + return source + p = Path(source) + if p.exists(): + try: + with p.open("r") as f: + return json.load(f) + except json.JSONDecodeError as e: + msg = f"Invalid JSON in {p}: {e}" + raise ValueError(msg) from e + try: + return json.loads(source) + except json.JSONDecodeError as e: + msg = f"Invalid JSON string: {e}" + raise ValueError(msg) from e + + +def _identity_score(entry: dict[str, Any]) -> float: + """Score for Identity dimension (0.0-1.0).""" + score = 0.0 + cid = entry.get("candidate_id", "") + seq = entry.get("sequence", "") + if isinstance(cid, str) and len(cid.strip()) > 0: + score += 0.5 + if isinstance(seq, str) and len(seq.strip()) > 0: + score += 0.5 + return score + + +def _context_score(entry: dict[str, Any]) -> float: + """Score for Context dimension (0.0-1.0).""" + score = 0.0 + pv = entry.get("pipeline_version") + sb = entry.get("source_batch") + dt = entry.get("date") + if isinstance(pv, str) and len(pv.strip()) > 0: + score += 0.5 + if isinstance(sb, str) and len(sb.strip()) > 0: + score += 0.5 + return score + + +def _specificity_score(entry: dict[str, Any]) -> float: + """Score for Specificity dimension (0.0-1.0). + + Checks reason_detail for numeric values/thresholds and driving features. + """ + score = 0.0 + detail = entry.get("reason_detail", "") + if not isinstance(detail, str) or len(detail.strip()) == 0: + return score + if len(detail) >= 20: + score += 0.25 + if any(c.isdigit() for c in detail): + score += 0.25 + descriptive_indicators = [ + "because", "driving", "due to", "from", "score", "threshold", + "below", "above", "exceeds", "gate", + ] + if any(ind in detail.lower() for ind in descriptive_indicators): + score += 0.25 + feature_markers = ["GRAVY", "charge", "hydrophob", "hemolysis", "helic", + "prolin", "cys", "length", "weight", "score", + "similarity", "pattern", "motif"] + if any(m in detail for m in feature_markers): + score += 0.25 + return min(score, 1.0) + + +def _actionability_score(entry: dict[str, Any]) -> float: + """Score for Actionability dimension (0.0-1.0). + + Looks for indicators that enable specific actions or reveal systematic issues. + """ + score = 0.0 + notes = entry.get("reviewer_notes", "") + detail = entry.get("reason_detail", "") + if not isinstance(notes, str): + notes = "" + + actionable_indicators = [ + "should", "consider", "recommend", "suggest", "track", "flag", + "tighten", "loosen", "adjust", "recalibrat", "improve", "fix", + "systematic", "pattern", "recurring", "consistent", "all candidate", + "batch-level", "class", + ] + if any(ind in notes.lower() for ind in actionable_indicators): + score += 0.5 + if any(ind in detail.lower() for ind in ["systematic", "pattern", "batch-level", + "all candidate"]): + score += 0.25 + if len(notes) >= 50: + score += 0.25 + return min(score, 1.0) + + +def _verifiability_score(entry: dict[str, Any]) -> float: + """Score for Verifiability dimension (0.0-1.0).""" + score = 0.0 + for key in ["score_activity", "score_safety", "score_novelty", "score_ensemble"]: + val = entry.get(key) + if isinstance(val, (int, float)) and 0 <= val <= 1: + score += 0.25 + score = min(score, 0.5) + result = entry.get("assay_result") + unit = entry.get("assay_unit") + if isinstance(result, str) and len(result.strip()) > 0: + score += 0.25 + if isinstance(unit, str) and len(unit.strip()) > 0: + score += 0.25 + return min(score, 1.0) + + +def _structured_metadata_score(entry: dict[str, Any]) -> float: + """Score for Structured Metadata dimension (0.0-1.0). + + Checks category-appropriate conditional fields and superseded_by. + """ + score = 0.0 + eid = entry.get("entry_id") + if eid is not None: + score += 0.25 + cat = entry.get("reason_category", "") + if isinstance(cat, str) and cat in ( + "pre_selection_reject", "selected_untested", "lab_inactive", + "lab_toxic", "control_failure", "synthesis_failure", + ): + score += 0.25 + if cat in ("lab_inactive", "lab_toxic", "synthesis_failure"): + at = entry.get("assay_type") + ar = entry.get("assay_result") + au = entry.get("assay_unit") + if isinstance(at, str) and len(at.strip()) > 0: + score += 0.25 + if isinstance(ar, str) and len(ar.strip()) > 0: + score += 0.25 + if isinstance(au, str) and len(au.strip()) > 0: + score += 0.25 + sb = entry.get("superseded_by") + if sb is not None: + score += 0.5 + return min(score, 1.0) + + +def _interpretation_score(entry: dict[str, Any]) -> float: + """Score for Interpretation dimension (0.0-1.0). + + Evaluates reviewer_notes for analytical content beyond restating the detail. + """ + score = 0.0 + notes = entry.get("reviewer_notes", "") + if not isinstance(notes, str) or len(notes.strip()) < 30: + return score + score += 0.25 + detail = entry.get("reason_detail", "") + if not isinstance(detail, str): + detail = "" + if notes != detail: + score += 0.25 + analytical_markers = [ + "because", "suggest", "indicate", "hypothesis", "consistent with", + "expected", "unexpected", "notable", "concern", "risk", "tradeoff", + "recommend", "consider", "should", "may", "could", "warrant", + ] + if any(m in notes.lower() for m in analytical_markers): + score += 0.25 + if len(notes) >= 100: + score += 0.25 + return min(score, 1.0) + + +def classify_entry(entry: dict[str, Any]) -> dict[str, Any]: + """Classify a single negative-result entry. + + Returns dict with per-dimension scores, total, classification, and + metadata. + """ + dimensions = { + "identity": _identity_score(entry), + "context": _context_score(entry), + "specificity": _specificity_score(entry), + "actionability": _actionability_score(entry), + "verifiability": _verifiability_score(entry), + "structured_metadata": _structured_metadata_score(entry), + "interpretation": _interpretation_score(entry), + } + total = round(sum(dimensions.values()), 2) + + if total >= INFORMATIVE_SCORE_MIN: + classification = "INFORMATIVE" + elif total >= NEUTRAL_SCORE_MIN: + classification = "NEUTRAL" + else: + classification = "NON_INFORMATIVE" + + return { + "entry_id": entry.get("entry_id"), + "candidate_id": entry.get("candidate_id"), + "reason_category": entry.get("reason_category"), + "classification": classification, + "total_score": total, + "dimensions": dimensions, + "caveat": ( + "Computational classification only. Does not validate scientific " + "accuracy. An INFORMATIVE entry may still contain incorrect " + "conclusions." + ), + } + + +def build_markdown(result: dict[str, Any]) -> str: + """Build human-readable Markdown summary from classification result.""" + dim_labels = { + "identity": "Identity", + "context": "Context", + "specificity": "Specificity", + "actionability": "Actionability", + "verifiability": "Verifiability", + "structured_metadata": "Structured metadata", + "interpretation": "Interpretation", + } + cid = result.get("candidate_id", "unknown") + eid = result.get("entry_id", "?") + lines = [ + f"# Negative-Result Informativeness Classification: {cid}", + "", + f"**Entry ID:** {eid}", + f"**Category:** {result.get('reason_category', 'N/A')}", + f"**Classification: {result['classification']}**", + f"**Total score:** {result['total_score']}/7.0", + "", + "| Dimension | Score |", + "|-----------|-------|", + ] + for key, label in dim_labels.items(): + lines.append(f"| {label} | {result['dimensions'].get(key, 0.0)} |") + lines.append("") + lines.append("### Classification thresholds") + lines.append(f"- INFORMATIVE: >= {INFORMATIVE_SCORE_MIN}") + lines.append(f"- NEUTRAL: >= {NEUTRAL_SCORE_MIN}") + lines.append(f"- NON_INFORMATIVE: < {NEUTRAL_SCORE_MIN}") + lines.append("") + lines.append(f"> {result['caveat']}") + return "\n".join(lines) + "\n" + + +def build_output( + entry: dict[str, Any], + result: dict[str, Any], +) -> dict[str, Any]: + """Build the full output dict with entry, result, and metadata.""" + return { + "classifier_version": "v1.0.0", + "framework": "NEGATIVE_RESULT_INFORMATIVENESS_GUIDE.md", + "input_entry": entry, + "result": result, + } + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Classify a negative-result entry as informative/neutral/non-informative." + ) + source = parser.add_mutually_exclusive_group(required=True) + source.add_argument("--entry", type=str, help="JSON string of a negative-result entry") + source.add_argument("--input", type=str, help="Path to JSON file with a negative-result entry") + parser.add_argument("--out-json", type=str, help="Path to write JSON output") + parser.add_argument("--out-md", type=str, help="Path to write Markdown output") + args = parser.parse_args() + + try: + if args.input: + entry = load_entry(args.input) + else: + entry = load_entry(args.entry) + except (FileNotFoundError, ValueError) as e: + print(f"ERROR: {e}", file=sys.stderr) + sys.exit(2) + + result = classify_entry(entry) + output = build_output(entry, result) + + print(f"Classification: {result['classification']}") + print(f"Total score: {result['total_score']}/7.0") + for dim_key, dim_score in result["dimensions"].items(): + print(f" {dim_key}: {dim_score}") + + if args.out_json: + p = Path(args.out_json) + with p.open("w") as f: + json.dump(output, f, indent=2) + print(f"JSON output written to {p}") + + if args.out_md: + p = Path(args.out_md) + md = build_markdown(result) + with p.open("w") as f: + f.write(md) + print(f"Markdown output written to {p}") + + +if __name__ == "__main__": + main() diff --git a/tests/evidence/test_negative_result_informativeness.py b/tests/evidence/test_negative_result_informativeness.py new file mode 100644 index 00000000..362cb64c --- /dev/null +++ b/tests/evidence/test_negative_result_informativeness.py @@ -0,0 +1,335 @@ +"""Tests for the negative-result informativeness classifier (F6).""" +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +from scripts.classify_negative_result_informativeness import ( + build_markdown, + build_output, + classify_entry, + load_entry, + _identity_score, + _context_score, + _specificity_score, + _actionability_score, + _verifiability_score, + _structured_metadata_score, + _interpretation_score, +) + +PYTHON = sys.executable +_REPO_DIR = Path(__file__).parents[2] + + +def _make_entry(overrides: dict | None = None) -> dict: + """Build a maximally informative entry, then override for test cases.""" + entry = { + "entry_id": 1, + "date": "2026-07-09", + "candidate_id": "TEST-001", + "sequence": "ACDEFGHIKLMNPQRSTVWY", + "reason_category": "pre_selection_reject", + "reason_detail": "Safety score 0.31 below minimum threshold 0.50. Driving factor: high hemolysis proxy (0.89) from elevated hydrophobicity (GRAVY=1.2). Batch-level pattern: all high-GRAVY candidates in this batch were rejected by the same gate.", + "pipeline_version": "v0.5.76", + "source_batch": "test-batch", + "score_activity": 0.82, + "score_safety": 0.31, + "score_novelty": 0.91, + "score_ensemble": 0.68, + "reviewer_notes": "Rejected at Gate W0.5-3. High GRAVY is structural for these designed amphipathic sequences. Recommend implementing a two-tier safety flag that distinguishes structural membrane activity from non-specific toxicity risk. This systematic pattern affects 12/40 candidates in the current batch and warrants a policy review.", + } + if overrides: + entry.update(overrides) + return entry + + +class TestClassifyEntry: + def test_informative_entry_classified_informative(self): + """A fully populated entry should be classified INFORMATIVE.""" + result = classify_entry(_make_entry()) + assert result["classification"] == "INFORMATIVE" + assert result["total_score"] >= 6.0 + + def test_non_informative_minimal_entry(self): + """A bare-minimum entry should be NON_INFORMATIVE.""" + entry = { + "entry_id": 2, + "candidate_id": "TEST-002", + "sequence": "ACDEFGHIKLMNPQRSTVWY", + "reason_category": "pre_selection_reject", + "reason_detail": "Failed.", + "pipeline_version": "v0.5.76", + "source_batch": "test", + "date": "2026-07-09", + } + result = classify_entry(entry) + assert result["classification"] == "NON_INFORMATIVE" + assert result["total_score"] < 3.5 + + def test_neutral_entry_partial_info(self): + """An entry with basic fields but no analysis should be NEUTRAL.""" + entry = { + "entry_id": 3, + "date": "2026-07-09", + "candidate_id": "TEST-003", + "sequence": "ACDEFGHIKLMNPQRSTVWY", + "reason_category": "lab_inactive", + "reason_detail": "MIC > 64 ug/mL against E. coli.", + "pipeline_version": "v0.5.76", + "source_batch": "wave0.5", + "assay_type": "MIC", + "assay_result": ">64", + "assay_unit": "ug/mL", + } + result = classify_entry(entry) + assert result["classification"] == "NEUTRAL" + assert 3.5 <= result["total_score"] < 6.0 + + def test_classification_score_range(self): + """Total score should always be in 0.0-7.0 range.""" + for entry_data in [ + {}, + {"candidate_id": "X"}, + _make_entry(), + _make_entry({"reviewer_notes": ""}), + ]: + result = classify_entry(entry_data) + assert 0.0 <= result["total_score"] <= 7.0 + + def test_result_contains_required_keys(self): + """Result dict should have all expected fields.""" + result = classify_entry(_make_entry()) + for key in ("entry_id", "candidate_id", "reason_category", "classification", + "total_score", "dimensions", "caveat"): + assert key in result + + def test_dimensions_has_seven_keys(self): + """Dimensions dict should have all 7 dimension scores.""" + result = classify_entry(_make_entry()) + for dim in ("identity", "context", "specificity", "actionability", + "verifiability", "structured_metadata", "interpretation"): + assert dim in result["dimensions"] + assert 0.0 <= result["dimensions"][dim] <= 1.0 + + def test_empty_entry_non_informative(self): + """An empty entry should score 0 and be NON_INFORMATIVE.""" + result = classify_entry({}) + assert result["classification"] == "NON_INFORMATIVE" + assert result["total_score"] == 0.0 + for dim_val in result["dimensions"].values(): + assert dim_val == 0.0 + + +class TestDimensionScorers: + def test_identity_present(self): + assert _identity_score({"candidate_id": "X", "sequence": "ACDEF"}) == 1.0 + + def test_identity_missing(self): + assert _identity_score({}) == 0.0 + assert _identity_score({"candidate_id": "", "sequence": ""}) == 0.0 + + def test_context_present(self): + entry = {"pipeline_version": "v1", "source_batch": "batch", "date": "2026-07-09"} + assert _context_score(entry) == 1.0 + + def test_context_missing(self): + assert _context_score({}) == 0.0 + + def test_specificity_vague_string(self): + assert _specificity_score({"reason_detail": "Failed."}) < 0.5 + + def test_specificity_with_numbers_and_features(self): + score = _specificity_score({ + "reason_detail": "Safety score 0.31 below threshold 0.50. GRAVY=1.2 driving hemolysis." + }) + assert score >= 0.75 + + def test_actionability_with_notes(self): + entry = _make_entry({"reviewer_notes": "Recommend tightening the safety gate threshold."}) + assert _actionability_score(entry) > 0.0 + + def test_actionability_missing_notes(self): + assert _actionability_score({"reason_detail": "no"}) == 0.0 + + def test_verifiability_with_scores_and_assay(self): + entry = { + "score_activity": 0.78, "score_safety": 0.85, + "score_ensemble": 0.75, + "assay_result": ">128", "assay_unit": "ug/mL", + } + assert _verifiability_score(entry) > 0.0 + + def test_verifiability_no_data(self): + assert _verifiability_score({}) == 0.0 + + def test_structured_metadata_lab_entry(self): + entry = { + "entry_id": 1, + "reason_category": "lab_inactive", + "assay_type": "MIC", "assay_result": ">64", "assay_unit": "ug/mL", + } + assert _structured_metadata_score(entry) >= 1.0 + + def test_structured_metadata_non_lab(self): + score = _structured_metadata_score({"reason_category": "pre_selection_reject"}) + assert score == 0.25 + + def test_structured_metadata_with_superseded(self): + entry = {"entry_id": 1, "reason_category": "pre_selection_reject", "superseded_by": 0} + assert _structured_metadata_score(entry) >= 1.0 + + def test_interpretation_long_analytical_notes(self): + entry = { + "reason_detail": "Score too low.", + "reviewer_notes": ( + "This failure suggests a systematic issue with the safety scorer " + "for high-GRAVY candidates. The current threshold may be too " + "conservative for designed amphipathic sequences. Consider a " + "two-tier flag approach that distinguishes structural from " + "non-specific membrane activity." + ), + } + assert _interpretation_score(entry) >= 0.5 + + def test_interpretation_no_notes(self): + assert _interpretation_score({"reason_detail": "Failed."}) == 0.0 + + +class TestBuildMarkdown: + def test_markdown_contains_classification(self): + entry = _make_entry() + result = classify_entry(entry) + md = build_markdown(result) + assert result["classification"] in md + assert str(result["total_score"]) in md + + def test_markdown_contains_caveat(self): + entry = _make_entry() + result = classify_entry(entry) + md = build_markdown(result) + assert "Computational classification only" in md + + +class TestBuildOutput: + def test_output_contains_entry_and_result(self): + entry = _make_entry() + result = classify_entry(entry) + output = build_output(entry, result) + assert output["input_entry"] is entry + assert output["result"] is result + assert output["classifier_version"] == "v1.0.0" + assert "framework" in output + + def test_output_json_serializable(self): + entry = _make_entry() + result = classify_entry(entry) + output = build_output(entry, result) + dumped = json.dumps(output) + assert len(dumped) > 0 + reloaded = json.loads(dumped) + assert reloaded["result"]["classification"] == result["classification"] + + +class TestLoadEntry: + def test_load_entry_from_dict(self): + entry = {"candidate_id": "X"} + assert load_entry(entry) is entry + + def test_load_entry_from_json_string(self): + entry = load_entry('{"candidate_id": "X"}') + assert entry["candidate_id"] == "X" + + def test_load_entry_invalid_raises(self): + with pytest.raises(ValueError, match="Invalid JSON string"): + load_entry("not-json") + + def test_load_entry_file_not_found_raises(self): + with pytest.raises(ValueError, match="Invalid JSON"): + load_entry("/nonexistent/path.json") + + +class TestCLI: + def test_cli_entry_flag(self): + r = subprocess.run( + [PYTHON, "scripts/classify_negative_result_informativeness.py", + "--entry", json.dumps(_make_entry())], + capture_output=True, text=True, + env={"PYTHONPATH": "src"}, + cwd=str(_REPO_DIR), + ) + assert r.returncode == 0 + assert "INFORMATIVE" in r.stdout + + def test_cli_input_file(self): + example = _REPO_DIR / "examples/negative_result_entry_example.json" + r = subprocess.run( + [PYTHON, "scripts/classify_negative_result_informativeness.py", + "--input", str(example)], + capture_output=True, text=True, + env={"PYTHONPATH": "src"}, + cwd=str(_REPO_DIR), + ) + assert r.returncode == 0 + assert "INFORMATIVE" in r.stdout + + def test_cli_missing_flag(self): + r = subprocess.run( + [PYTHON, "scripts/classify_negative_result_informativeness.py"], + capture_output=True, text=True, + env={"PYTHONPATH": "src"}, + cwd=str(_REPO_DIR), + ) + assert r.returncode == 2 + + def test_cli_out_json_creates_file(self, tmp_path): + out = tmp_path / "result.json" + r = subprocess.run( + [PYTHON, "scripts/classify_negative_result_informativeness.py", + "--entry", json.dumps(_make_entry()), + "--out-json", str(out)], + capture_output=True, text=True, + env={"PYTHONPATH": "src"}, + cwd=str(_REPO_DIR), + ) + assert r.returncode == 0 + assert out.exists() + data = json.loads(out.read_text()) + assert data["result"]["classification"] == "INFORMATIVE" + + def test_cli_out_md_creates_file(self, tmp_path): + out = tmp_path / "result.md" + r = subprocess.run( + [PYTHON, "scripts/classify_negative_result_informativeness.py", + "--entry", json.dumps(_make_entry()), + "--out-md", str(out)], + capture_output=True, text=True, + env={"PYTHONPATH": "src"}, + cwd=str(_REPO_DIR), + ) + assert r.returncode == 0 + assert out.exists() + content = out.read_text() + assert "INFORMATIVE" in content + assert "Computational classification only" in content + + +class TestExampleFile: + def test_example_file_validates_informative(self): + """The example file should classify as INFORMATIVE.""" + path = _REPO_DIR / "examples/negative_result_entry_example.json" + assert path.exists() + entry = load_entry(path) + result = classify_entry(entry) + assert result["classification"] == "INFORMATIVE" + assert result["total_score"] >= 6.0 + + def test_example_file_is_marked_example(self): + """The example file should have an EXAMPLE warning.""" + path = _REPO_DIR / "examples/negative_result_entry_example.json" + entry = load_entry(path) + assert "EXAMPLE" in json.dumps(entry).upper() or "NOT REAL" in json.dumps(entry).upper() diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index 2a0a8718..16acf88f 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -2,9 +2,9 @@ import subprocess import sys -# Baseline: 2758 tests as of Loop 75 (2026-07-09) — Phase F F5 (safe-publication filter) +# Baseline: 2796 tests as of Loop 76 (2026-07-09) — Phase F F6 (negative-result informativeness guide) # Raise threshold if legitimate tests are removed; lower threshold is okay. -BASELINE = 2758 +BASELINE = 2796 TOLERANCE = 0.05 # allow 5% fluctuation