From 99833d86117e8db08b0f47cadbd3194f3684d83c Mon Sep 17 00:00:00 2001 From: OpenCode Date: Fri, 10 Jul 2026 00:47:21 +0700 Subject: [PATCH] feat: Phase N N2 hypothesis outcome record schema -- closes pre-registration loop by documenting confirmed/refuted outcomes --- Makefile | 5 +- docs/evidence/HYPOTHESIS_OUTCOME_GUIDE.md | 59 +++ docs/evidence/METRICS_CURRENT.md | 5 +- docs/research/NEXT_100_PR_MAP.md | 2 +- docs/research/ROADMAP.md | 8 + src/openamp_foundry/cli/commands/reports.py | 20 + src/openamp_foundry/cli/main.py | 16 +- .../evidence/hypothesis_outcome_record.py | 163 +++++++ .../test_hypothesis_outcome_record.py | 432 ++++++++++++++++++ tests/test_test_count_regression.py | 2 +- 10 files changed, 706 insertions(+), 6 deletions(-) create mode 100644 docs/evidence/HYPOTHESIS_OUTCOME_GUIDE.md create mode 100644 src/openamp_foundry/evidence/hypothesis_outcome_record.py create mode 100644 tests/evidence/test_hypothesis_outcome_record.py diff --git a/Makefile b/Makefile index fb13cd5e..d297ffc7 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance simulation-ensemble-check simulation-ci-report simulation-deprecation-check simulation-scope-check simulation-evidence-packet artifact-version candidate-manifest artifact-changelog integration-check adapter-author-check license-check artifact-compat-check contribution-check adoption-scorecard release-gate-check decision-log release-request-check coi-check rotation-plan-check security-report-check citation-check roadmap-sync-check advisory-review-check annual-review-check selection-rationale-check batch-priority-check pilot-package-check calibration-intake-check uncertainty-report-check preprint-bundle-check reproducibility-manifest-check candidate-summary-card-check multi-candidate-comparison-check pipeline-decision-audit-check claim-to-evidence-check score-decomposition-check reviewer-briefing-check audit-chain-check pre-registration-check dataset-release-check +.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance simulation-ensemble-check simulation-ci-report simulation-deprecation-check simulation-scope-check simulation-evidence-packet artifact-version candidate-manifest artifact-changelog integration-check adapter-author-check license-check artifact-compat-check contribution-check adoption-scorecard release-gate-check decision-log release-request-check coi-check rotation-plan-check security-report-check citation-check roadmap-sync-check advisory-review-check annual-review-check selection-rationale-check batch-priority-check pilot-package-check calibration-intake-check uncertainty-report-check preprint-bundle-check reproducibility-manifest-check candidate-summary-card-check multi-candidate-comparison-check pipeline-decision-audit-check claim-to-evidence-check score-decomposition-check reviewer-briefing-check audit-chain-check pre-registration-check hypothesis-outcome-check dataset-release-check PYTHON := $(shell [ -f .venv/bin/python ] && echo .venv/bin/python || echo python3) PYTEST := $(shell [ -f .venv/bin/pytest ] && echo .venv/bin/pytest || echo pytest) @@ -951,6 +951,9 @@ audit-chain-check: pre-registration-check: openamp-foundry pre-registration-check --entry-json '{"registration_id":"PRE-001","batch_id":"BATCH-001","pipeline_version":"0.9.4","registration_date":"2026-07-10","primary_hypothesis":"Candidates selected by OpenAMP will show MIC values at least 2-fold lower than random length/charge-matched peptides in broth microdilution against E. coli ATCC 25922.","primary_outcome_metric":"mic_value","success_threshold":4.0,"baseline_comparators":["random_selection","charge_matched_random"],"candidate_ids":["AMP-001","AMP-002","AMP-003"],"assay_type":"mic_assay","statistical_test":"Mann-Whitney U test, two-sided, alpha=0.05","registered_by":"test@example.com","dry_lab_only":true}' --format text +hypothesis-outcome-check: + openamp-foundry hypothesis-outcome-check --entry-json '{"outcome_id":"HOR-001","registration_id":"PRE-001","batch_id":"BATCH-001","pipeline_version":"0.9.5","outcome_date":"2026-07-15","outcome_verdict":"confirmed","observed_metric_value":2.5,"success_threshold_met":true,"interpretation":"Selected candidates showed median MIC of 2.5 ug/mL, meeting the threshold. Random baseline showed 32 ug/mL. Hypothesis confirmed with strong signal vs random.","deviation_from_plan":"","recorded_by":"test@example.com","dry_lab_only":true}' --format text + dataset-release-check: .venv/bin/openamp-foundry dataset-release-check \ --entry-json '{"release_id":"DSR-001","dataset_name":"OpenAMP Wave 0.5 Candidates","dataset_version":"1.0.0","release_date":"2026-07-09","license_identifier":"CC-BY-4.0","data_sources":["UniProt AMP database","APD3","in-house pipeline"],"contains_sequences":true,"contains_activity_data":false,"dual_use_assessed":true,"usage_policy_url":"https://openamp.example.org/data-policy","contact_email":"data@openamp.example.org","release_approved":true,"dry_lab_only":true}' \ diff --git a/docs/evidence/HYPOTHESIS_OUTCOME_GUIDE.md b/docs/evidence/HYPOTHESIS_OUTCOME_GUIDE.md new file mode 100644 index 00000000..5efe68b7 --- /dev/null +++ b/docs/evidence/HYPOTHESIS_OUTCOME_GUIDE.md @@ -0,0 +1,59 @@ +# Hypothesis Outcome Record Guide + +## Purpose + +A hypothesis outcome record documents whether a pre-registered hypothesis was +confirmed or refuted after experimental results are received. It links the +outcome to the pre-registration form, records the observed metric value, +and requires an explicit interpretation. This closes the HARKing-prevention +loop: the pre-registration (N1) commits to what will be tested; the outcome +record (N2) reports what was found, whether convenient or not. + +## Schema Fields + +| Field | Type | Description | +|---|---|---| +| outcome_id | str | Unique identifier (prefix "HOR-") | +| registration_id | str | Matching PRE- registration this outcome closes | +| batch_id | str | Batch that was tested | +| pipeline_version | str | Pipeline version used | +| outcome_date | str | ISO 8601 date results were recorded | +| outcome_verdict | str | Whether hypothesis was confirmed or refuted (see valid verdicts) | +| observed_metric_value | float | Actual observed value for the primary outcome metric | +| success_threshold_met | bool | Whether the pre-specified threshold was met | +| interpretation | str | Researcher's plain-language interpretation (max 500 chars) | +| deviation_from_plan | str | Any deviation from pre-registered plan (empty string if none) | +| recorded_by | str | Person or system that recorded this outcome | +| dry_lab_only | bool | False if real lab data; True if computational simulation only | + +## Valid Outcome Verdicts + +- `confirmed` — hypothesis was supported by observations +- `refuted` — hypothesis was not supported by observations +- `inconclusive` — results were ambiguous or experiment had quality issues +- `partially_confirmed` — hypothesis was supported for a subset of candidates + +## Validation Rules + +1. outcome_id must start with "HOR-" +2. registration_id must start with "PRE-" +3. outcome_verdict must be one of the valid verdicts +4. observed_metric_value must be finite (not NaN or Inf) +5. interpretation must not be empty and must not exceed 500 characters +6. recorded_by must not be empty + +## Warning Conditions + +- Verdict/threshold mismatch: success_threshold_met=True but outcome_verdict="refuted" → warn about inconsistency +- Verdict/threshold mismatch: success_threshold_met=False but outcome_verdict="confirmed" → warn about inconsistency +- No deviation documented: outcome_verdict is "inconclusive" but deviation_from_plan is empty → warn that deviations should be documented when results are inconclusive +- Short interpretation: interpretation < 50 chars → warn that interpretation may be underspecified + +## Honest Use Boundary + +A hypothesis outcome record is scientific evidence of what happened. A +"refuted" verdict is not a failure — it is information. Do not suppress, +reframe, or omit refuted outcomes. The value of pre-registration comes +entirely from honest reporting of all outcomes including inconvenient ones. +Selectively reporting confirmed outcomes while omitting refuted ones +defeats the entire purpose and should be treated as a protocol violation. diff --git a/docs/evidence/METRICS_CURRENT.md b/docs/evidence/METRICS_CURRENT.md index 42383035..0f8cccaa 100644 --- a/docs/evidence/METRICS_CURRENT.md +++ b/docs/evidence/METRICS_CURRENT.md @@ -5,8 +5,9 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **Purpose:** One authoritative table of current pipeline metrics. If any doc disagrees > with this file, this file wins. Updated whenever benchmark/benchmark config changes. > -> **Last updated:** 2026-07-10 (Phase N N1 — pre-registration form — v0.9.4) -> **New in v0.9.4:** Pre-registration form schema (N1 — Loop 133) — docs/evidence/PRE_REGISTRATION_GUIDE.md with purpose, schema fields table (13 fields: registration_id, batch_id, pipeline_version, registration_date, primary_hypothesis, primary_outcome_metric, success_threshold, baseline_comparators, candidate_ids, assay_type, statistical_test, registered_by, dry_lab_only), 6 valid primary outcome metrics, 5 valid assay types, 10 validation rules, 4 warning conditions, honest-use boundary. PreRegistrationEntry dataclass (13 fields), PreRegistrationResult dataclass (6 fields), validate_pre_registration() (10+ error checks: PRE- prefix, non-empty hypothesis<=500 chars, valid outcome metric, finite success_threshold, >=1 baseline_comparators, >=1 candidate_ids, valid assay_type, non-empty statistical_test<=200 chars, non-empty registered_by, dry_lab_only must be True; 4 warning conditions: no random baseline, large candidate set >20, short hypothesis <50 chars, placeholder statistical test TBD/N/A/NA/none), validate_pre_registration_dict() (12 required fields guard). CLI (openamp-foundry pre-registration-check) with --entry-json, --format text|json. make pre-registration-check target. 62 tests. Phase N (Pre-registration & Baseline Honesty) started with N1. +> **Last updated:** 2026-07-10 (Phase N N2 — hypothesis outcome record — v0.9.5) +> **New in v0.9.5:** Hypothesis outcome record schema (N2 — Loop 134) — docs/evidence/HYPOTHESIS_OUTCOME_GUIDE.md with purpose, schema fields table (12 fields: outcome_id, registration_id, batch_id, pipeline_version, outcome_date, outcome_verdict, observed_metric_value, success_threshold_met, interpretation, deviation_from_plan, recorded_by, dry_lab_only), 4 valid outcome verdicts (confirmed, refuted, inconclusive, partially_confirmed), 6 validation rules, 4 warning conditions (threshold/verdict inconsistency, inconclusive no deviation, short interpretation), honest-use boundary. HypothesisOutcomeEntry dataclass (12 fields), HypothesisOutcomeResult dataclass (7 fields), validate_hypothesis_outcome() (6+ error checks: HOR- prefix, PRE- prefix, valid verdict, finite metric, non-empty interpretation<=500, non-empty recorded_by; 4 warning conditions: threshold/verdict mismatch, inconclusive no deviation, short interpretation), validate_hypothesis_outcome_dict() (11 required fields guard). CLI (openamp-foundry hypothesis-outcome-check) with --entry-json, --format text|json. make hypothesis-outcome-check target. 48 tests. Phase N N2: the pre-registration loop is now closed — outcomes are recorded and matched to their commitments. +> > **New in v0.9.4:** Pre-registration form schema (N1 — Loop 133) — docs/evidence/PRE_REGISTRATION_GUIDE.md with purpose, schema fields table (13 fields: registration_id, batch_id, pipeline_version, registration_date, primary_hypothesis, primary_outcome_metric, success_threshold, baseline_comparators, candidate_ids, assay_type, statistical_test, registered_by, dry_lab_only), 6 valid primary outcome metrics, 5 valid assay types, 10 validation rules, 4 warning conditions, honest-use boundary. PreRegistrationEntry dataclass (13 fields), PreRegistrationResult dataclass (6 fields), validate_pre_registration() (10+ error checks: PRE- prefix, non-empty hypothesis<=500 chars, valid outcome metric, finite success_threshold, >=1 baseline_comparators, >=1 candidate_ids, valid assay_type, non-empty statistical_test<=200 chars, non-empty registered_by, dry_lab_only must be True; 4 warning conditions: no random baseline, large candidate set >20, short hypothesis <50 chars, placeholder statistical test TBD/N/A/NA/none), validate_pre_registration_dict() (12 required fields guard). CLI (openamp-foundry pre-registration-check) with --entry-json, --format text|json. make pre-registration-check target. 62 tests. Phase N (Pre-registration & Baseline Honesty) started with N1. > > **New in v0.9.3:** Audit chain completeness checker (M5 — Loop 132) — docs/evidence/AUDIT_CHAIN_GUIDE.md with purpose, schema fields table (16 fields: chain_id, batch_id, pipeline_version, audit_date, 9 has_* bools, missing_links, auditor, dry_lab_only), 9 required chain links, validation rules (5), warning conditions (2: single auditor, future date), honest-use boundary. src/openamp_foundry/evidence/audit_chain_completeness.py with AuditChainEntry dataclass (16 fields: chain_id, batch_id, pipeline_version, audit_date, 9 has_* bools, missing_links list[str], auditor, dry_lab_only=True), AuditChainResult dataclass (7 fields: chain_id, batch_id, missing_link_count, passed, errors, warnings, dry_lab_only=True), CHAIN_LINK_FIELDS (9 entries), CHAIN_LINK_COUNT (9), AUDITOR_EMAIL_HINT ("@"), IMPLAUSIBLE_YEAR_THRESHOLD (2030), validate_audit_chain() (5+ error checks: ACH- prefix, non-empty auditor, dry_lab_only must be True, each false chain link errors, missing_links consistency; 2 warnings: no email, future date), validate_audit_chain_dict() (15 required fields guard). CLI (openamp-foundry audit-chain-check) with --entry-json, --format text|json. make audit-chain-check target. 39 tests. Phase M M5: the evidence chain is now machine-checkable end to end. Phase M (Audit Trail Infrastructure) is fully complete with M1-M5. > **New in v0.9.1:** Score decomposition report schema (M3 — Loop 130) — docs/evidence/SCORE_DECOMPOSITION_GUIDE.md with purpose, schema fields table (12 fields), valid scoring methods (6), validation rules (9), warning conditions (4), honest-use boundary. src/openamp_foundry/evidence/score_decomposition_report.py with ScoreDecompositionEntry dataclass (12 fields: report_id, batch_id, candidate_id, pipeline_version, composite_score, component_scores, component_weights, scoring_method, score_range_min, score_range_max, reviewer, dry_lab_only=True), ScoreDecompositionResult dataclass (7 fields: report_id, batch_id, candidate_id, scoring_method, passed, errors, warnings, dry_lab_only=True), VALID_SCORING_METHODS (6: additive_weighted, geometric_mean, harmonic_mean, max_component, min_component, rank_aggregation), MINIMUM_COMPONENTS (2), WEIGHT_SUM_TOLERANCE (0.01), DOMINANT_WEIGHT_THRESHOLD (0.6), UNBALANCED_RATIO_THRESHOLD (5.0), MAX_COMPONENTS_WARNING (8), LOW_SCORE_FRACTION (0.2), validate_score_decomposition() (9 error checks: SDR- prefix, score_range_min=2 components, weight keys match score keys, weights sum to ~1.0, valid scoring_method, non-empty reviewer, dry_lab_only must be True; 4 warning conditions: dominant component, unbalanced weights, many components, low composite score), validate_score_decomposition_dict() (11 required fields guard). CLI (openamp-foundry score-decomposition-check) with --entry-json, --format text|json. make score-decomposition-check target. Phase M M3: every composite score is now machine-decomposable into its named components for external audit. > **New in v0.9.0:** Claim-to-evidence mapper schema (M2 — Loop 129) — docs/evidence/CLAIM_TO_EVIDENCE_GUIDE.md with purpose, schema fields table (10 fields), valid claim types (7), validation rules (7), warning conditions (4), honest-use boundary, CLI usage. src/openamp_foundry/evidence/claim_to_evidence_mapper.py with ClaimToEvidenceEntry dataclass (10 fields: mapping_id, batch_id, pipeline_version, claim_text, claim_type, supporting_artifact_ids, evidence_level, pre_specified, reviewer, dry_lab_only=True), ClaimToEvidenceResult dataclass (7 fields: mapping_id, batch_id, claim_type, passed, errors, warnings, dry_lab_only=True), VALID_CLAIM_TYPES (7: activity_prediction, calibration_statement, novelty_claim, performance_comparison, reproducibility_claim, safety_assessment, selection_rationale), VALID_EVIDENCE_LEVELS (6: 1-6), MAX_CLAIM_TEXT_LENGTH (500), LONG_CLAIM_TEXT_THRESHOLD (300), WEAK_EVIDENCE_THRESHOLD (2), validate_claim_to_evidence() (7 error checks: CEM- prefix, non-empty claim_text<=500 chars, valid claim_type, non-empty supporting_artifact_ids, valid evidence_level 1-6, non-empty reviewer, dry_lab_only must be True; 4 warning conditions: post-hoc, weak evidence, single artifact, long claim text), validate_claim_to_evidence_dict() (9 required fields guard). CLI (openamp-foundry claim-to-evidence-check) with --entry-json, --format text|json. make claim-to-evidence-check target. Phase M M2: every scientific claim is now machine-mapped to its supporting artifacts for external audit. diff --git a/docs/research/NEXT_100_PR_MAP.md b/docs/research/NEXT_100_PR_MAP.md index 60356267..cdb60bc6 100644 --- a/docs/research/NEXT_100_PR_MAP.md +++ b/docs/research/NEXT_100_PR_MAP.md @@ -232,7 +232,7 @@ Make experiment predictions falsifiable before results are observed. Guard again | PR | Task | Why it matters | Review class | |---:|---|---|---| | N1 | Add pre-registration form schema (complete — Loop 133). | Records what will be tested and how before experiments run. | B | -| N2 | Add hypothesis outcome record schema. | Documents whether pre-registered hypotheses were confirmed or refuted. | B/D | +| N2 | Add hypothesis outcome record schema (complete — Loop 134). | Documents whether pre-registered hypotheses were confirmed or refuted. | B/D | | N3 | Add baseline comparison manifest schema. | Machine-verifiable proof the model beats cheap baselines. | B | | N4 | Add negative result record schema. | Ensures failed experiments are documented, not discarded. | B | | N5 | Add experiment priority justification schema. | Documents why this batch was selected over alternatives. | B | diff --git a/docs/research/ROADMAP.md b/docs/research/ROADMAP.md index 44db82ef..3b0e20f9 100644 --- a/docs/research/ROADMAP.md +++ b/docs/research/ROADMAP.md @@ -1,5 +1,13 @@ # Roadmap +### v0.9.5 — Phase N N2: Hypothesis Outcome Record +- Added `HypothesisOutcomeEntry` schema linking pre-registration to actual experimental outcomes +- Records confirmed/refuted/inconclusive/partially_confirmed verdicts with observed metric value +- Warns on threshold/verdict inconsistency and undocumented inconclusive deviations +- dry_lab_only=False supported (real lab data can be recorded) +- CLI: `openamp-foundry hypothesis-outcome-check` +- 48 tests; all passing + ### v0.9.4 — Phase N N1: Pre-Registration Form (starts Phase N) - Added `PreRegistrationEntry` schema for machine-verifiable experiment pre-commitment - Records hypothesis, outcome metric, success threshold, and baseline comparators before results are observed diff --git a/src/openamp_foundry/cli/commands/reports.py b/src/openamp_foundry/cli/commands/reports.py index 3ba99555..aa32e9de 100644 --- a/src/openamp_foundry/cli/commands/reports.py +++ b/src/openamp_foundry/cli/commands/reports.py @@ -3328,3 +3328,23 @@ def _run_pre_registration_check(args: argparse.Namespace) -> None: print(f" ERROR: {e}") for w in result.warnings: print(f" WARN: {w}") + + +def _run_hypothesis_outcome_check(args: argparse.Namespace) -> None: + import json + from openamp_foundry.evidence.hypothesis_outcome_record import validate_hypothesis_outcome_dict + + entry_dict = json.loads(args.entry_json) + result = validate_hypothesis_outcome_dict(entry_dict) + + if args.format == "json": + import dataclasses + print(json.dumps(dataclasses.asdict(result), indent=2)) + else: + status = "PASS" if result.passed else "FAIL" + threshold_status = "MET" if result.success_threshold_met else "NOT MET" + print(f"[{status}] Hypothesis Outcome: {result.outcome_id} -> {result.outcome_verdict} (threshold {threshold_status})") + for e in result.errors: + print(f" ERROR: {e}") + for w in result.warnings: + print(f" WARN: {w}") diff --git a/src/openamp_foundry/cli/main.py b/src/openamp_foundry/cli/main.py index 34a3a9bf..c3dc772a 100644 --- a/src/openamp_foundry/cli/main.py +++ b/src/openamp_foundry/cli/main.py @@ -5,7 +5,7 @@ from openamp_foundry.cli.commands.selection import _run_pilot_panel, _run_pilot_confident, _run_diversity_check, _run_select_batch, _run_batch_rationale from openamp_foundry.cli.commands.external import _run_external_predict, _run_external_consensus from openamp_foundry.cli.commands.qc import _run_synthesis_order, _run_presynth_qc -from openamp_foundry.cli.commands.reports import _run_reviewer_questionnaire, _run_ip_report, _run_batch_pack, _run_gold_standard, _run_novelty_check_broad, _run_lab_result_report, _run_calibration_intake, _run_recalibration_gate, _run_recalibration_engine, _run_validate_policy_version, _run_calibration_audit, _run_calibration_overfit_check, _run_result_quality_filter, _run_synthetic_result_policy_check, _run_calibration_decision_checklist, _run_calibration_rollback_plan, _run_simulation_registry, _run_validate_simulation_result, _run_simulation_baseline_check, _run_adapter_gate_check, _run_simulation_provenance, _run_simulation_ensemble_check, _run_simulation_ci_report, _run_simulation_deprecation_check, _run_simulation_scope_check, _run_simulation_evidence_packet, _run_artifact_version, _run_candidate_manifest, _run_benchmark_card, _run_artifact_changelog, _run_integration_check, _run_adapter_check, _run_license_check, _run_artifact_compat_check, _run_adoption_scorecard, _run_reviewer_briefing_check, _run_audit_chain_check, _run_pre_registration_check +from openamp_foundry.cli.commands.reports import _run_reviewer_questionnaire, _run_ip_report, _run_batch_pack, _run_gold_standard, _run_novelty_check_broad, _run_lab_result_report, _run_calibration_intake, _run_recalibration_gate, _run_recalibration_engine, _run_validate_policy_version, _run_calibration_audit, _run_calibration_overfit_check, _run_result_quality_filter, _run_synthetic_result_policy_check, _run_calibration_decision_checklist, _run_calibration_rollback_plan, _run_simulation_registry, _run_validate_simulation_result, _run_simulation_baseline_check, _run_adapter_gate_check, _run_simulation_provenance, _run_simulation_ensemble_check, _run_simulation_ci_report, _run_simulation_deprecation_check, _run_simulation_scope_check, _run_simulation_evidence_packet, _run_artifact_version, _run_candidate_manifest, _run_benchmark_card, _run_artifact_changelog, _run_integration_check, _run_adapter_check, _run_license_check, _run_artifact_compat_check, _run_adoption_scorecard, _run_reviewer_briefing_check, _run_audit_chain_check, _run_pre_registration_check, _run_hypothesis_outcome_check from openamp_foundry.cli.commands.gates import _run_gate_check, _run_release_gate_check from openamp_foundry.cli.commands.reports import _run_contribution_check from openamp_foundry.cli.commands.reports import _run_decision_log @@ -2253,6 +2253,17 @@ def build_parser() -> argparse.ArgumentParser: ) pre_registration_parser.set_defaults(func=_run_pre_registration_check) + # ── Hypothesis outcome check (Phase N N2) ──────────────────────── + hypothesis_outcome_parser = sub.add_parser( + "hypothesis-outcome-check", + help="Validate a hypothesis outcome record entry", + ) + hypothesis_outcome_parser.add_argument("--entry-json", required=True) + hypothesis_outcome_parser.add_argument( + "--format", choices=["text", "json"], default="text" + ) + hypothesis_outcome_parser.set_defaults(func=_run_hypothesis_outcome_check) + # ── Selection rationale check (Phase K K1) ─────────────────────── src2 = sub.add_parser( "selection-rationale-check", @@ -2606,6 +2617,9 @@ def main(argv: list[str] | None = None) -> int: if args.command == "pre-registration-check": return _run_pre_registration_check(args) + if args.command == "hypothesis-outcome-check": + return _run_hypothesis_outcome_check(args) + parser.error("unknown command") return 2 diff --git a/src/openamp_foundry/evidence/hypothesis_outcome_record.py b/src/openamp_foundry/evidence/hypothesis_outcome_record.py new file mode 100644 index 00000000..69e9a800 --- /dev/null +++ b/src/openamp_foundry/evidence/hypothesis_outcome_record.py @@ -0,0 +1,163 @@ +from __future__ import annotations + +import math +from dataclasses import dataclass, field +from typing import List + +VALID_OUTCOME_VERDICTS: set[str] = { + "confirmed", + "inconclusive", + "partially_confirmed", + "refuted", +} + +MAX_INTERPRETATION_LENGTH: int = 500 +MIN_INTERPRETATION_LENGTH: int = 50 +REGISTRATION_ID_PREFIX: str = "PRE-" +OUTCOME_ID_PREFIX: str = "HOR-" + + +@dataclass +class HypothesisOutcomeEntry: + outcome_id: str + registration_id: str + batch_id: str + pipeline_version: str + outcome_date: str + outcome_verdict: str + observed_metric_value: float + success_threshold_met: bool + interpretation: str + deviation_from_plan: str + recorded_by: str + dry_lab_only: bool = True + + +@dataclass +class HypothesisOutcomeResult: + outcome_id: str + registration_id: str + outcome_verdict: str + success_threshold_met: bool + passed: bool + errors: List[str] = field(default_factory=list) + warnings: List[str] = field(default_factory=list) + dry_lab_only: bool = True + + +def validate_hypothesis_outcome(entry: HypothesisOutcomeEntry) -> HypothesisOutcomeResult: + errors: List[str] = [] + warnings: List[str] = [] + + if not entry.outcome_id.startswith(OUTCOME_ID_PREFIX): + errors.append( + f"outcome_id must start with '{OUTCOME_ID_PREFIX}', got '{entry.outcome_id}'" + ) + + if not entry.registration_id.startswith(REGISTRATION_ID_PREFIX): + errors.append( + f"registration_id must start with '{REGISTRATION_ID_PREFIX}', " + f"got '{entry.registration_id}'" + ) + + if entry.outcome_verdict not in VALID_OUTCOME_VERDICTS: + errors.append( + f"outcome_verdict '{entry.outcome_verdict}' is not valid; " + f"must be one of {sorted(VALID_OUTCOME_VERDICTS)}" + ) + + if math.isnan(entry.observed_metric_value) or math.isinf(entry.observed_metric_value): + errors.append( + f"observed_metric_value must be finite, got {entry.observed_metric_value}" + ) + + if not entry.interpretation: + errors.append("interpretation must not be empty") + elif len(entry.interpretation) > MAX_INTERPRETATION_LENGTH: + errors.append( + f"interpretation exceeds {MAX_INTERPRETATION_LENGTH} characters " + f"(got {len(entry.interpretation)})" + ) + + if not entry.recorded_by: + errors.append("recorded_by must not be empty") + + if not errors: + if entry.success_threshold_met and entry.outcome_verdict == "refuted": + warnings.append( + "success_threshold_met=True but outcome_verdict='refuted'; " + "these are inconsistent — verify threshold logic and verdict" + ) + + if not entry.success_threshold_met and entry.outcome_verdict == "confirmed": + warnings.append( + "success_threshold_met=False but outcome_verdict='confirmed'; " + "these are inconsistent — verify threshold logic and verdict" + ) + + if entry.outcome_verdict == "inconclusive" and not entry.deviation_from_plan: + warnings.append( + "outcome_verdict='inconclusive' but deviation_from_plan is empty; " + "document what went wrong or deviated so the inconclusive result is interpretable" + ) + + if entry.interpretation and len(entry.interpretation) < MIN_INTERPRETATION_LENGTH: + warnings.append( + f"interpretation is short ({len(entry.interpretation)} chars < " + f"{MIN_INTERPRETATION_LENGTH}); may be underspecified" + ) + + passed = len(errors) == 0 + return HypothesisOutcomeResult( + outcome_id=entry.outcome_id, + registration_id=entry.registration_id, + outcome_verdict=entry.outcome_verdict, + success_threshold_met=entry.success_threshold_met, + passed=passed, + errors=errors, + warnings=warnings, + dry_lab_only=entry.dry_lab_only, + ) + + +def validate_hypothesis_outcome_dict(d: dict) -> HypothesisOutcomeResult: + required_fields = [ + "outcome_id", + "registration_id", + "batch_id", + "pipeline_version", + "outcome_date", + "outcome_verdict", + "observed_metric_value", + "success_threshold_met", + "interpretation", + "deviation_from_plan", + "recorded_by", + ] + missing = [f for f in required_fields if f not in d] + if missing: + return HypothesisOutcomeResult( + outcome_id=d.get("outcome_id", "UNKNOWN"), + registration_id=d.get("registration_id", "UNKNOWN"), + outcome_verdict=d.get("outcome_verdict", "UNKNOWN"), + success_threshold_met=False, + passed=False, + errors=[f"Missing required fields: {missing}"], + dry_lab_only=d.get("dry_lab_only", True), + ) + + entry = HypothesisOutcomeEntry( + outcome_id=d["outcome_id"], + registration_id=d["registration_id"], + batch_id=d["batch_id"], + pipeline_version=d["pipeline_version"], + outcome_date=d["outcome_date"], + outcome_verdict=d["outcome_verdict"], + observed_metric_value=float(d["observed_metric_value"]), + success_threshold_met=bool(d["success_threshold_met"]), + interpretation=d["interpretation"], + deviation_from_plan=d["deviation_from_plan"], + recorded_by=d["recorded_by"], + dry_lab_only=d.get("dry_lab_only", True), + ) + return validate_hypothesis_outcome(entry) diff --git a/tests/evidence/test_hypothesis_outcome_record.py b/tests/evidence/test_hypothesis_outcome_record.py new file mode 100644 index 00000000..4c4abd7e --- /dev/null +++ b/tests/evidence/test_hypothesis_outcome_record.py @@ -0,0 +1,432 @@ +import math +import pytest +from openamp_foundry.evidence.hypothesis_outcome_record import ( + HypothesisOutcomeEntry, + HypothesisOutcomeResult, + validate_hypothesis_outcome, + validate_hypothesis_outcome_dict, + VALID_OUTCOME_VERDICTS, + MAX_INTERPRETATION_LENGTH, + MIN_INTERPRETATION_LENGTH, + REGISTRATION_ID_PREFIX, + OUTCOME_ID_PREFIX, +) + + +def _valid_entry(**overrides) -> HypothesisOutcomeEntry: + base = dict( + outcome_id="HOR-001", + registration_id="PRE-001", + batch_id="BATCH-001", + pipeline_version="0.9.5", + outcome_date="2026-07-15", + outcome_verdict="confirmed", + observed_metric_value=2.5, + success_threshold_met=True, + interpretation=( + "The selected candidates showed median MIC of 2.5 ug/mL, meeting the " + "pre-specified threshold of 4.0 ug/mL. The hypothesis is confirmed. " + "Random baseline showed 32 ug/mL median MIC." + ), + deviation_from_plan="", + recorded_by="researcher@example.com", + dry_lab_only=True, + ) + base.update(overrides) + return HypothesisOutcomeEntry(**base) + + +def _valid_dict(**overrides) -> dict: + base = dict( + outcome_id="HOR-001", + registration_id="PRE-001", + batch_id="BATCH-001", + pipeline_version="0.9.5", + outcome_date="2026-07-15", + outcome_verdict="confirmed", + observed_metric_value=2.5, + success_threshold_met=True, + interpretation=( + "The selected candidates showed median MIC of 2.5 ug/mL, meeting the " + "pre-specified threshold of 4.0 ug/mL. The hypothesis is confirmed." + ), + deviation_from_plan="", + recorded_by="researcher@example.com", + dry_lab_only=True, + ) + base.update(overrides) + return base + + +# --- Happy path --- + +def test_valid_confirmed_entry_passes(): + result = validate_hypothesis_outcome(_valid_entry()) + assert result.passed + assert result.errors == [] + assert result.warnings == [] + + +def test_valid_refuted_entry_passes(): + result = validate_hypothesis_outcome( + _valid_entry( + outcome_verdict="refuted", + success_threshold_met=False, + interpretation=( + "Candidates showed median MIC of 128 ug/mL, failing the 4 ug/mL threshold. " + "Performance was worse than the random baseline. Hypothesis refuted. " + "This is a negative result and should be recorded as such." + ), + ) + ) + assert result.passed + assert result.errors == [] + + +def test_valid_inconclusive_with_deviation_passes(): + result = validate_hypothesis_outcome( + _valid_entry( + outcome_verdict="inconclusive", + success_threshold_met=False, + interpretation=( + "Results were ambiguous due to contamination in the assay. " + "Unable to draw conclusions from this experiment. " + "Repeat assay is planned with fresh reagents." + ), + deviation_from_plan="Assay contamination detected; results discarded.", + ) + ) + assert result.passed + + +def test_valid_partially_confirmed_passes(): + result = validate_hypothesis_outcome( + _valid_entry( + outcome_verdict="partially_confirmed", + success_threshold_met=False, + interpretation=( + "3 of 10 candidates met the MIC threshold. The hypothesis is partially " + "confirmed. The pipeline shows positive signal but not broad coverage. " + "Random baseline showed 0 of 10 candidates meeting threshold." + ), + ) + ) + assert result.passed + + +def test_result_fields_populated(): + result = validate_hypothesis_outcome(_valid_entry()) + assert result.outcome_id == "HOR-001" + assert result.registration_id == "PRE-001" + assert result.outcome_verdict == "confirmed" + assert result.success_threshold_met is True + + +def test_dry_lab_only_preserved_in_result(): + result = validate_hypothesis_outcome(_valid_entry(dry_lab_only=False)) + assert result.passed + assert result.dry_lab_only is False + + +def test_all_verdicts_valid(): + for verdict in VALID_OUTCOME_VERDICTS: + entry = _valid_entry( + outcome_verdict=verdict, + success_threshold_met=(verdict == "confirmed"), + deviation_from_plan=("some deviation" if verdict == "inconclusive" else ""), + ) + result = validate_hypothesis_outcome(entry) + assert result.passed or (not result.passed and result.errors), f"verdict={verdict}" + + +def test_negative_metric_value_passes(): + result = validate_hypothesis_outcome(_valid_entry(observed_metric_value=-1.0)) + assert result.passed + + +def test_zero_metric_value_passes(): + result = validate_hypothesis_outcome(_valid_entry(observed_metric_value=0.0)) + assert result.passed + + +def test_empty_deviation_from_plan_passes_for_confirmed(): + result = validate_hypothesis_outcome(_valid_entry(deviation_from_plan="")) + assert result.passed + + +def test_deviation_documented_for_inconclusive_passes(): + result = validate_hypothesis_outcome( + _valid_entry( + outcome_verdict="inconclusive", + success_threshold_met=False, + interpretation="Results were ambiguous due to equipment failure in the assay run.", + deviation_from_plan="Equipment malfunction on day 2.", + ) + ) + assert result.passed + assert not any("deviation_from_plan" in w for w in result.warnings) + + +# --- outcome_id validation --- + +def test_outcome_id_must_start_with_hor(): + result = validate_hypothesis_outcome(_valid_entry(outcome_id="OUT-001")) + assert not result.passed + assert any("HOR-" in e for e in result.errors) + + +def test_outcome_id_empty_fails(): + result = validate_hypothesis_outcome(_valid_entry(outcome_id="")) + assert not result.passed + + +def test_outcome_id_hor_prefix_valid(): + result = validate_hypothesis_outcome(_valid_entry(outcome_id="HOR-999")) + assert result.passed + + +# --- registration_id validation --- + +def test_registration_id_must_start_with_pre(): + result = validate_hypothesis_outcome(_valid_entry(registration_id="REG-001")) + assert not result.passed + assert any("PRE-" in e for e in result.errors) + + +def test_registration_id_empty_fails(): + result = validate_hypothesis_outcome(_valid_entry(registration_id="")) + assert not result.passed + + +def test_registration_id_pre_prefix_valid(): + result = validate_hypothesis_outcome(_valid_entry(registration_id="PRE-999")) + assert result.passed + + +# --- outcome_verdict validation --- + +def test_invalid_verdict_fails(): + result = validate_hypothesis_outcome(_valid_entry(outcome_verdict="maybe")) + assert not result.passed + assert any("outcome_verdict" in e for e in result.errors) + + +def test_empty_verdict_fails(): + result = validate_hypothesis_outcome(_valid_entry(outcome_verdict="")) + assert not result.passed + + +# --- observed_metric_value --- + +def test_nan_metric_value_fails(): + result = validate_hypothesis_outcome(_valid_entry(observed_metric_value=float("nan"))) + assert not result.passed + assert any("finite" in e for e in result.errors) + + +def test_inf_metric_value_fails(): + result = validate_hypothesis_outcome(_valid_entry(observed_metric_value=float("inf"))) + assert not result.passed + + +def test_negative_inf_metric_value_fails(): + result = validate_hypothesis_outcome(_valid_entry(observed_metric_value=float("-inf"))) + assert not result.passed + + +# --- interpretation --- + +def test_empty_interpretation_fails(): + result = validate_hypothesis_outcome(_valid_entry(interpretation="")) + assert not result.passed + assert any("interpretation" in e for e in result.errors) + + +def test_interpretation_at_max_length_passes(): + result = validate_hypothesis_outcome( + _valid_entry(interpretation="A" * MAX_INTERPRETATION_LENGTH) + ) + assert result.passed + + +def test_interpretation_over_max_fails(): + result = validate_hypothesis_outcome( + _valid_entry(interpretation="A" * (MAX_INTERPRETATION_LENGTH + 1)) + ) + assert not result.passed + + +def test_short_interpretation_warns(): + result = validate_hypothesis_outcome(_valid_entry(interpretation="Too short.")) + assert result.passed + assert any("underspecified" in w.lower() or "short" in w.lower() for w in result.warnings) + + +def test_interpretation_at_min_length_no_warn(): + result = validate_hypothesis_outcome( + _valid_entry(interpretation="A" * MIN_INTERPRETATION_LENGTH) + ) + assert not any("underspecified" in w.lower() for w in result.warnings) + + +# --- recorded_by --- + +def test_empty_recorded_by_fails(): + result = validate_hypothesis_outcome(_valid_entry(recorded_by="")) + assert not result.passed + assert any("recorded_by" in e for e in result.errors) + + +# --- warnings --- + +def test_threshold_met_but_refuted_warns(): + result = validate_hypothesis_outcome( + _valid_entry(outcome_verdict="refuted", success_threshold_met=True) + ) + assert result.passed + assert any("inconsistent" in w.lower() for w in result.warnings) + + +def test_threshold_not_met_but_confirmed_warns(): + result = validate_hypothesis_outcome( + _valid_entry(outcome_verdict="confirmed", success_threshold_met=False) + ) + assert result.passed + assert any("inconsistent" in w.lower() for w in result.warnings) + + +def test_confirmed_threshold_met_no_inconsistency_warn(): + result = validate_hypothesis_outcome( + _valid_entry(outcome_verdict="confirmed", success_threshold_met=True) + ) + assert not any("inconsistent" in w.lower() for w in result.warnings) + + +def test_refuted_threshold_not_met_no_inconsistency_warn(): + result = validate_hypothesis_outcome( + _valid_entry( + outcome_verdict="refuted", + success_threshold_met=False, + interpretation=( + "Candidates did not meet threshold. Random baseline also failed. " + "Pipeline shows no signal on this assay. Results are negative and " + "should be recorded for future calibration." + ), + ) + ) + assert not any("inconsistent" in w.lower() for w in result.warnings) + + +def test_inconclusive_no_deviation_warns(): + result = validate_hypothesis_outcome( + _valid_entry( + outcome_verdict="inconclusive", + success_threshold_met=False, + interpretation=( + "Results were ambiguous and could not be interpreted clearly. " + "Unable to determine if hypothesis was supported or refuted. " + "Further experiments are needed to resolve this question." + ), + deviation_from_plan="", + ) + ) + assert result.passed + assert any("deviation" in w.lower() for w in result.warnings) + + +def test_inconclusive_with_deviation_no_warn(): + result = validate_hypothesis_outcome( + _valid_entry( + outcome_verdict="inconclusive", + success_threshold_met=False, + interpretation=( + "Results were ambiguous due to contamination detected on day 3. " + "The experiment was terminated early. Unable to draw conclusions. " + "A repeat experiment is planned with fresh reagents and controls." + ), + deviation_from_plan="Contamination detected on day 3; assay terminated early.", + ) + ) + assert not any("deviation" in w.lower() for w in result.warnings) + + +# --- dict interface --- + +def test_valid_dict_passes(): + result = validate_hypothesis_outcome_dict(_valid_dict()) + assert result.passed + + +def test_missing_outcome_id_fails(): + d = _valid_dict() + del d["outcome_id"] + result = validate_hypothesis_outcome_dict(d) + assert not result.passed + assert any("Missing" in e for e in result.errors) + + +def test_missing_registration_id_fails(): + d = _valid_dict() + del d["registration_id"] + result = validate_hypothesis_outcome_dict(d) + assert not result.passed + + +def test_missing_outcome_verdict_fails(): + d = _valid_dict() + del d["outcome_verdict"] + result = validate_hypothesis_outcome_dict(d) + assert not result.passed + + +def test_dict_dry_lab_only_defaults_true(): + d = _valid_dict() + del d["dry_lab_only"] + result = validate_hypothesis_outcome_dict(d) + assert result.passed + assert result.dry_lab_only is True + + +def test_dict_dry_lab_only_false_allowed(): + d = _valid_dict() + d["dry_lab_only"] = False + result = validate_hypothesis_outcome_dict(d) + assert result.passed + assert result.dry_lab_only is False + + +def test_multiple_missing_fields(): + d = {} + result = validate_hypothesis_outcome_dict(d) + assert not result.passed + assert any("Missing" in e for e in result.errors) + + +# --- constants --- + +def test_valid_outcome_verdicts_count(): + assert len(VALID_OUTCOME_VERDICTS) == 4 + + +def test_max_interpretation_length_value(): + assert MAX_INTERPRETATION_LENGTH == 500 + + +def test_min_interpretation_length_value(): + assert MIN_INTERPRETATION_LENGTH == 50 + + +def test_registration_id_prefix_value(): + assert REGISTRATION_ID_PREFIX == "PRE-" + + +def test_outcome_id_prefix_value(): + assert OUTCOME_ID_PREFIX == "HOR-" + + +def test_valid_verdicts_contains_refuted(): + assert "refuted" in VALID_OUTCOME_VERDICTS + + +def test_valid_verdicts_contains_confirmed(): + assert "confirmed" in VALID_OUTCOME_VERDICTS diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index 6a593f1f..eafddeba 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -6,7 +6,7 @@ # Note: 3 pre-existing collection errors in test_benchmark_calibration.py, # test_benchmark_charge_distribution.py, test_benchmark_cheap_enemies.py. # Raise threshold if legitimate tests are removed; lower threshold is okay. -BASELINE = 4257 +BASELINE = 4305 TOLERANCE = 0.05 # allow 5% fluctuation