diff --git a/Makefile b/Makefile index fd461bc5..843fcc10 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance simulation-ensemble-check simulation-ci-report simulation-deprecation-check simulation-scope-check simulation-evidence-packet artifact-version candidate-manifest artifact-changelog integration-check adapter-author-check license-check artifact-compat-check contribution-check adoption-scorecard release-gate-check decision-log release-request-check coi-check rotation-plan-check security-report-check citation-check roadmap-sync-check advisory-review-check annual-review-check selection-rationale-check batch-priority-check pilot-package-check calibration-intake-check uncertainty-report-check preprint-bundle-check reproducibility-manifest-check candidate-summary-card-check multi-candidate-comparison-check pipeline-decision-audit-check claim-to-evidence-check dataset-release-check +.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance simulation-ensemble-check simulation-ci-report simulation-deprecation-check simulation-scope-check simulation-evidence-packet artifact-version candidate-manifest artifact-changelog integration-check adapter-author-check license-check artifact-compat-check contribution-check adoption-scorecard release-gate-check decision-log release-request-check coi-check rotation-plan-check security-report-check citation-check roadmap-sync-check advisory-review-check annual-review-check selection-rationale-check batch-priority-check pilot-package-check calibration-intake-check uncertainty-report-check preprint-bundle-check reproducibility-manifest-check candidate-summary-card-check multi-candidate-comparison-check pipeline-decision-audit-check claim-to-evidence-check score-decomposition-check dataset-release-check PYTHON := $(shell [ -f .venv/bin/python ] && echo .venv/bin/python || echo python3) PYTEST := $(shell [ -f .venv/bin/pytest ] && echo .venv/bin/pytest || echo pytest) @@ -939,6 +939,9 @@ pipeline-decision-audit-check: claim-to-evidence-check: .venv/bin/openamp-foundry claim-to-evidence-check --entry-json '{"mapping_id":"CEM-001","batch_id":"BATCH-001","pipeline_version":"0.9.0","claim_text":"Test claim","claim_type":"activity_prediction","supporting_artifact_ids":["CERT-001","SEL-001"],"evidence_level":3,"pre_specified":true,"reviewer":"test@example.com","dry_lab_only":true}' --format text +score-decomposition-check: + .venv/bin/openamp-foundry score-decomposition-check --entry-json '{"report_id":"SDR-001","batch_id":"BATCH-001","candidate_id":"AMP-042","pipeline_version":"0.9.1","composite_score":0.75,"component_scores":{"activity":0.8,"safety":0.7,"novelty":0.75},"component_weights":{"activity":0.5,"safety":0.3,"novelty":0.2},"scoring_method":"additive_weighted","score_range_min":0.0,"score_range_max":1.0,"reviewer":"test@example.com","dry_lab_only":true}' --format text + dataset-release-check: .venv/bin/openamp-foundry dataset-release-check \ --entry-json '{"release_id":"DSR-001","dataset_name":"OpenAMP Wave 0.5 Candidates","dataset_version":"1.0.0","release_date":"2026-07-09","license_identifier":"CC-BY-4.0","data_sources":["UniProt AMP database","APD3","in-house pipeline"],"contains_sequences":true,"contains_activity_data":false,"dual_use_assessed":true,"usage_policy_url":"https://openamp.example.org/data-policy","contact_email":"data@openamp.example.org","release_approved":true,"dry_lab_only":true}' \ diff --git a/docs/evidence/METRICS_CURRENT.md b/docs/evidence/METRICS_CURRENT.md index 242621d5..885369a8 100644 --- a/docs/evidence/METRICS_CURRENT.md +++ b/docs/evidence/METRICS_CURRENT.md @@ -5,7 +5,8 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **Purpose:** One authoritative table of current pipeline metrics. If any doc disagrees > with this file, this file wins. Updated whenever benchmark/benchmark config changes. > -> **Last updated:** 2026-07-09 (Phase M M2 — claim-to-evidence mapper schema — v0.9.0) +> **Last updated:** 2026-07-10 (Phase M M3 — score decomposition report schema — v0.9.1) +> **New in v0.9.1:** Score decomposition report schema (M3 — Loop 130) — docs/evidence/SCORE_DECOMPOSITION_GUIDE.md with purpose, schema fields table (12 fields), valid scoring methods (6), validation rules (9), warning conditions (4), honest-use boundary. src/openamp_foundry/evidence/score_decomposition_report.py with ScoreDecompositionEntry dataclass (12 fields: report_id, batch_id, candidate_id, pipeline_version, composite_score, component_scores, component_weights, scoring_method, score_range_min, score_range_max, reviewer, dry_lab_only=True), ScoreDecompositionResult dataclass (7 fields: report_id, batch_id, candidate_id, scoring_method, passed, errors, warnings, dry_lab_only=True), VALID_SCORING_METHODS (6: additive_weighted, geometric_mean, harmonic_mean, max_component, min_component, rank_aggregation), MINIMUM_COMPONENTS (2), WEIGHT_SUM_TOLERANCE (0.01), DOMINANT_WEIGHT_THRESHOLD (0.6), UNBALANCED_RATIO_THRESHOLD (5.0), MAX_COMPONENTS_WARNING (8), LOW_SCORE_FRACTION (0.2), validate_score_decomposition() (9 error checks: SDR- prefix, score_range_min=2 components, weight keys match score keys, weights sum to ~1.0, valid scoring_method, non-empty reviewer, dry_lab_only must be True; 4 warning conditions: dominant component, unbalanced weights, many components, low composite score), validate_score_decomposition_dict() (11 required fields guard). CLI (openamp-foundry score-decomposition-check) with --entry-json, --format text|json. make score-decomposition-check target. Phase M M3: every composite score is now machine-decomposable into its named components for external audit. > **New in v0.9.0:** Claim-to-evidence mapper schema (M2 — Loop 129) — docs/evidence/CLAIM_TO_EVIDENCE_GUIDE.md with purpose, schema fields table (10 fields), valid claim types (7), validation rules (7), warning conditions (4), honest-use boundary, CLI usage. src/openamp_foundry/evidence/claim_to_evidence_mapper.py with ClaimToEvidenceEntry dataclass (10 fields: mapping_id, batch_id, pipeline_version, claim_text, claim_type, supporting_artifact_ids, evidence_level, pre_specified, reviewer, dry_lab_only=True), ClaimToEvidenceResult dataclass (7 fields: mapping_id, batch_id, claim_type, passed, errors, warnings, dry_lab_only=True), VALID_CLAIM_TYPES (7: activity_prediction, calibration_statement, novelty_claim, performance_comparison, reproducibility_claim, safety_assessment, selection_rationale), VALID_EVIDENCE_LEVELS (6: 1-6), MAX_CLAIM_TEXT_LENGTH (500), LONG_CLAIM_TEXT_THRESHOLD (300), WEAK_EVIDENCE_THRESHOLD (2), validate_claim_to_evidence() (7 error checks: CEM- prefix, non-empty claim_text<=500 chars, valid claim_type, non-empty supporting_artifact_ids, valid evidence_level 1-6, non-empty reviewer, dry_lab_only must be True; 4 warning conditions: post-hoc, weak evidence, single artifact, long claim text), validate_claim_to_evidence_dict() (9 required fields guard). CLI (openamp-foundry claim-to-evidence-check) with --entry-json, --format text|json. make claim-to-evidence-check target. Phase M M2: every scientific claim is now machine-mapped to its supporting artifacts for external audit. > **New in v0.8.9:** Pipeline decision audit entry schema (M1 — Loop 128) — docs/evidence/PIPELINE_DECISION_AUDIT_GUIDE.md with purpose, required field table (13 fields), valid decision types (7), warnings, validation workflow, honest-use boundary. src/openamp_foundry/evidence/pipeline_decision_audit.py with PipelineDecisionAuditEntry dataclass (13 fields: audit_id, batch_id, pipeline_version, decision_date, decision_type, decision_description, rationale, alternatives_considered, affected_candidate_count, evidence_level, pre_specified, reviewer, dry_lab_only), PipelineDecisionAuditResult dataclass (5 fields: audit_id, batch_id, decision_type, passed, errors, warnings, dry_lab_only=True), VALID_DECISION_TYPES (7: benchmark_updated, calibration_adjusted, candidate_ranked, candidate_rejected, filter_applied, safety_flag_applied, threshold_chosen), VALID_EVIDENCE_LEVELS (6: 1-6), MAX_DESCRIPTION_LENGTH (500), MAX_RATIONALE_LENGTH (1000), validate_pipeline_decision_audit() (12 checks: AUD- prefix, non-empty batch_id/pipeline_version/reviewer, YYYY-MM-DD decision_date, valid decision_type, non-empty description<=500 chars, non-empty rationale<=1000 chars, affected_candidate_count>=0, valid evidence_level, dry_lab_only must be True; post-hoc warns, empty alternatives warns, low evidence warns, zero affected warns), validate_pipeline_decision_audit_dict() (12 required fields guard). CLI (openamp-foundry pipeline-decision-audit-check) with --entry-json, --format text|json. make pipeline-decision-audit-check target. Phase M M1: every pipeline decision is now machine-validated and traceable for external audit. > **New in v0.8.8:** Dataset release package checker (L5 — Loop 127) — docs/evidence/DATASET_RELEASE_GUIDE.md with purpose, required field table (13 fields), valid license identifiers (5), warnings, validation workflow, honest-use boundary. src/openamp_foundry/evidence/dataset_release.py with DatasetReleaseEntry dataclass (13 fields: release_id, dataset_name, dataset_version, release_date, license_identifier, data_sources, contains_sequences, contains_activity_data, dual_use_assessed, usage_policy_url, contact_email, release_approved, dry_lab_only), DatasetReleaseResult dataclass (4 fields: release_id, dataset_name, passed, errors, warnings, dry_lab_only=True), VALID_LICENSE_IDENTIFIERS (5: Apache-2.0, CC-BY-4.0, CC-BY-NC-4.0, CC0-1.0, MIT), MINIMUM_DATA_SOURCES (1), validate_dataset_release() (11 checks: DSR- prefix, non-empty dataset_name/dataset_version/usage_policy_url/contact_email, YYYY-MM-DD release_date, valid license_identifier, data_sources>=1, dual_use_assessed must be True, release_approved must be True, dry_lab_only must be True; CC-BY-NC-4.0 warns, single source warns), validate_dataset_release_dict() (12 required fields guard). CLI (openamp-foundry dataset-release-check) with --entry-json, --format text|json. make dataset-release-check target. Phase L L5 completes Phase L: open dataset releases now have machine-validated data governance checks. @@ -125,7 +126,7 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **New in v0.5.31:** Added dipeptide-order features for sequence-order awareness. `dipeptide_order_score` achieves AUROC 0.7861 on AMP-vs-scrambled discrimination — the strongest order-dependent feature in the pipeline. Only 7/31 features survive scrambling (amphipathicity/helix-wheel + dipeptide). All composition features are purely position-independent (exactly 0.5000 AUROC on scrambled test). > **New in v0.5.30:** Easy baseline benchmark added — charge density alone (AUROC 0.8166) outperforms the full pipeline ensemble (0.7792) on AMP-vs-Swiss-Prot-decoy discrimination. Honest finding documented: expected because pipeline optimizes for safety, not raw discrimination. > **New in v0.5.29:** Expanded benchmark to 500 AMPs + 500 composition-matched decoys (n=1000). AUROC 0.7792 (CI₉₅: 0.7505–0.8065) confirms signal generalizes. Cluster-aware CI: 0.746–0.8102. Representative AUROC: 0.778. Standard benchmark (n=191) retained for backward comparison. -> **Pipeline version:** v0.7.7 (J9 — external advisory review process) +> **Pipeline version:** v0.9.1 (M3 — score decomposition report) > **Branch:** main --- @@ -1195,7 +1196,7 @@ Decoys score low on activity. Selective AMPs score moderately on both. | Metric | Value | |--------|-------| -| Total tests | 3516 | +| Total tests | 4162 | | Coverage (branch) | 99% (6 CLI guard lines only) | | Source modules at 100% | All pipeline, QC, scoring, adapter modules | diff --git a/docs/evidence/SCORE_DECOMPOSITION_GUIDE.md b/docs/evidence/SCORE_DECOMPOSITION_GUIDE.md new file mode 100644 index 00000000..9b9e5312 --- /dev/null +++ b/docs/evidence/SCORE_DECOMPOSITION_GUIDE.md @@ -0,0 +1,58 @@ +# Score Decomposition Report Guide + +## Purpose + +Every composite score used to rank or filter candidates must be decomposable +into its constituent sub-scores. This schema records that decomposition so +external auditors can verify that the ranking algorithm behaves as documented. + +## Schema Fields + +| Field | Type | Description | +|---|---|---| +| report_id | str | Unique identifier (prefix \"SDR-\") | +| batch_id | str | Batch this report belongs to | +| candidate_id | str | Candidate being scored | +| pipeline_version | str | Pipeline version string | +| composite_score | float | Final aggregated score | +| component_scores | dict[str, float] | Named sub-scores (at least 2 required) | +| component_weights | dict[str, float] | Weight assigned to each component (must sum to ~1.0) | +| scoring_method | str | Algorithm used (see valid methods) | +| score_range_min | float | Minimum possible composite score | +| score_range_max | float | Maximum possible composite score | +| reviewer | str | Reviewer who validated this decomposition | +| dry_lab_only | bool | Always True for computational scoring | + +## Valid Scoring Methods + +- `additive_weighted` — weighted sum of components +- `geometric_mean` — geometric mean of components +- `harmonic_mean` — harmonic mean of components +- `max_component` — maximum of component scores +- `min_component` — minimum of component scores (conservative) +- `rank_aggregation` — aggregated from per-component ranks + +## Validation Rules + +1. report_id must start with \"SDR-\" +2. composite_score must be between score_range_min and score_range_max (inclusive) +3. component_scores must contain at least 2 components +4. component_weights keys must match component_scores keys exactly +5. component_weights values must sum to between 0.99 and 1.01 (tolerance for float rounding) +6. scoring_method must be one of the valid methods +7. score_range_min must be less than score_range_max +8. reviewer must not be empty +9. dry_lab_only must be True + +## Warning Conditions + +- Dominant component: any single component weight > 0.6 → warn about over-reliance +- Low composite: composite_score < 20% of score range → warn that candidate is weakly ranked +- Unbalanced: max weight / min weight > 5.0 → warn about highly unbalanced weighting +- Many components: len(component_scores) > 8 → warn about scoring complexity + +## Honest Use Boundary + +Score decomposition records are computational. They document how the pipeline +ranks candidates, not how candidates perform in biological assays. A high +composite score means \"ranked highly by the algorithm\" — not \"active AMP\". diff --git a/docs/research/NEXT_100_PR_MAP.md b/docs/research/NEXT_100_PR_MAP.md index bb91d2e8..05149585 100644 --- a/docs/research/NEXT_100_PR_MAP.md +++ b/docs/research/NEXT_100_PR_MAP.md @@ -221,7 +221,7 @@ Make every pipeline decision traceable, independently verifiable, and resistant |---:|---|---|---| | M1 | Add pipeline decision audit entry schema (complete — Loop 128). | Records filter/threshold/rank decisions with rationale for external audit. | B | | M2 | Add claim-to-evidence mapper schema (complete — Loop 129). | Maps each scientific claim to the artifact that supports it. | B | -| M3 | Add score decomposition report schema. | Documents how composite scores are computed from components. | B | +| M3 | Add score decomposition report schema (complete — Loop 130). | Documents how composite scores are computed from components. | B | | M4 | Add reviewer briefing package checker. | One-stop summary that external reviewers need before auditing. | B | | M5 | Add audit chain completeness checker. | Validates the evidence chain has no gaps from sequences to submission. | B/D | diff --git a/docs/research/ROADMAP.md b/docs/research/ROADMAP.md index eb1c0d1d..f5a6794c 100644 --- a/docs/research/ROADMAP.md +++ b/docs/research/ROADMAP.md @@ -1,5 +1,11 @@ # Roadmap +### v0.9.1 — Phase M M3: Score Decomposition Report +- Added `ScoreDecompositionEntry` schema documenting how composite scores decompose into components +- 6 valid scoring methods; weight-sum tolerance; dominant/unbalanced/low-score warnings +- CLI: `openamp-foundry score-decomposition-check` +- 54 tests; all passing + ### v0.9.0 — Phase M M2: Claim-to-Evidence Mapper - Added `ClaimToEvidenceEntry` schema mapping each claim to supporting artifacts - 7 valid claim types; 4 warning conditions for exploratory or weakly-evidenced claims diff --git a/src/openamp_foundry/cli/commands/reports.py b/src/openamp_foundry/cli/commands/reports.py index 8d316af4..2957def3 100644 --- a/src/openamp_foundry/cli/commands/reports.py +++ b/src/openamp_foundry/cli/commands/reports.py @@ -3225,3 +3225,36 @@ def _run_claim_to_evidence_check(args: argparse.Namespace) -> int: print(" Claim-to-evidence mapping validated. Claim is traceable to supporting artifacts.") return 0 if result.passed else 3 + + +def _run_score_decomposition_check(args: argparse.Namespace) -> int: + import json + from openamp_foundry.evidence.score_decomposition_report import validate_score_decomposition_dict + + try: + entry_dict = json.loads(args.entry_json) + except json.JSONDecodeError as exc: + print(json.dumps({"status": "error", "error": f"Invalid JSON: {exc}"})) + return 2 + + if not isinstance(entry_dict, dict): + print(json.dumps({"status": "error", "error": "--entry-json must be a JSON object"})) + return 2 + + result = validate_score_decomposition_dict(entry_dict) + output_format = getattr(args, "format", "text") + + if output_format == "json": + import dataclasses + print(json.dumps(dataclasses.asdict(result), indent=2)) + else: + status = "PASS" if result.passed else "FAIL" + print(f"[{status}] Score Decomposition Check: {result.report_id} ({result.candidate_id})") + for e in result.errors: + print(f" ERROR: {e}") + for w in result.warnings: + print(f" WARN: {w}") + if result.passed: + print(" Score decomposition validated. Composite score is auditable.") + + return 0 if result.passed else 3 diff --git a/src/openamp_foundry/cli/main.py b/src/openamp_foundry/cli/main.py index 6329db3f..eed6eeaf 100644 --- a/src/openamp_foundry/cli/main.py +++ b/src/openamp_foundry/cli/main.py @@ -15,7 +15,7 @@ from openamp_foundry.cli.commands.reports import _run_security_report_check from openamp_foundry.cli.commands.reports import _run_citation_check from openamp_foundry.cli.commands.reports import _run_roadmap_sync_check -from openamp_foundry.cli.commands.reports import _run_advisory_review_check, _run_annual_review_check, _run_selection_rationale_check, _run_batch_priority_check, _run_pilot_package_check, _run_calibration_intake_check, _run_uncertainty_report_check, _run_preprint_bundle_check, _run_reproducibility_manifest_check, _run_candidate_summary_card_check, _run_multi_candidate_comparison_check, _run_dataset_release_check, _run_pipeline_decision_audit_check, _run_claim_to_evidence_check +from openamp_foundry.cli.commands.reports import _run_advisory_review_check, _run_annual_review_check, _run_selection_rationale_check, _run_batch_priority_check, _run_pilot_package_check, _run_calibration_intake_check, _run_uncertainty_report_check, _run_preprint_bundle_check, _run_reproducibility_manifest_check, _run_candidate_summary_card_check, _run_multi_candidate_comparison_check, _run_dataset_release_check, _run_pipeline_decision_audit_check, _run_claim_to_evidence_check, _run_score_decomposition_check import argparse import json @@ -2197,6 +2197,24 @@ def build_parser() -> argparse.ArgumentParser: help="Output format (default: text).", ) + # ── Score decomposition check (Phase M M3) ───────────────────────── + sdc = sub.add_parser( + "score-decomposition-check", + help=( + "Validate a score decomposition report entry. Documents how " + "composite scores are computed from named components. Dry-lab only." + ), + ) + sdc.add_argument( + "--entry-json", type=str, required=True, + help="JSON string of a ScoreDecompositionEntry dict (required).", + ) + sdc.add_argument( + "--format", type=str, default="text", + choices=["text", "json"], + help="Output format (default: text).", + ) + # ── Selection rationale check (Phase K K1) ─────────────────────── src2 = sub.add_parser( "selection-rationale-check", @@ -2538,6 +2556,9 @@ def main(argv: list[str] | None = None) -> int: if args.command == "claim-to-evidence-check": return _run_claim_to_evidence_check(args) + if args.command == "score-decomposition-check": + return _run_score_decomposition_check(args) + parser.error("unknown command") return 2 diff --git a/src/openamp_foundry/evidence/score_decomposition_report.py b/src/openamp_foundry/evidence/score_decomposition_report.py new file mode 100644 index 00000000..fd823fe7 --- /dev/null +++ b/src/openamp_foundry/evidence/score_decomposition_report.py @@ -0,0 +1,203 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Dict, List + +VALID_SCORING_METHODS: set[str] = { + "additive_weighted", + "geometric_mean", + "harmonic_mean", + "max_component", + "min_component", + "rank_aggregation", +} + +MINIMUM_COMPONENTS: int = 2 +WEIGHT_SUM_TOLERANCE: float = 0.01 +DOMINANT_WEIGHT_THRESHOLD: float = 0.6 +UNBALANCED_RATIO_THRESHOLD: float = 5.0 +MAX_COMPONENTS_WARNING: int = 8 +LOW_SCORE_FRACTION: float = 0.2 + + +@dataclass +class ScoreDecompositionEntry: + report_id: str + batch_id: str + candidate_id: str + pipeline_version: str + composite_score: float + component_scores: Dict[str, float] + component_weights: Dict[str, float] + scoring_method: str + score_range_min: float + score_range_max: float + reviewer: str + dry_lab_only: bool = True + + +@dataclass +class ScoreDecompositionResult: + report_id: str + batch_id: str + candidate_id: str + scoring_method: str + passed: bool + errors: List[str] = field(default_factory=list) + warnings: List[str] = field(default_factory=list) + dry_lab_only: bool = True + + +def validate_score_decomposition(entry: ScoreDecompositionEntry) -> ScoreDecompositionResult: + errors: List[str] = [] + warnings: List[str] = [] + + if not entry.report_id.startswith("SDR-"): + errors.append(f"report_id must start with 'SDR-', got '{entry.report_id}'") + + if entry.score_range_min >= entry.score_range_max: + errors.append( + f"score_range_min ({entry.score_range_min}) must be less than " + f"score_range_max ({entry.score_range_max})" + ) + + if not errors: + if not (entry.score_range_min <= entry.composite_score <= entry.score_range_max): + errors.append( + f"composite_score {entry.composite_score} is outside " + f"[{entry.score_range_min}, {entry.score_range_max}]" + ) + + if len(entry.component_scores) < MINIMUM_COMPONENTS: + errors.append( + f"component_scores must have at least {MINIMUM_COMPONENTS} components, " + f"got {len(entry.component_scores)}" + ) + + score_keys = set(entry.component_scores.keys()) + weight_keys = set(entry.component_weights.keys()) + if score_keys != weight_keys: + missing_in_weights = score_keys - weight_keys + extra_in_weights = weight_keys - score_keys + if missing_in_weights: + errors.append( + f"component_weights missing keys present in component_scores: " + f"{sorted(missing_in_weights)}" + ) + if extra_in_weights: + errors.append( + f"component_weights has keys not in component_scores: " + f"{sorted(extra_in_weights)}" + ) + + if entry.component_weights: + weight_sum = sum(entry.component_weights.values()) + if abs(weight_sum - 1.0) > WEIGHT_SUM_TOLERANCE: + errors.append( + f"component_weights must sum to 1.0 (tolerance ±{WEIGHT_SUM_TOLERANCE}), " + f"got {weight_sum:.4f}" + ) + + if entry.scoring_method not in VALID_SCORING_METHODS: + errors.append( + f"scoring_method '{entry.scoring_method}' is not valid; " + f"must be one of {sorted(VALID_SCORING_METHODS)}" + ) + + if not entry.reviewer: + errors.append("reviewer must not be empty") + + if not entry.dry_lab_only: + errors.append( + "dry_lab_only must be True for score decomposition reports " + "(computational scoring only)" + ) + + if not errors and entry.component_weights: + weights = list(entry.component_weights.values()) + max_weight = max(weights) + min_weight = min(weights) + + if max_weight > DOMINANT_WEIGHT_THRESHOLD: + dominant_key = max(entry.component_weights, key=entry.component_weights.__getitem__) + warnings.append( + f"component '{dominant_key}' has weight {max_weight:.2f} > " + f"{DOMINANT_WEIGHT_THRESHOLD}; composite score is dominated by one component" + ) + + if min_weight > 0 and max_weight / min_weight > UNBALANCED_RATIO_THRESHOLD: + warnings.append( + f"weight ratio max/min = {max_weight / min_weight:.1f} > " + f"{UNBALANCED_RATIO_THRESHOLD}; scoring weights are highly unbalanced" + ) + + if len(entry.component_scores) > MAX_COMPONENTS_WARNING: + warnings.append( + f"{len(entry.component_scores)} components is complex (> {MAX_COMPONENTS_WARNING}); " + "consider simplifying the scoring model" + ) + + if not errors: + score_range = entry.score_range_max - entry.score_range_min + low_threshold = entry.score_range_min + LOW_SCORE_FRACTION * score_range + if entry.composite_score < low_threshold: + warnings.append( + f"composite_score {entry.composite_score:.3f} is in the bottom " + f"{int(LOW_SCORE_FRACTION * 100)}% of the score range; " + "candidate is weakly ranked" + ) + + passed = len(errors) == 0 + return ScoreDecompositionResult( + report_id=entry.report_id, + batch_id=entry.batch_id, + candidate_id=entry.candidate_id, + scoring_method=entry.scoring_method, + passed=passed, + errors=errors, + warnings=warnings, + dry_lab_only=True, + ) + + +def validate_score_decomposition_dict(d: dict) -> ScoreDecompositionResult: + required_fields = [ + "report_id", + "batch_id", + "candidate_id", + "pipeline_version", + "composite_score", + "component_scores", + "component_weights", + "scoring_method", + "score_range_min", + "score_range_max", + "reviewer", + ] + missing = [f for f in required_fields if f not in d] + if missing: + return ScoreDecompositionResult( + report_id=d.get("report_id", "UNKNOWN"), + batch_id=d.get("batch_id", "UNKNOWN"), + candidate_id=d.get("candidate_id", "UNKNOWN"), + scoring_method=d.get("scoring_method", "UNKNOWN"), + passed=False, + errors=[f"Missing required fields: {missing}"], + dry_lab_only=True, + ) + + entry = ScoreDecompositionEntry( + report_id=d["report_id"], + batch_id=d["batch_id"], + candidate_id=d["candidate_id"], + pipeline_version=d["pipeline_version"], + composite_score=float(d["composite_score"]), + component_scores={k: float(v) for k, v in d["component_scores"].items()}, + component_weights={k: float(v) for k, v in d["component_weights"].items()}, + scoring_method=d["scoring_method"], + score_range_min=float(d["score_range_min"]), + score_range_max=float(d["score_range_max"]), + reviewer=d["reviewer"], + dry_lab_only=d.get("dry_lab_only", True), + ) + return validate_score_decomposition(entry) diff --git a/tests/evidence/test_score_decomposition_report.py b/tests/evidence/test_score_decomposition_report.py new file mode 100644 index 00000000..a5dfbd93 --- /dev/null +++ b/tests/evidence/test_score_decomposition_report.py @@ -0,0 +1,469 @@ +import pytest +from openamp_foundry.evidence.score_decomposition_report import ( + ScoreDecompositionEntry, + ScoreDecompositionResult, + validate_score_decomposition, + validate_score_decomposition_dict, + VALID_SCORING_METHODS, + MINIMUM_COMPONENTS, + WEIGHT_SUM_TOLERANCE, + DOMINANT_WEIGHT_THRESHOLD, + UNBALANCED_RATIO_THRESHOLD, + MAX_COMPONENTS_WARNING, + LOW_SCORE_FRACTION, +) + + +def _valid_entry(**overrides) -> ScoreDecompositionEntry: + base = dict( + report_id="SDR-001", + batch_id="BATCH-001", + candidate_id="AMP-042", + pipeline_version="0.9.1", + composite_score=0.75, + component_scores={"activity": 0.8, "safety": 0.7, "novelty": 0.75}, + component_weights={"activity": 0.5, "safety": 0.3, "novelty": 0.2}, + scoring_method="additive_weighted", + score_range_min=0.0, + score_range_max=1.0, + reviewer="reviewer@example.com", + dry_lab_only=True, + ) + base.update(overrides) + return ScoreDecompositionEntry(**base) + + +def _valid_dict(**overrides) -> dict: + base = dict( + report_id="SDR-001", + batch_id="BATCH-001", + candidate_id="AMP-042", + pipeline_version="0.9.1", + composite_score=0.75, + component_scores={"activity": 0.8, "safety": 0.7, "novelty": 0.75}, + component_weights={"activity": 0.5, "safety": 0.3, "novelty": 0.2}, + scoring_method="additive_weighted", + score_range_min=0.0, + score_range_max=1.0, + reviewer="reviewer@example.com", + dry_lab_only=True, + ) + base.update(overrides) + return base + + +# --- Happy path --- + +def test_valid_entry_passes(): + result = validate_score_decomposition(_valid_entry()) + assert result.passed + assert result.errors == [] + assert result.warnings == [] + + +def test_result_dry_lab_only_always_true(): + result = validate_score_decomposition(_valid_entry()) + assert result.dry_lab_only is True + + +def test_result_fields_populated(): + result = validate_score_decomposition(_valid_entry()) + assert result.report_id == "SDR-001" + assert result.batch_id == "BATCH-001" + assert result.candidate_id == "AMP-042" + assert result.scoring_method == "additive_weighted" + + +def test_all_scoring_methods_valid(): + for method in VALID_SCORING_METHODS: + result = validate_score_decomposition(_valid_entry(scoring_method=method)) + assert result.passed, f"scoring_method={method} should pass" + + +def test_exactly_two_components_passes(): + result = validate_score_decomposition( + _valid_entry( + component_scores={"a": 0.6, "b": 0.8}, + component_weights={"a": 0.5, "b": 0.5}, + ) + ) + assert result.passed + + +def test_composite_at_min_passes(): + result = validate_score_decomposition(_valid_entry(composite_score=0.0)) + assert result.passed + + +def test_composite_at_max_passes(): + result = validate_score_decomposition(_valid_entry(composite_score=1.0)) + assert result.passed + + +# --- report_id validation --- + +def test_report_id_must_start_with_sdr(): + result = validate_score_decomposition(_valid_entry(report_id="RPT-001")) + assert not result.passed + assert any("SDR-" in e for e in result.errors) + + +def test_report_id_empty_fails(): + result = validate_score_decomposition(_valid_entry(report_id="")) + assert not result.passed + + +def test_report_id_sdr_prefix_valid(): + result = validate_score_decomposition(_valid_entry(report_id="SDR-999")) + assert result.passed + + +# --- score range validation --- + +def test_score_range_min_equals_max_fails(): + result = validate_score_decomposition( + _valid_entry(score_range_min=0.5, score_range_max=0.5) + ) + assert not result.passed + assert any("score_range_min" in e for e in result.errors) + + +def test_score_range_min_greater_than_max_fails(): + result = validate_score_decomposition( + _valid_entry(score_range_min=1.0, score_range_max=0.0) + ) + assert not result.passed + + +def test_composite_below_min_fails(): + result = validate_score_decomposition( + _valid_entry(composite_score=-0.1, score_range_min=0.0, score_range_max=1.0) + ) + assert not result.passed + assert any("outside" in e for e in result.errors) + + +def test_composite_above_max_fails(): + result = validate_score_decomposition( + _valid_entry(composite_score=1.1, score_range_min=0.0, score_range_max=1.0) + ) + assert not result.passed + + +# --- component counts --- + +def test_single_component_fails(): + result = validate_score_decomposition( + _valid_entry( + component_scores={"activity": 0.8}, + component_weights={"activity": 1.0}, + ) + ) + assert not result.passed + assert any("at least" in e for e in result.errors) + + +def test_empty_components_fails(): + result = validate_score_decomposition( + _valid_entry(component_scores={}, component_weights={}) + ) + assert not result.passed + + +def test_many_components_warns(): + scores = {f"comp_{i}": 0.5 for i in range(9)} + weights = {k: 1.0 / 9 for k in scores} + result = validate_score_decomposition( + _valid_entry(component_scores=scores, component_weights=weights) + ) + assert result.passed + assert any("complex" in w.lower() or str(MAX_COMPONENTS_WARNING) in w for w in result.warnings) + + +# --- weight key matching --- + +def test_weights_missing_key_fails(): + result = validate_score_decomposition( + _valid_entry( + component_scores={"activity": 0.8, "safety": 0.7}, + component_weights={"activity": 1.0}, + ) + ) + assert not result.passed + assert any("missing" in e.lower() for e in result.errors) + + +def test_weights_extra_key_fails(): + result = validate_score_decomposition( + _valid_entry( + component_scores={"activity": 0.8, "safety": 0.7}, + component_weights={"activity": 0.5, "safety": 0.3, "extra": 0.2}, + ) + ) + assert not result.passed + assert any("not in component_scores" in e for e in result.errors) + + +# --- weight sum validation --- + +def test_weights_sum_to_one_passes(): + result = validate_score_decomposition( + _valid_entry( + component_scores={"a": 0.5, "b": 0.5}, + component_weights={"a": 0.5, "b": 0.5}, + ) + ) + assert result.passed + + +def test_weights_sum_to_0_992_passes(): + result = validate_score_decomposition( + _valid_entry( + component_scores={"a": 0.5, "b": 0.5}, + component_weights={"a": 0.496, "b": 0.496}, + ) + ) + assert result.passed + + +def test_weights_sum_to_0_98_fails(): + result = validate_score_decomposition( + _valid_entry( + component_scores={"a": 0.5, "b": 0.5}, + component_weights={"a": 0.49, "b": 0.49}, + ) + ) + assert not result.passed + assert any("sum" in e for e in result.errors) + + +def test_weights_sum_over_1_01_fails(): + result = validate_score_decomposition( + _valid_entry( + component_scores={"a": 0.5, "b": 0.5}, + component_weights={"a": 0.51, "b": 0.51}, + ) + ) + assert not result.passed + + +# --- scoring method --- + +def test_invalid_scoring_method_fails(): + result = validate_score_decomposition(_valid_entry(scoring_method="made_up_method")) + assert not result.passed + assert any("scoring_method" in e for e in result.errors) + + +def test_empty_scoring_method_fails(): + result = validate_score_decomposition(_valid_entry(scoring_method="")) + assert not result.passed + + +# --- reviewer --- + +def test_empty_reviewer_fails(): + result = validate_score_decomposition(_valid_entry(reviewer="")) + assert not result.passed + assert any("reviewer" in e for e in result.errors) + + +# --- dry_lab_only --- + +def test_dry_lab_only_false_fails(): + result = validate_score_decomposition(_valid_entry(dry_lab_only=False)) + assert not result.passed + assert any("dry_lab_only" in e for e in result.errors) + + +# --- warnings --- + +def test_dominant_weight_warns(): + result = validate_score_decomposition( + _valid_entry( + component_scores={"activity": 0.8, "safety": 0.7}, + component_weights={"activity": 0.7, "safety": 0.3}, + ) + ) + assert result.passed + assert any("dominated" in w.lower() or "dominant" in w.lower() or str(DOMINANT_WEIGHT_THRESHOLD) in w for w in result.warnings) + + +def test_unbalanced_weights_warn(): + result = validate_score_decomposition( + _valid_entry( + component_scores={"a": 0.5, "b": 0.5, "c": 0.5}, + component_weights={"a": 0.9, "b": 0.09, "c": 0.01}, + ) + ) + assert result.passed + assert any("unbalanced" in w.lower() for w in result.warnings) + + +def test_low_score_warns(): + result = validate_score_decomposition( + _valid_entry(composite_score=0.1, score_range_min=0.0, score_range_max=1.0) + ) + assert result.passed + assert any("weakly ranked" in w.lower() or "bottom" in w.lower() for w in result.warnings) + + +def test_score_above_low_threshold_no_low_warning(): + result = validate_score_decomposition( + _valid_entry(composite_score=0.5, score_range_min=0.0, score_range_max=1.0) + ) + assert not any("weakly ranked" in w.lower() for w in result.warnings) + + +# --- dict interface --- + +def test_valid_dict_passes(): + result = validate_score_decomposition_dict(_valid_dict()) + assert result.passed + + +def test_missing_report_id_fails(): + d = _valid_dict() + del d["report_id"] + result = validate_score_decomposition_dict(d) + assert not result.passed + assert any("Missing" in e for e in result.errors) + + +def test_missing_component_scores_fails(): + d = _valid_dict() + del d["component_scores"] + result = validate_score_decomposition_dict(d) + assert not result.passed + + +def test_missing_reviewer_fails(): + d = _valid_dict() + del d["reviewer"] + result = validate_score_decomposition_dict(d) + assert not result.passed + + +def test_dict_dry_lab_only_defaults_true(): + d = _valid_dict() + del d["dry_lab_only"] + result = validate_score_decomposition_dict(d) + assert result.passed + assert result.dry_lab_only is True + + +def test_multiple_missing_fields(): + d = {} + result = validate_score_decomposition_dict(d) + assert not result.passed + assert any("Missing" in e for e in result.errors) + + +# --- constants --- + +def test_valid_scoring_methods_count(): + assert len(VALID_SCORING_METHODS) == 6 + + +def test_minimum_components_value(): + assert MINIMUM_COMPONENTS == 2 + + +def test_weight_sum_tolerance_value(): + assert WEIGHT_SUM_TOLERANCE == 0.01 + + +def test_dominant_weight_threshold_value(): + assert DOMINANT_WEIGHT_THRESHOLD == 0.6 + + +def test_unbalanced_ratio_threshold_value(): + assert UNBALANCED_RATIO_THRESHOLD == 5.0 + + +def test_max_components_warning_value(): + assert MAX_COMPONENTS_WARNING == 8 + + +def test_low_score_fraction_value(): + assert LOW_SCORE_FRACTION == 0.2 + + +# --- Additional edge cases --- + +def test_negative_score_range_min_passes(): + result = validate_score_decomposition( + _valid_entry(score_range_min=-1.0, score_range_max=1.0, composite_score=0.0) + ) + assert result.passed + + +def test_negative_composite_score_passes(): + result = validate_score_decomposition( + _valid_entry(score_range_min=-1.0, score_range_max=1.0, composite_score=-0.5) + ) + assert result.passed + + +def test_dict_missing_pipeline_version_fails(): + d = _valid_dict() + del d["pipeline_version"] + result = validate_score_decomposition_dict(d) + assert not result.passed + + +def test_dict_missing_batch_id_fails(): + d = _valid_dict() + del d["batch_id"] + result = validate_score_decomposition_dict(d) + assert not result.passed + + +def test_dict_empty_component_scores_fails(): + result = validate_score_decomposition_dict(_valid_dict(component_scores={}, component_weights={})) + assert not result.passed + + +def test_result_fields_match(): + entry = _valid_entry(composite_score=0.5, scoring_method="harmonic_mean") + result = validate_score_decomposition(entry) + assert result.report_id == "SDR-001" + assert result.scoring_method == "harmonic_mean" + assert result.passed is True + assert result.dry_lab_only is True + + +def test_dict_all_valid_methods(): + for method in VALID_SCORING_METHODS: + d = _valid_dict(scoring_method=method) + result = validate_score_decomposition_dict(d) + assert result.passed, f"dict scoring_method={method} should pass" + + +def test_score_at_low_threshold_boundary_no_warning(): + result = validate_score_decomposition( + _valid_entry(composite_score=0.2, score_range_min=0.0, score_range_max=1.0) + ) + assert result.passed + assert not any("weakly ranked" in w.lower() for w in result.warnings) + + +def test_no_warnings_on_balanced_weights(): + result = validate_score_decomposition( + _valid_entry( + component_scores={"a": 0.5, "b": 0.5}, + component_weights={"a": 0.5, "b": 0.5}, + ) + ) + assert result.passed + assert len(result.warnings) == 0 + + +def test_dict_float_conversion(): + d = _valid_dict( + composite_score="0.75", + component_scores={"a": "0.6", "b": "0.8"}, + component_weights={"a": "0.5", "b": "0.5"}, + ) + result = validate_score_decomposition_dict(d) + assert result.passed diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index c919d3b9..f1d02052 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -6,7 +6,7 @@ # Note: 3 pre-existing collection errors in test_benchmark_calibration.py, # test_benchmark_charge_distribution.py, test_benchmark_cheap_enemies.py. # Raise threshold if legitimate tests are removed; lower threshold is okay. -BASELINE = 4108 +BASELINE = 4162 TOLERANCE = 0.05 # allow 5% fluctuation