diff --git a/Makefile b/Makefile index 8b095434..c52ddc48 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance simulation-ensemble-check simulation-ci-report simulation-deprecation-check simulation-scope-check simulation-evidence-packet artifact-version candidate-manifest artifact-changelog integration-check adapter-author-check license-check artifact-compat-check contribution-check +.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance simulation-ensemble-check simulation-ci-report simulation-deprecation-check simulation-scope-check simulation-evidence-packet artifact-version candidate-manifest artifact-changelog integration-check adapter-author-check license-check artifact-compat-check contribution-check adoption-scorecard PYTHON := $(shell [ -f .venv/bin/python ] && echo .venv/bin/python || echo python3) PYTEST := $(shell [ -f .venv/bin/pytest ] && echo .venv/bin/pytest || echo pytest) @@ -829,3 +829,8 @@ contribution-check: .venv/bin/openamp-foundry contribution-check \ --intake-json '{"institution_name":"Example University","contact_email":"research@example.edu","contribution_type":"dataset_donation","proposed_scope":"Curated AMP dataset 500 sequences","human_review_required":false,"dry_lab_only":true,"data_license":"CC-BY-4.0","dataset_description":"AMP sequences with MIC data","record_count":500}' @echo "Contribution intake check complete." + +adoption-scorecard: + .venv/bin/openamp-foundry adoption-scorecard \ + --scores-json '{"integration_check":{"passed_checks":5,"total_checks":5,"notes":""},"license_compliance":{"passed_checks":3,"total_checks":3,"notes":""},"adapter_validation":{"passed_checks":2,"total_checks":2,"notes":""},"schema_compatibility":{"passed_checks":4,"total_checks":4,"notes":""},"contribution_readiness":{"passed_checks":1,"total_checks":1,"notes":""}}' + @echo "Adoption scorecard complete." diff --git a/docs/evidence/METRICS_CURRENT.md b/docs/evidence/METRICS_CURRENT.md index bd7b24e7..e8c55c4a 100644 --- a/docs/evidence/METRICS_CURRENT.md +++ b/docs/evidence/METRICS_CURRENT.md @@ -5,7 +5,8 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **Purpose:** One authoritative table of current pipeline metrics. If any doc disagrees > with this file, this file wins. Updated whenever benchmark/benchmark config changes. > -> **Last updated:** 2026-07-09 (Phase I I9 — public-good contribution guide — v0.6.7) +> **Last updated:** 2026-07-09 (Phase I I10 — adoption scorecard dashboard — v0.6.8) +> **New in v0.6.8:** Adoption scorecard dashboard (I10 — Loop 108) — `src/openamp_foundry/adoption/scorecard.py` with `SCORECARD_DIMENSIONS` (5 weighted dimensions summing to 1.0: integration_check 0.25, license_compliance 0.20, adapter_validation 0.20, schema_compatibility 0.20, contribution_readiness 0.15), `ADOPTION_TIERS` (4 tiers: not_ready 0.0-0.40, emerging 0.40-0.65, established 0.65-0.85, mature 0.85-1.01), `DimensionScore` dataclass (8 fields, dry_lab_only=True), `AdoptionScorecard` dataclass (6 fields, dry_lab_only=True), `build_scorecard()` (aggregates dimension inputs into weighted total score with tier and actionable recommendations), `compute_adoption_tier()` (maps score to tier). CLI (`openamp-foundry adoption-scorecard`) with `--scores-json`, `--format text|json`. `make adoption-scorecard` target. 17 tests. **3446 total.** **Phase I (Interoperability and Adoption) is now complete** — all 10 items I1–I10 implemented (artifact versioning, candidate manifest, benchmark card, artifact changelog, integration checker, adapter validator, data license checker, schema compatibility, contribution intake, adoption scorecard). > **New in v0.6.7:** Public-good contribution guide (I9 — Loop 107) — `docs/community/PUBLIC_GOOD_CONTRIBUTION_GUIDE.md` with 6 contribution types (wet_lab_validation, dataset_donation, compute_sponsorship, expert_review, governance_participation, algorithm_contribution), review classes A-D, minimum requirements table, initiation process, data governance, and safety constraints. `src/openamp_foundry/community/contribution_intake.py` with `ContributionIntake` dataclass (7 fields), `IntakeValidationResult` dataclass (7 fields), `VALID_CONTRIBUTION_TYPES` (6 entries), `VALID_REVIEW_CLASSES` (4 entries), `REQUIRED_FIELDS_BY_TYPE` (6 entries), `validate_contribution_intake()` (checks all top-level fields, type-specific required fields, wet_lab_validation human_review_required=True enforcement), `validate_intake_dict()` (dict input with missing-fields guard). CLI (`openamp-foundry contribution-check`) with `--intake-json`, `--format text|json`. `make contribution-check` target. 16 tests. Dry-lab only. Funders and institutions now have a clear, validated pathway for contribution. > **New in v0.6.5:** Data license checker (I7 — Loop 105) — `src/openamp_foundry/licensing/license_checker.py` with `DataLicenseDeclaration` dataclass (11 fields: source_id, source_name, license_id, use_context, attribution_required, commercial_use_allowed, redistribution_allowed, modifications_allowed, human_review_required, notes, dry_lab_only), `LicenseCheckResult` dataclass (8 fields: source_id, license_id, use_context, passed, status, errors, warnings, dry_lab_only), `check_data_license()` (validates declarations against `APPROVED_LICENSES` (11 entries: CC0-1.0, CC-BY-4.0, CC-BY-SA-4.0, MIT, Apache-2.0, GPL-3.0, LGPL-2.1, BSD-2-Clause, BSD-3-Clause, ODbL-1.0, PDDL-1.0), `RESTRICTED_LICENSES` (4 entries: CC-BY-NC-4.0, CC-BY-NC-SA-4.0, custom, proprietary), `BLOCKED_LICENSES` (3 entries: unknown, unlicensed, all-rights-reserved), `VALID_USE_CONTEXTS` (6 entries: training, scoring, benchmarking, reporting, publication, internal). Blocked licenses fail immediately; restricted require human_review_required=True; unknown licenses require governance review before use. `check_license_batch()` summarizes total/passed/failed/blocked/any_blocked/all_passed with dry_lab_only=True. All results carry dry_lab_only=True. CLI (`openamp-foundry license-check`) with `--source-json`, `--format text|json`. `make license-check` target. 16 tests. **3429 total.** External data sources used in pipeline outputs now require explicit license declarations, preventing hidden legal risk. > **New in v0.6.3:** Downstream project template (I5 — Loop 103) — `docs/adoption/DOWNSTREAM_PROJECT_TEMPLATE.md` with overview of OpenAMP artifacts, minimum viable integration (consume candidate manifest, validate against schema, use Python library), schema validation step, evidence level interpretation table (L1-L6), safety flag conventions table, benchmark card consumption guide, explicit dry-lab limitations section, and contact/contribution section. `src/openamp_foundry/adoption/integration_checker.py` with `REQUIRED_INTEGRATION_CHECKS` (5 checks: manifest_schema_valid, evidence_level_in_range, dry_lab_only_acknowledged, safety_flags_reviewed, baseline_comparison_present), `IntegrationCheckResult` dataclass (4 fields: check_name, passed, detail, dry_lab_only), `run_integration_checks()` (returns dict with checks, passed_count, failed_count, all_passed, dry_lab_only). Exported from `src/openamp_foundry/adoption/__init__.py`. CLI (`openamp-foundry integration-check`) with `--manifest-json`, `--format text|json`. `make integration-check` target. 14 tests. **3356 total.** External researchers who want to use OpenAMP artifacts now have a guide and checker to validate their integration. @@ -95,7 +96,7 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak > **New in v0.5.31:** Added dipeptide-order features for sequence-order awareness. `dipeptide_order_score` achieves AUROC 0.7861 on AMP-vs-scrambled discrimination — the strongest order-dependent feature in the pipeline. Only 7/31 features survive scrambling (amphipathicity/helix-wheel + dipeptide). All composition features are purely position-independent (exactly 0.5000 AUROC on scrambled test). > **New in v0.5.30:** Easy baseline benchmark added — charge density alone (AUROC 0.8166) outperforms the full pipeline ensemble (0.7792) on AMP-vs-Swiss-Prot-decoy discrimination. Honest finding documented: expected because pipeline optimizes for safety, not raw discrimination. > **New in v0.5.29:** Expanded benchmark to 500 AMPs + 500 composition-matched decoys (n=1000). AUROC 0.7792 (CI₉₅: 0.7505–0.8065) confirms signal generalizes. Cluster-aware CI: 0.746–0.8102. Representative AUROC: 0.778. Standard benchmark (n=191) retained for backward comparison. -> **Pipeline version:** v0.6.7 (I9 — public-good contribution guide) +> **Pipeline version:** v0.6.8 (I10 — adoption scorecard dashboard, Phase I complete) > **Branch:** main --- @@ -1165,7 +1166,7 @@ Decoys score low on activity. Selective AMPs score moderately on both. | Metric | Value | |--------|-------| -| Total tests | 3413 | +| Total tests | 3446 | | Coverage (branch) | 99% (6 CLI guard lines only) | | Source modules at 100% | All pipeline, QC, scoring, adapter modules | @@ -1189,6 +1190,7 @@ Decoys score low on activity. Selective AMPs score moderately on both. | Date | Change | Author | |------|--------|--------| +| 2026-07-09 | **Adoption scorecard dashboard added (I10):** `src/openamp_foundry/adoption/scorecard.py` with `SCORECARD_DIMENSIONS` (5, weights sum 1.0), `ADOPTION_TIERS` (4), `DimensionScore`, `AdoptionScorecard`, `build_scorecard()`, `compute_adoption_tier()`. CLI `adoption-scorecard` with `--scores-json` and `--format`. `make adoption-scorecard`. 17 tests. **Phase I (Interoperability and Adoption) is now complete** — all 10 items I1–I10 implemented. **3446 total.** | OpenAMP Loop 108 | | 2026-07-09 | **Adapter author validator added (I6):** `src/openamp_foundry/adapters/adapter_validator.py` with `AdapterDeclaration` (14 fields), `AdapterValidationResult` (5 fields), `validate_adapter_declaration()` (10 checks enforcing ADAPTER_AUTHOR_GUIDE contract), `validate_adapter_dict()` (dict input with missing-fields guard). 4 valid-value sets (VALID_ADAPTER_MODES, VALID_OUTPUT_STATUSES, VALID_RANKING_EFFECTS, VALID_RELEASE_STATUSES). CLI (`openamp-foundry adapter-check`) with `--adapter-json`, `--format text|json`. `make adapter-author-check` target. 31 tests. **3387 total.** | OpenAMP Loop 104 | | 2026-07-08 | **Calibration benchmark added:** Brier score decomposition, reliability diagram, and calibration slope for pipeline ensemble scores. Brier=0.3178 (>0.25=uninformative), skill=-0.27 (worse than base rate), slope=0.43 (ideal=1.0). Honest finding: pipeline ranks well (AUROC~0.78) but scores are not meaningful probabilities. Expanded 500-AMP set confirms same pattern (Brier=0.2772, slope=2.31 — dataset-dependent). Integrated into `make bench-500` and `make bench-calibration`. `scripts/benchmarks/benchmark_calibration.py`, JSON output to `outputs/bench_calibration*.json`. | OpenAMP loop 18 | | 2026-06-29 | Novelty audit v2: BioPython BLOSUM62 local alignment vs 27,234 AMPs (APD6+DRAMP+UniProt); panel updated (15 families, 4 SAR_CONTROL); all 7 gates PASS | OpenAMP Wave 0.5 | diff --git a/docs/research/ROADMAP.md b/docs/research/ROADMAP.md index 073a8809..efe87122 100644 --- a/docs/research/ROADMAP.md +++ b/docs/research/ROADMAP.md @@ -1,5 +1,73 @@ # Roadmap +## v0.6.8 — Loop 108: Phase I I10 — Adoption Scorecard Dashboard ✓ (2026-07-09) + +`src/openamp_foundry/adoption/scorecard.py` with `SCORECARD_DIMENSIONS` +(5 weighted dimensions summing to 1.0: integration_check 0.25, +license_compliance 0.20, adapter_validation 0.20, schema_compatibility 0.20, +contribution_readiness 0.15), `ADOPTION_TIERS` (4 tiers: not_ready 0.0-0.40, +emerging 0.40-0.65, established 0.65-0.85, mature 0.85-1.01), +`DimensionScore` dataclass (8 fields: dimension, weight, raw_score, +weighted_score, passed_checks, total_checks, notes, dry_lab_only=True), +`AdoptionScorecard` dataclass (6 fields: total_score, adoption_tier, +dimensions, summary, recommendations, dry_lab_only=True), +`build_scorecard()` (aggregates dimension inputs into weighted total score +with tier classification and actionable recommendations), +`compute_adoption_tier()` (maps float score 0.0-1.0 to tier string). + +CLI (`openamp-foundry adoption-scorecard`) with `--scores-json` and +`--format text|json`. Handler `_run_adoption_scorecard` in reports.py. + +`make adoption-scorecard` target. 17 tests. **3446 total.** + +**Phase I (Interoperability and Adoption) is now fully complete** — all 10 +items I1–I10 implemented (artifact versioning, candidate manifest, benchmark +card, artifact changelog, integration checker, adapter validator, data license +checker, schema compatibility, contribution intake, adoption scorecard). + +Changes: +- `src/openamp_foundry/adoption/scorecard.py` (I10) — Core module with + `SCORECARD_DIMENSIONS` (5 weighted dimensions summing to 1.0), + `ADOPTION_TIERS` (4 tiers), `DimensionScore` (8 fields, dry_lab_only=True + default), `AdoptionScorecard` (6 fields, dry_lab_only=True default), + `build_scorecard()` (aggregates dimension inputs into weighted total score, + tier, and per-dimension scores with recommendations), + `compute_adoption_tier()` (maps float 0.0-1.0 to tier string). +- `tests/adoption/__init__.py` (I10) — Empty package init. +- `tests/adoption/test_scorecard.py` (I10) — 17 tests covering: perfect scores + → mature, all zeros → not_ready, score 0.50 → emerging, score 0.75 → + established, weights sum to 1.0, SCORECARD_DIMENSIONS has 5 entries, + ADOPTION_TIERS has 4 entries, DimensionScore dry_lab_only=True default, + AdoptionScorecard dry_lab_only=True default, recommendations empty when + all pass, recommendations populated on failures, weighted sum correctness, + missing dimension defaults to 0/0, compute_adoption_tier boundaries, + build_scorecard returns correct fields. +- `src/openamp_foundry/cli/main.py` (I10) — Registered `adoption-scorecard` + subcommand with `--scores-json` (required), `--format` flags. Added import + and dispatch. +- `src/openamp_foundry/cli/commands/reports.py` (I10) — Added + `_run_adoption_scorecard()` CLI handler with JSON parsing, `build_scorecard` + call, text and JSON output. +- `Makefile` (I10) — Added `adoption-scorecard` target with demo invocation + using all 5 dimensions at perfect scores. Added to `.PHONY`. +- `docs/evidence/METRICS_CURRENT.md` (I10) — v0.6.8 I10 changelog. Pipeline + version: v0.6.8. Test count: 3446. Phase I complete note. +- `tests/test_test_count_regression.py` — baseline updated to 3446. + +Honest boundaries: +- Scorecard measures adoption readiness only — whether the checks pass or fail. + It does not measure actual downstream adoption, user satisfaction, or + biological validity. +- Dimension weights are heuristic and may need adjustment as adoption data + accumulates. The current weights (integration 0.25, license 0.20, adapter + 0.20, schema 0.20, contribution 0.15) are the initial guess. +- `dry_lab_only: true` is a const field on all dataclasses — the scorecard is + a computational governance tool, not a biological assessment. +- The scorecard aggregates structural signals from other Phase I modules. If + any of those modules have blind spots, the aggregation inherits them. +- Perfect scores mean all structural checks pass, not that real-world adoption + is happening. Adoption is a social outcome, not a pipeline metric. + ## v0.6.7 — Loop 107: Phase I I9 — Public-Good Contribution Guide ✓ (2026-07-09) `docs/community/PUBLIC_GOOD_CONTRIBUTION_GUIDE.md` with 6 contribution types diff --git a/src/openamp_foundry/adoption/scorecard.py b/src/openamp_foundry/adoption/scorecard.py new file mode 100644 index 00000000..a3cb2aed --- /dev/null +++ b/src/openamp_foundry/adoption/scorecard.py @@ -0,0 +1,101 @@ +"""Adoption scorecard — aggregates Phase I adoption signals into a dashboard. + +Measures real adoption readiness, not vanity metrics. +Dry-lab only. +""" +from __future__ import annotations +from dataclasses import dataclass +from typing import Any + +# Scorecard dimensions and their weights (must sum to 1.0) +SCORECARD_DIMENSIONS: dict[str, float] = { + "integration_check": 0.25, + "license_compliance": 0.20, + "adapter_validation": 0.20, + "schema_compatibility": 0.20, + "contribution_readiness": 0.15, +} + +ADOPTION_TIERS: dict[str, tuple[float, float]] = { + "not_ready": (0.0, 0.40), + "emerging": (0.40, 0.65), + "established": (0.65, 0.85), + "mature": (0.85, 1.01), +} + + +@dataclass +class DimensionScore: + dimension: str + weight: float + raw_score: float + weighted_score: float + passed_checks: int + total_checks: int + notes: str = "" + dry_lab_only: bool = True + + +@dataclass +class AdoptionScorecard: + total_score: float + adoption_tier: str + dimensions: list[DimensionScore] + summary: str + recommendations: list[str] + dry_lab_only: bool = True + + +def compute_adoption_tier(score: float) -> str: + for tier, (low, high) in ADOPTION_TIERS.items(): + if low <= score < high: + return tier + return "not_ready" + + +def build_scorecard(dimension_inputs: dict[str, dict[str, Any]]) -> AdoptionScorecard: + dimensions: list[DimensionScore] = [] + total_weighted = 0.0 + recommendations: list[str] = [] + + for dim_name, weight in SCORECARD_DIMENSIONS.items(): + inp = dimension_inputs.get(dim_name, {}) + passed = inp.get("passed_checks", 0) + total = inp.get("total_checks", 0) + notes = inp.get("notes", "") + raw = passed / total if total > 0 else 0.0 + weighted = raw * weight + total_weighted += weighted + dimensions.append(DimensionScore( + dimension=dim_name, + weight=weight, + raw_score=raw, + weighted_score=weighted, + passed_checks=passed, + total_checks=total, + notes=notes, + dry_lab_only=True, + )) + if raw < 1.0 and total > 0: + failed = total - passed + recommendations.append( + f"Fix {failed} failing check(s) in {dim_name} (current: {passed}/{total})" + ) + + tier = compute_adoption_tier(total_weighted) + tier_descriptions = { + "not_ready": "Pipeline does not yet meet minimum adoption standards.", + "emerging": "Pipeline is emerging but has significant gaps.", + "established": "Pipeline meets most adoption standards.", + "mature": "Pipeline is fully adoption-ready.", + } + summary = f"Adoption score: {total_weighted:.2f} ({tier}). {tier_descriptions[tier]}" + + return AdoptionScorecard( + total_score=round(total_weighted, 4), + adoption_tier=tier, + dimensions=dimensions, + summary=summary, + recommendations=recommendations, + dry_lab_only=True, + ) diff --git a/src/openamp_foundry/cli/commands/reports.py b/src/openamp_foundry/cli/commands/reports.py index c1696f06..d485a378 100644 --- a/src/openamp_foundry/cli/commands/reports.py +++ b/src/openamp_foundry/cli/commands/reports.py @@ -2387,3 +2387,65 @@ def _run_contribution_check(args: argparse.Namespace) -> int: print("safeguards, not biological proof.") return 0 if result.passed else 3 + + +def _run_adoption_scorecard(args: argparse.Namespace) -> int: + """Build an adoption scorecard from dimension inputs.""" + from openamp_foundry.adoption.scorecard import build_scorecard + + try: + dimension_inputs = json.loads(args.scores_json) + except (json.JSONDecodeError, TypeError) as e: + print(json.dumps({"status": "error", "error": f"Invalid --scores-json: {e}"})) + return 2 + + if not isinstance(dimension_inputs, dict): + print(json.dumps({"status": "error", "error": "--scores-json must be a JSON object"})) + return 2 + + card = build_scorecard(dimension_inputs) + output_format = getattr(args, "format", "text") + + if output_format == "json": + print(json.dumps({ + "total_score": card.total_score, + "adoption_tier": card.adoption_tier, + "dimensions": [ + { + "dimension": d.dimension, + "weight": d.weight, + "raw_score": d.raw_score, + "weighted_score": d.weighted_score, + "passed_checks": d.passed_checks, + "total_checks": d.total_checks, + "notes": d.notes, + "dry_lab_only": d.dry_lab_only, + } + for d in card.dimensions + ], + "summary": card.summary, + "recommendations": card.recommendations, + "dry_lab_only": card.dry_lab_only, + }, indent=2)) + else: + print("Adoption Scorecard") + print("=" * 60) + print(f"Total score: {card.total_score:.4f}") + print(f"Adoption tier: {card.adoption_tier}") + print() + print("Dimensions:") + for d in card.dimensions: + status = "PASS" if d.raw_score >= 1.0 else f"FAIL ({d.passed_checks}/{d.total_checks})" + print(f" {d.dimension:30s} {d.weight:.2f}w raw={d.raw_score:.2f} wtd={d.weighted_score:.3f} {status}") + print() + print(f"Summary: {card.summary}") + if card.recommendations: + print() + print("Recommendations:") + for r in card.recommendations: + print(f" - {r}") + print() + print("Dry-lab only. Scorecard measures adoption readiness,") + print("not biological validity.") + + return 0 diff --git a/src/openamp_foundry/cli/main.py b/src/openamp_foundry/cli/main.py index 2ec2b930..3137b060 100644 --- a/src/openamp_foundry/cli/main.py +++ b/src/openamp_foundry/cli/main.py @@ -5,7 +5,7 @@ from openamp_foundry.cli.commands.selection import _run_pilot_panel, _run_pilot_confident, _run_diversity_check, _run_select_batch, _run_batch_rationale from openamp_foundry.cli.commands.external import _run_external_predict, _run_external_consensus from openamp_foundry.cli.commands.qc import _run_synthesis_order, _run_presynth_qc -from openamp_foundry.cli.commands.reports import _run_reviewer_questionnaire, _run_ip_report, _run_batch_pack, _run_gold_standard, _run_novelty_check_broad, _run_lab_result_report, _run_calibration_intake, _run_recalibration_gate, _run_recalibration_engine, _run_validate_policy_version, _run_calibration_audit, _run_calibration_overfit_check, _run_result_quality_filter, _run_synthetic_result_policy_check, _run_calibration_decision_checklist, _run_calibration_rollback_plan, _run_simulation_registry, _run_validate_simulation_result, _run_simulation_baseline_check, _run_adapter_gate_check, _run_simulation_provenance, _run_simulation_ensemble_check, _run_simulation_ci_report, _run_simulation_deprecation_check, _run_simulation_scope_check, _run_simulation_evidence_packet, _run_artifact_version, _run_candidate_manifest, _run_benchmark_card, _run_artifact_changelog, _run_integration_check, _run_adapter_check, _run_license_check, _run_artifact_compat_check +from openamp_foundry.cli.commands.reports import _run_reviewer_questionnaire, _run_ip_report, _run_batch_pack, _run_gold_standard, _run_novelty_check_broad, _run_lab_result_report, _run_calibration_intake, _run_recalibration_gate, _run_recalibration_engine, _run_validate_policy_version, _run_calibration_audit, _run_calibration_overfit_check, _run_result_quality_filter, _run_synthetic_result_policy_check, _run_calibration_decision_checklist, _run_calibration_rollback_plan, _run_simulation_registry, _run_validate_simulation_result, _run_simulation_baseline_check, _run_adapter_gate_check, _run_simulation_provenance, _run_simulation_ensemble_check, _run_simulation_ci_report, _run_simulation_deprecation_check, _run_simulation_scope_check, _run_simulation_evidence_packet, _run_artifact_version, _run_candidate_manifest, _run_benchmark_card, _run_artifact_changelog, _run_integration_check, _run_adapter_check, _run_license_check, _run_artifact_compat_check, _run_adoption_scorecard from openamp_foundry.cli.commands.gates import _run_gate_check from openamp_foundry.cli.commands.reports import _run_contribution_check @@ -1793,6 +1793,26 @@ def build_parser() -> argparse.ArgumentParser: help="Output format (default: text).", ) + # ── Adoption scorecard (Phase I I10) ───────────────────────────── + asc = sub.add_parser( + "adoption-scorecard", + help=( + "Build an adoption scorecard aggregating Phase I adoption signals " + "(integration_check, license_compliance, adapter_validation, " + "schema_compatibility, contribution_readiness) into a dashboard. " + "Dry-lab only." + ), + ) + asc.add_argument( + "--scores-json", type=str, required=True, + help="JSON dict of dimension inputs (required).", + ) + asc.add_argument( + "--format", type=str, default="text", + choices=["text", "json"], + help="Output format (default: text).", + ) + # ── Simulation-result confidence interval report (H7) ──────────── sicr = sub.add_parser( "simulation-ci-report", @@ -2028,6 +2048,9 @@ def main(argv: list[str] | None = None) -> int: if args.command == "artifact-compat-check": return _run_artifact_compat_check(args) + if args.command == "adoption-scorecard": + return _run_adoption_scorecard(args) + parser.error("unknown command") return 2 diff --git a/tests/adoption/test_scorecard.py b/tests/adoption/test_scorecard.py new file mode 100644 index 00000000..3b0519b4 --- /dev/null +++ b/tests/adoption/test_scorecard.py @@ -0,0 +1,167 @@ +"""Tests for adoption scorecard module.""" +from __future__ import annotations + +from openamp_foundry.adoption.scorecard import ( + SCORECARD_DIMENSIONS, + ADOPTION_TIERS, + DimensionScore, + AdoptionScorecard, + compute_adoption_tier, + build_scorecard, +) + + +class TestScorecard: + def test_perfect_scores_mature(self): + inputs = { + "integration_check": {"passed_checks": 5, "total_checks": 5}, + "license_compliance": {"passed_checks": 3, "total_checks": 3}, + "adapter_validation": {"passed_checks": 2, "total_checks": 2}, + "schema_compatibility": {"passed_checks": 4, "total_checks": 4}, + "contribution_readiness": {"passed_checks": 1, "total_checks": 1}, + } + card = build_scorecard(inputs) + assert card.adoption_tier == "mature" + assert card.total_score == 1.0 + + def test_all_zeros_not_ready(self): + inputs = { + "integration_check": {"passed_checks": 0, "total_checks": 5}, + "license_compliance": {"passed_checks": 0, "total_checks": 3}, + "adapter_validation": {"passed_checks": 0, "total_checks": 2}, + "schema_compatibility": {"passed_checks": 0, "total_checks": 4}, + "contribution_readiness": {"passed_checks": 0, "total_checks": 1}, + } + card = build_scorecard(inputs) + assert card.adoption_tier == "not_ready" + assert card.total_score == 0.0 + + def test_score_050_emerging(self): + inputs = { + "integration_check": {"passed_checks": 2, "total_checks": 5}, + "license_compliance": {"passed_checks": 2, "total_checks": 5}, + "adapter_validation": {"passed_checks": 2, "total_checks": 5}, + "schema_compatibility": {"passed_checks": 2, "total_checks": 5}, + "contribution_readiness": {"passed_checks": 2, "total_checks": 5}, + } + card = build_scorecard(inputs) + assert card.adoption_tier == "emerging" + assert 0.40 <= card.total_score < 0.65 + + def test_score_075_established(self): + inputs = { + "integration_check": {"passed_checks": 4, "total_checks": 5}, + "license_compliance": {"passed_checks": 3, "total_checks": 4}, + "adapter_validation": {"passed_checks": 3, "total_checks": 4}, + "schema_compatibility": {"passed_checks": 3, "total_checks": 4}, + "contribution_readiness": {"passed_checks": 3, "total_checks": 4}, + } + card = build_scorecard(inputs) + assert card.adoption_tier == "established" + assert 0.65 <= card.total_score < 0.85 + + def test_scorecard_dimensions_sum_to_one(self): + total = sum(SCORECARD_DIMENSIONS.values()) + assert abs(total - 1.0) < 1e-9 + + def test_scorecard_dimensions_has_5_entries(self): + assert len(SCORECARD_DIMENSIONS) == 5 + + def test_adoption_tiers_has_4_entries(self): + assert len(ADOPTION_TIERS) == 4 + + def test_dimensionscore_dry_lab_only_default(self): + ds = DimensionScore( + dimension="test", weight=0.5, raw_score=0.8, + weighted_score=0.4, passed_checks=4, total_checks=5, + ) + assert ds.dry_lab_only is True + + def test_adoption_scorecard_dry_lab_only_default(self): + card = AdoptionScorecard( + total_score=0.0, adoption_tier="not_ready", + dimensions=[], summary="", recommendations=[], + ) + assert card.dry_lab_only is True + + def test_recommendations_empty_when_all_pass(self): + inputs = { + "integration_check": {"passed_checks": 5, "total_checks": 5}, + "license_compliance": {"passed_checks": 3, "total_checks": 3}, + "adapter_validation": {"passed_checks": 2, "total_checks": 2}, + "schema_compatibility": {"passed_checks": 4, "total_checks": 4}, + "contribution_readiness": {"passed_checks": 1, "total_checks": 1}, + } + card = build_scorecard(inputs) + assert card.recommendations == [] + + def test_recommendations_populated_on_failures(self): + inputs = { + "integration_check": {"passed_checks": 3, "total_checks": 5}, + "license_compliance": {"passed_checks": 3, "total_checks": 3}, + "adapter_validation": {"passed_checks": 2, "total_checks": 2}, + "schema_compatibility": {"passed_checks": 4, "total_checks": 4}, + "contribution_readiness": {"passed_checks": 1, "total_checks": 1}, + } + card = build_scorecard(inputs) + assert len(card.recommendations) == 1 + assert "integration_check" in card.recommendations[0] + assert "2" in card.recommendations[0] + + def test_total_score_weighted_sum(self): + inputs = { + "integration_check": {"passed_checks": 5, "total_checks": 5}, + "license_compliance": {"passed_checks": 2, "total_checks": 4}, + "adapter_validation": {"passed_checks": 2, "total_checks": 2}, + "schema_compatibility": {"passed_checks": 4, "total_checks": 4}, + "contribution_readiness": {"passed_checks": 1, "total_checks": 1}, + } + card = build_scorecard(inputs) + expected = ( + 1.0 * 0.25 + + 0.5 * 0.20 + + 1.0 * 0.20 + + 1.0 * 0.20 + + 1.0 * 0.15 + ) + assert abs(card.total_score - expected) < 1e-4 + + def test_missing_dimension_defaults_to_zero(self): + inputs = {} + card = build_scorecard(inputs) + assert card.total_score == 0.0 + for dim in card.dimensions: + assert dim.raw_score == 0.0 + assert dim.total_checks == 0 + + def test_compute_tier_zero(self): + assert compute_adoption_tier(0.0) == "not_ready" + + def test_compute_tier_one(self): + assert compute_adoption_tier(1.0) == "mature" + + def test_compute_tier_boundaries(self): + assert compute_adoption_tier(0.39) == "not_ready" + assert compute_adoption_tier(0.40) == "emerging" + assert compute_adoption_tier(0.64) == "emerging" + assert compute_adoption_tier(0.65) == "established" + assert compute_adoption_tier(0.84) == "established" + assert compute_adoption_tier(0.85) == "mature" + + def test_build_scorecard_returns_correct_fields(self): + inputs = { + "integration_check": {"passed_checks": 5, "total_checks": 5}, + "license_compliance": {"passed_checks": 3, "total_checks": 3}, + "adapter_validation": {"passed_checks": 2, "total_checks": 2}, + "schema_compatibility": {"passed_checks": 4, "total_checks": 4}, + "contribution_readiness": {"passed_checks": 1, "total_checks": 1}, + } + card = build_scorecard(inputs) + assert isinstance(card, AdoptionScorecard) + assert isinstance(card.total_score, float) + assert isinstance(card.adoption_tier, str) + assert isinstance(card.dimensions, list) + assert len(card.dimensions) == 5 + assert isinstance(card.summary, str) + assert isinstance(card.recommendations, list) + assert card.dry_lab_only is True diff --git a/tests/test_test_count_regression.py b/tests/test_test_count_regression.py index d58afb77..54ff1a78 100644 --- a/tests/test_test_count_regression.py +++ b/tests/test_test_count_regression.py @@ -2,11 +2,11 @@ import subprocess import sys -# Baseline: 3429 tests as of Loop 107 (2026-07-09) -- Phase I I9 (public-good contribution guide) +# Baseline: 3446 tests as of Loop 108 (2026-07-09) -- Phase I I10 (adoption scorecard, Phase I complete) # Note: 3 pre-existing collection errors in test_benchmark_calibration.py, # test_benchmark_charge_distribution.py, test_benchmark_cheap_enemies.py. # Raise threshold if legitimate tests are removed; lower threshold is okay. -BASELINE = 3429 +BASELINE = 3446 TOLERANCE = 0.05 # allow 5% fluctuation