Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance simulation-ensemble-check simulation-ci-report simulation-deprecation-check simulation-scope-check simulation-evidence-packet artifact-version candidate-manifest artifact-changelog integration-check adapter-author-check
.PHONY: help demo test lint ci clean bench-leakage bench-multi-negatives bench-baseline bench-hidden-active bench-cluster-split bench-expert-ablation bench-expert-ablation-500 bench-selectivity bench-feature-decomp bench-gate bench-easy-baseline bench-charge-matched bench-order-dependent bench-precision-at-k bench-per-family bench-simulation-ablation bench-simulation-ablation-within-amp simulation-gate bench-strategy-compare regenerate-all generate phase3 pilot validate-scoring validate-scoring-phase3 validate-scoring-strict external-pilot pilot-confident presynth-qc gold-standard diversity synthesis-order novelty-broad external-consensus questionnaire gate-check ip-report benchmark-card wave0-5-gate-check wave0-5-novelty-audit wave0-5-novelty-audit-v2 wave0-5-panel wave0-5-evidence wave0-5-fill-external wave0-5b-generate wave0-5b-filter recalibration-engine recalibration-engine-dry-run validate-policy-version generate-synthetic-lab-results bump-policy-version generate-review-packet failed-candidate-report safe-publication-filter classify-negative-informativeness validate-rejection-events negative-result-dashboard calibration-audit calibration-audit-example calibration-overfit-check result-quality-filter synthetic-result-policy-check calibration-decision-checklist calibration-rollback-plan simulation-registry validate-simulation-result-schema simulation-baseline-check adapter-gate-check simulation-provenance simulation-ensemble-check simulation-ci-report simulation-deprecation-check simulation-scope-check simulation-evidence-packet artifact-version candidate-manifest artifact-changelog integration-check adapter-author-check license-check

PYTHON := $(shell [ -f .venv/bin/python ] && echo .venv/bin/python || echo python3)
PYTEST := $(shell [ -f .venv/bin/pytest ] && echo .venv/bin/pytest || echo pytest)
Expand Down Expand Up @@ -815,3 +815,8 @@ adapter-author-check:
.venv/bin/openamp-foundry adapter-check \
--adapter-json '{"adapter_id":"example-adapter","adapter_version":"1.0.0","mode":"info","output_status":"ok","score_fields":{"activity":0.72},"uncertainty":0.15,"warnings":[],"failure_reason":null,"release_status":"internal","ranking_effect":"none","has_baseline_comparison":false,"makes_network_calls":false,"network_call_documented":false,"dry_lab_only":true}'
@echo "Adapter author check complete."

license-check:
.venv/bin/openamp-foundry license-check \
--source-json '{"source_id":"apd-v2","source_name":"Antimicrobial Peptide Database v2","license_id":"CC-BY-4.0","use_context":"training","attribution_required":true,"commercial_use_allowed":true,"redistribution_allowed":true,"modifications_allowed":true,"human_review_required":false,"notes":"","dry_lab_only":true}'
@echo "License check complete."
8 changes: 4 additions & 4 deletions docs/evidence/METRICS_CURRENT.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak
> **Purpose:** One authoritative table of current pipeline metrics. If any doc disagrees
> with this file, this file wins. Updated whenever benchmark/benchmark config changes.
>
> **Last updated:** 2026-07-09 (Phase I I6 — adapter author validator — v0.6.4)
> **New in v0.6.4:** Adapter author validator (I6 — Loop 104) — `src/openamp_foundry/adapters/adapter_validator.py` with `AdapterDeclaration` dataclass (14 fields: adapter_id, adapter_version, mode, output_status, score_fields, uncertainty, warnings, failure_reason, release_status, ranking_effect, has_baseline_comparison, makes_network_calls, network_call_documented, dry_lab_only), `AdapterValidationResult` dataclass (5 fields: adapter_id, passed, errors, warnings_list, dry_lab_only), `validate_adapter_declaration()` (10 checks: non-empty adapter_id, non-empty adapter_version, valid mode in VALID_ADAPTER_MODES, valid output_status in VALID_OUTPUT_STATUSES, valid ranking_effect in VALID_RANKING_EFFECTS, valid release_status in VALID_RELEASE_STATUSES, dry_lab_only must be True, ranking_effect in proposed/active requires has_baseline_comparison=True, network calls require network_call_documented=True, uncertainty in 0.0-1.0 or None), `validate_adapter_dict()` for raw dict input with missing-fields guard. `VALID_ADAPTER_MODES` (4), `VALID_OUTPUT_STATUSES` (4), `VALID_RANKING_EFFECTS` (3), `VALID_RELEASE_STATUSES` (5), `REQUIRED_OUTPUT_CONTRACT_FIELDS` (10). All results carry dry_lab_only=True. CLI (`openamp-foundry adapter-check`) with `--adapter-json`, `--format text|json`. `make adapter-author-check` target. 31 tests. **3387 total.** External adapter authors can now validate their declarations against the ADAPTER_AUTHOR_GUIDE contract before submitting.
> **Last updated:** 2026-07-09 (Phase I I7 — data license checker — v0.6.5)
> **New in v0.6.5:** Data license checker (I7 — Loop 105) — `src/openamp_foundry/licensing/license_checker.py` with `DataLicenseDeclaration` dataclass (11 fields: source_id, source_name, license_id, use_context, attribution_required, commercial_use_allowed, redistribution_allowed, modifications_allowed, human_review_required, notes, dry_lab_only), `LicenseCheckResult` dataclass (8 fields: source_id, license_id, use_context, passed, status, errors, warnings, dry_lab_only), `check_data_license()` (validates declarations against `APPROVED_LICENSES` (11 entries: CC0-1.0, CC-BY-4.0, CC-BY-SA-4.0, MIT, Apache-2.0, GPL-3.0, LGPL-2.1, BSD-2-Clause, BSD-3-Clause, ODbL-1.0, PDDL-1.0), `RESTRICTED_LICENSES` (4 entries: CC-BY-NC-4.0, CC-BY-NC-SA-4.0, custom, proprietary), `BLOCKED_LICENSES` (3 entries: unknown, unlicensed, all-rights-reserved), `VALID_USE_CONTEXTS` (6 entries: training, scoring, benchmarking, reporting, publication, internal). Blocked licenses fail immediately; restricted require human_review_required=True; unknown licenses require governance review before use. `check_license_batch()` summarizes total/passed/failed/blocked/any_blocked/all_passed with dry_lab_only=True. All results carry dry_lab_only=True. CLI (`openamp-foundry license-check`) with `--source-json`, `--format text|json`. `make license-check` target. 16 tests. **3403 total.** External data sources used in pipeline outputs now require explicit license declarations, preventing hidden legal risk.
> **New in v0.6.3:** Downstream project template (I5 — Loop 103) — `docs/adoption/DOWNSTREAM_PROJECT_TEMPLATE.md` with overview of OpenAMP artifacts, minimum viable integration (consume candidate manifest, validate against schema, use Python library), schema validation step, evidence level interpretation table (L1-L6), safety flag conventions table, benchmark card consumption guide, explicit dry-lab limitations section, and contact/contribution section. `src/openamp_foundry/adoption/integration_checker.py` with `REQUIRED_INTEGRATION_CHECKS` (5 checks: manifest_schema_valid, evidence_level_in_range, dry_lab_only_acknowledged, safety_flags_reviewed, baseline_comparison_present), `IntegrationCheckResult` dataclass (4 fields: check_name, passed, detail, dry_lab_only), `run_integration_checks()` (returns dict with checks, passed_count, failed_count, all_passed, dry_lab_only). Exported from `src/openamp_foundry/adoption/__init__.py`. CLI (`openamp-foundry integration-check`) with `--manifest-json`, `--format text|json`. `make integration-check` target. 14 tests. **3356 total.** External researchers who want to use OpenAMP artifacts now have a guide and checker to validate their integration.
> **New in v0.6.2:** Evidence-certificate changelog (I4 — Loop 102) — `docs/engineering/ARTIFACT_CHANGELOG.md` with structured changelog format (version, date, artifact_name, change_type, description, breaking flag). Unreleased section at top. `src/openamp_foundry/versioning/artifact_changelog.py` with `CHANGE_TYPES` set (6 values), `ChangelogEntry` dataclass (7 fields), `ARTIFACT_CHANGELOG` list (6 entries: candidate_manifest, benchmark_card, simulation_result, simulation_module_registry, artifact_versioning_policy, artifact_changelog — all "added", non-breaking, v1.0.0), `get_changelog_entries()` (filters by artifact_name, version, change_type, breaking_only), `validate_changelog()` (5 checks: version MAJOR.MINOR.PATCH, date YYYY-MM-DD with dashes, artifact_name non-empty, change_type in CHANGE_TYPES, description non-empty), `changelog_summary()` (total, by_change_type, breaking_changes, artifacts_covered sorted, dry_lab_only). CLI (`openamp-foundry artifact-changelog`) with `--artifact`, `--version`, `--change-type`, `--breaking-only`, `--format text|json`. `make artifact-changelog` target. 13 tests. **3342 total.** External tools that consume OpenAMP artifacts now have a machine-readable changelog to detect breaking changes and adapt consumers.
> **New in v0.6.1:** Benchmark card schema (I3 — Loop 101) — `schemas/benchmark_card.schema.json` (Draft 2020-12, 15 required fields: benchmark_id, benchmark_name, version, date, metric, metric_value, baseline_name, baseline_value, delta, beats_baseline, dataset, dataset_size, scope, caveats, dry_lab_only). `$schema`, `$id`, `title`, `additionalProperties: false`. `src/openamp_foundry/benchmarks/` module with `BenchmarkCard` dataclass (15 fields), `make_benchmark_card()` (auto-computes delta, beats_baseline), `validate_benchmark_card()` (10 checks: non-empty benchmark_id, non-empty benchmark_name, non-empty metric, non-empty dataset, non-empty baseline_name, dataset_size >= 1, delta matches metric_value - baseline_value within 1e-9, beats_baseline matches delta > 0, dry_lab_only must be True), `benchmark_card_summary()` (total, beats_baseline_count, fails_baseline_count, dry_lab_only). CLI (`openamp-foundry benchmark-card`) with `--benchmark-id`, `--benchmark-name`, `--metric`, `--metric-value`, `--baseline-name`, `--baseline-value`, `--dataset`, `--dataset-size`, `--validate`, `--format text|json`. `make benchmark-card` target. 19 tests. **3329 total.** A benchmark card is the standard format for describing external benchmark results — what was benchmarked, what the baseline was, what the result was, and what claims are supported.
Expand Down Expand Up @@ -94,7 +94,7 @@ Machine-readable snapshot: `outputs/metrics_snapshot.json` regenerated with `mak
> **New in v0.5.31:** Added dipeptide-order features for sequence-order awareness. `dipeptide_order_score` achieves AUROC 0.7861 on AMP-vs-scrambled discrimination — the strongest order-dependent feature in the pipeline. Only 7/31 features survive scrambling (amphipathicity/helix-wheel + dipeptide). All composition features are purely position-independent (exactly 0.5000 AUROC on scrambled test).
> **New in v0.5.30:** Easy baseline benchmark added — charge density alone (AUROC 0.8166) outperforms the full pipeline ensemble (0.7792) on AMP-vs-Swiss-Prot-decoy discrimination. Honest finding documented: expected because pipeline optimizes for safety, not raw discrimination.
> **New in v0.5.29:** Expanded benchmark to 500 AMPs + 500 composition-matched decoys (n=1000). AUROC 0.7792 (CI₉₅: 0.7505–0.8065) confirms signal generalizes. Cluster-aware CI: 0.746–0.8102. Representative AUROC: 0.778. Standard benchmark (n=191) retained for backward comparison.
> **Pipeline version:** v0.6.4 (I6 — adapter author validator)
> **Pipeline version:** v0.6.5 (I7 — data license checker)
> **Branch:** main

---
Expand Down Expand Up @@ -1164,7 +1164,7 @@ Decoys score low on activity. Selective AMPs score moderately on both.

| Metric | Value |
|--------|-------|
| Total tests | 3387 |
| Total tests | 3393 |
| Coverage (branch) | 99% (6 CLI guard lines only) |
| Source modules at 100% | All pipeline, QC, scoring, adapter modules |

Expand Down
65 changes: 65 additions & 0 deletions docs/research/ROADMAP.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,70 @@
# Roadmap

## v0.6.5 — Loop 105: Phase I I7 — Data License Checker ✓ (2026-07-09)

`src/openamp_foundry/licensing/license_checker.py` with `DataLicenseDeclaration`
dataclass (11 fields: source_id, source_name, license_id, use_context,
attribution_required, commercial_use_allowed, redistribution_allowed,
modifications_allowed, human_review_required, notes, dry_lab_only).
`LicenseCheckResult` dataclass (8 fields: source_id, license_id, use_context,
passed, status, errors, warnings, dry_lab_only). `check_data_license()` validates
declarations against three license sets: `APPROVED_LICENSES` (11 entries: CC0-1.0,
CC-BY-4.0, CC-BY-SA-4.0, MIT, Apache-2.0, GPL-3.0, LGPL-2.1, BSD-2-Clause,
BSD-3-Clause, ODbL-1.0, PDDL-1.0), `RESTRICTED_LICENSES` (4 entries: CC-BY-NC-4.0,
CC-BY-NC-SA-4.0, custom, proprietary), `BLOCKED_LICENSES` (3 entries: unknown,
unlicensed, all-rights-reserved). Blocked licenses fail immediately; restricted
require human_review_required=True; unknown licenses require governance review.
`check_license_batch()` summarizes total/passed/failed/blocked/any_blocked/
all_passed with dry_lab_only=True. `VALID_USE_CONTEXTS` (6 entries: training,
scoring, benchmarking, reporting, publication, internal).

CLI (`openamp-foundry license-check`) with `--source-json` (required) and
`--format text|json`. Handler `_run_license_check` in reports.py.

`make license-check` target. 20 tests. **3393 total.** External data sources
used in pipeline outputs now require explicit license declarations, preventing
hidden legal risk. This is Loop 105 — the 105th PR in the NEXT_100_PR_MAP series.

Changes:
- `src/openamp_foundry/licensing/__init__.py` (I7) — Empty package init.
- `src/openamp_foundry/licensing/license_checker.py` (I7) — Core module with
`DataLicenseDeclaration` (11 fields), `LicenseCheckResult` (8 fields),
`check_data_license()` (validates against APPROVED_LICENSES 11 entries,
RESTRICTED_LICENSES 4, BLOCKED_LICENSES 3, VALID_USE_CONTEXTS 6),
`check_license_batch()` (summary with counts, any_blocked, all_passed,
dry_lab_only=True). All dataclasses carry dry_lab_only=True.
- `tests/licensing/__init__.py` (I7) — Empty package init.
- `tests/licensing/test_license_checker.py` (I7) — 20 tests covering: valid CC0
passes, valid MIT passes, blocked unknown/unlicensed/all-rights-reserved fail,
restricted CC-BY-NC-4.0 without human_review fails, restricted with human_review
passes, empty source_id/source_name/license_id fail, invalid use_context fails,
dry_lab_only=False fails, publication without redistribution fails, unknown
license gets status=unknown_license, batch counts correct, all results
dry_lab_only, constant counts.
- `src/openamp_foundry/cli/main.py` (I7) — Registered `license-check` subcommand
with `--source-json`, `--format` flags. Added import and dispatch.
- `src/openamp_foundry/cli/commands/reports.py` (I7) — Added `_run_license_check()`
CLI handler with JSON parsing, DataLicenseDeclaration creation, text and JSON
output, and exit code 3 on failure.
- `Makefile` (I7) — Added `license-check` target with demo invocation using
apd-v2 source and CC-BY-4.0 license. Added to `.PHONY`.
- `docs/evidence/METRICS_CURRENT.md` — v0.6.5 I7 changelog. Test count: 3393.
- `tests/test_test_count_regression.py` — baseline updated to 3393.

Honest boundaries:
- License validation checks declared license identifiers against known lists.
It does not verify that the data source actually uses the declared license,
that the license is legally enforceable in a given jurisdiction, or that the
data was lawfully obtained.
- Approved and blocked lists are policy-defined and may need updating as the
legal landscape evolves.
- `dry_lab_only: true` is a const field on all dataclasses — license checks
are inherently computational and must never be presented as legal advice.
- External adapter authors who produce data from new sources must declare their
licenses before those data can influence pipeline outputs.
- Restricted licenses require human_review_required=True — this is a structural
check that the flag is set, not verification that human review actually occurred.

## v0.6.4 — Loop 104: Phase I I6 — Adapter Author Validator ✓ (2026-07-09)

`src/openamp_foundry/adapters/adapter_validator.py` with `AdapterDeclaration`
Expand Down
56 changes: 56 additions & 0 deletions src/openamp_foundry/cli/commands/reports.py
Original file line number Diff line number Diff line change
Expand Up @@ -2247,3 +2247,59 @@ def _run_adapter_check(args: argparse.Namespace) -> int:
print("not biological proof.")

return 0 if result.passed else 3


def _run_license_check(args: argparse.Namespace) -> int:
"""Validate a data license declaration."""
from openamp_foundry.licensing.license_checker import (
DataLicenseDeclaration,
check_data_license,
)

try:
source_dict = json.loads(args.source_json)
except (json.JSONDecodeError, TypeError) as e:
print(json.dumps({"status": "error", "error": f"Invalid --source-json: {e}"}))
return 2

if not isinstance(source_dict, dict):
print(json.dumps({"status": "error", "error": "--source-json must be a JSON object"}))
return 2

decl = DataLicenseDeclaration(**source_dict)
result = check_data_license(decl)
output_format = getattr(args, "format", "text")

if output_format == "json":
print(json.dumps({
"source_id": result.source_id,
"license_id": result.license_id,
"use_context": result.use_context,
"passed": result.passed,
"status": result.status,
"errors": result.errors,
"warnings": result.warnings,
"dry_lab_only": result.dry_lab_only,
}, indent=2))
else:
print("License Check")
print("=" * 60)
print(f"Source ID: {result.source_id}")
print(f"License ID: {result.license_id}")
print(f"Use Context: {result.use_context}")
print(f"Status: {result.status.upper()}")
print(f"Result: {'PASS' if result.passed else 'FAIL'}")
print()
if result.errors:
print("Errors:")
for err in result.errors:
print(f" - {err}")
if result.warnings:
print("Warnings:")
for w in result.warnings:
print(f" - {w}")
print()
print("Dry-lab only. License checks are computational safeguards,")
print("not legal advice.")

return 0 if result.passed else 3
Loading
Loading