From 01073d941d2968e05800e00646e96c030db3e7b1 Mon Sep 17 00:00:00 2001 From: OpenCode Date: Wed, 8 Jul 2026 21:56:19 +0700 Subject: [PATCH] Add 10 data management files Closes #330, Closes #331, Closes #332, Closes #333, Closes #334, Closes #335, Closes #336, Closes #337, Closes #338, Closes #339 --- docs/evidence/GUIDE_CLEANUP_DRY_RUN.md | 29 +++++++++++++++ docs/evidence/GUIDE_SAFE_EXAMPLE_PACK.md | 22 ++++++++++++ docs/evidence/INDEX_ARCHIVED_ARTIFACTS.md | 18 ++++++++++ docs/evidence/REGISTRY_DATASET_SPLITS.md | 15 ++++++++ .../REPORT_STRATIFICATION_COVERAGE.md | 19 ++++++++++ docs/evidence/SCHEMA_DATA_SOURCE_CARD.md | 36 +++++++++++++++++++ tests/test_artifact_size_budget.py | 24 +++++++++++++ tests/test_data_source_card_validator.py | 35 ++++++++++++++++++ tests/test_leakage_sentinels.py | 22 ++++++++++++ tests/test_split_overlap.py | 33 +++++++++++++++++ 10 files changed, 253 insertions(+) create mode 100644 docs/evidence/GUIDE_CLEANUP_DRY_RUN.md create mode 100644 docs/evidence/GUIDE_SAFE_EXAMPLE_PACK.md create mode 100644 docs/evidence/INDEX_ARCHIVED_ARTIFACTS.md create mode 100644 docs/evidence/REGISTRY_DATASET_SPLITS.md create mode 100644 docs/evidence/REPORT_STRATIFICATION_COVERAGE.md create mode 100644 docs/evidence/SCHEMA_DATA_SOURCE_CARD.md create mode 100644 tests/test_artifact_size_budget.py create mode 100644 tests/test_data_source_card_validator.py create mode 100644 tests/test_leakage_sentinels.py create mode 100644 tests/test_split_overlap.py diff --git a/docs/evidence/GUIDE_CLEANUP_DRY_RUN.md b/docs/evidence/GUIDE_CLEANUP_DRY_RUN.md new file mode 100644 index 00000000..6ec74d62 --- /dev/null +++ b/docs/evidence/GUIDE_CLEANUP_DRY_RUN.md @@ -0,0 +1,29 @@ +# Generated Output Cleanup Dry-Run + +Preview which generated files would be cleaned up. + +## Usage +```bash +# Dry-run: list files that would be removed +ls outputs/*.json outputs/*.jsonl outputs/*.md 2>/dev/null + +# Actual cleanup +rm -rf outputs/*.json outputs/*.jsonl outputs/*.md +``` + +## What Would Be Removed +- `outputs/*.json` — benchmark results, manifests +- `outputs/*.jsonl` — ranked candidate lists +- `outputs/*.md` — reports +- `outputs/evidence/` — evidence certificates (regenerable) + +## What Would NOT Be Removed +- `outputs/.gitkeep` — keeps directory in git +- `outputs/lab_results/` — lab data (if present) + +## Regeneration +After cleanup, regenerate with: +```bash +make demo +make regenerate-all +``` diff --git a/docs/evidence/GUIDE_SAFE_EXAMPLE_PACK.md b/docs/evidence/GUIDE_SAFE_EXAMPLE_PACK.md new file mode 100644 index 00000000..13e365ae --- /dev/null +++ b/docs/evidence/GUIDE_SAFE_EXAMPLE_PACK.md @@ -0,0 +1,22 @@ +# Safe Example Artifact Pack + +Guidelines for creating example artifacts that are safe to distribute. + +## Rules +- Use only synthetic or public-domain data. +- Label all data as `example`, `demo`, or `synthetic`. +- No real candidate sequences from unpublished work. +- No lab results from real experiments. +- No PII (personally identifiable information). +- All example artifacts should pass `check_claims.py` and `check_doc_links.py`. + +## Current Examples +| Artifact | Type | Safety Check | +|----------|:----:|:------------:| +| demo_candidates.csv | synthetic | ✅ Synthetic, no real data | +| demo_known_amps.csv | public | ✅ Public AMP sequences | +| lab_results/ | synthetic | ✅ Labeled SYNTHETIC | + +## Related +- `docs/evidence/SYNTHETIC_DATA_POLICY.md` +- `docs/evidence/GUIDE_TOY_DATA_MARKERS.md` diff --git a/docs/evidence/INDEX_ARCHIVED_ARTIFACTS.md b/docs/evidence/INDEX_ARCHIVED_ARTIFACTS.md new file mode 100644 index 00000000..c2fc1867 --- /dev/null +++ b/docs/evidence/INDEX_ARCHIVED_ARTIFACTS.md @@ -0,0 +1,18 @@ +# Archived Artifact Index + +Index of archived artifacts and their locations. + +## Index Format +| Artifact | Archive Path | Date Archived | Superseded By | +|----------|-------------|:-------------:|:-------------:| +| — | — | — | — | + +## Rules +- When an artifact is archived, add it to this index. +- Include the date and reason for archiving. +- If superseded, link to the replacement artifact. +- Archived artifacts should be moved to `archive/` directories. +- The index should be checked before creating new artifacts to avoid duplication. + +## Current Status +No artifacts have been archived yet. This index will grow as the project matures. diff --git a/docs/evidence/REGISTRY_DATASET_SPLITS.md b/docs/evidence/REGISTRY_DATASET_SPLITS.md new file mode 100644 index 00000000..94911632 --- /dev/null +++ b/docs/evidence/REGISTRY_DATASET_SPLITS.md @@ -0,0 +1,15 @@ +# Dataset Split Registry + +Registry of dataset splits used in benchmarks and validation. + +| Split | Dataset | Method | Ratio | Seed | +|-------|---------|--------|:-----:|:----:| +| 500-AMP cluster split | known_amps_500.csv | Cluster at 70% similarity | 374/500 clusters | 20260705 | +| Standard benchmark | known_amps.csv + random_background.csv | Random | 95/96 | 1729 | +| Cross-dataset | DRAMP + Swiss-Prot decoys | Source-based | 500/500 | 20260705 | + +## Rules +- All splits must be registered here. +- Splits must be reproducible (fixed seed, documented method). +- Split methods must prevent leakage between train and test. +- If a split is updated, increment the version and document the change. diff --git a/docs/evidence/REPORT_STRATIFICATION_COVERAGE.md b/docs/evidence/REPORT_STRATIFICATION_COVERAGE.md new file mode 100644 index 00000000..32e22c16 --- /dev/null +++ b/docs/evidence/REPORT_STRATIFICATION_COVERAGE.md @@ -0,0 +1,19 @@ +# Stratification Coverage Report + +Reports how well benchmark stratification covers different AMP classes. + +## Current Coverage +| Class | Count in 500-AMP | Per-family AUROC | Coverage | +|-------|:----------------:|:----------------:|:--------:| +| highly_cationic | 73 | 0.958 | ✅ Good | +| moderately_cationic | 115 | 0.894 | ✅ Good | +| cysteine_rich | 153 | 0.723 | ✅ Adequate | +| low_charge | 118 | 0.693 | ✅ Adequate | +| short | 21 | 0.610 | ⚠️ Low count | +| proline_rich | 20 | 0.586 | ⚠️ Low count | + +## Recommendations +- Classes with < 30 sequences should be expanded. +- Proline-rich and short AMPs need more representatives. +- Consider adding synthetic variants for underrepresented classes. +- Stratification should be reviewed when new data is added. diff --git a/docs/evidence/SCHEMA_DATA_SOURCE_CARD.md b/docs/evidence/SCHEMA_DATA_SOURCE_CARD.md new file mode 100644 index 00000000..f422dc37 --- /dev/null +++ b/docs/evidence/SCHEMA_DATA_SOURCE_CARD.md @@ -0,0 +1,36 @@ +# Data Source Card Schema + +Standard schema for dataset documentation. + +## Required Fields +| Field | Type | Description | +|-------|------|-------------| +| name | string | Dataset name | +| source | string | Source URL or reference | +| license | string | SPDX license identifier | +| version | string | Dataset version or date | +| description | string | Description of the data | +| labeled | boolean | Whether labels are available | + +## Optional Fields +| Field | Type | Description | +|-------|------|-------------| +| size | integer | Number of records | +| format | string | File format (CSV, FASTA, JSON) | +| limitations | list | Known biases or limitations | +| citation | string | Preferred citation | + +## Example +```json +{ + "name": "known_amps_500", + "source": "UniProt reviewed + APD6 natural", + "license": "CC BY 4.0", + "version": "2026-07-01", + "description": "500 antimicrobial peptide sequences", + "labeled": true, + "size": 500, + "format": "CSV", + "limitations": ["Enriched for helical AMPs"] +} +``` diff --git a/tests/test_artifact_size_budget.py b/tests/test_artifact_size_budget.py new file mode 100644 index 00000000..58d0753b --- /dev/null +++ b/tests/test_artifact_size_budget.py @@ -0,0 +1,24 @@ +"""Test that artifact sizes are within budget.""" +import os +from pathlib import Path + + +def test_schema_files_under_budget(): + """Schema files should be under 50 KB each.""" + for f in Path("schemas").rglob("*.json"): + size = os.path.getsize(f) + assert size < 50000, f"{f} is {size} bytes (limit 50000)" + + +def test_docs_files_under_budget(): + """Documentation files should be under 200 KB each.""" + for f in Path("docs").rglob("*.md"): + size = os.path.getsize(f) + assert size < 200000, f"{f} is {size} bytes (limit 200000)" + + +def test_example_files_under_budget(): + """Example data files should be under 5 MB each.""" + for f in Path("examples").rglob("*.csv"): + size = os.path.getsize(f) + assert size < 5000000, f"{f} is {size} bytes (limit 5MB)" diff --git a/tests/test_data_source_card_validator.py b/tests/test_data_source_card_validator.py new file mode 100644 index 00000000..59f43380 --- /dev/null +++ b/tests/test_data_source_card_validator.py @@ -0,0 +1,35 @@ +"""Test data source card validation.""" +import json +from pathlib import Path + +REQUIRED = {"name", "source", "license", "version", "description", "labeled"} + + +def test_example_data_source_card(): + """Verify a valid data source card passes checks.""" + card = { + "name": "test_dataset", + "source": "test_source", + "license": "CC0", + "version": "2026-01-01", + "description": "Test dataset", + "labeled": True, + } + assert REQUIRED.issubset(card.keys()) + + +def test_data_card_has_required_fields(): + """Check the first few CSV files for basic metadata.""" + for csv_file in list(Path("examples").rglob("*.csv"))[:3]: + assert csv_file.exists() + # CSV files should have headers + lines = csv_file.read_text().strip().split("\n") + assert len(lines) >= 2, f"{csv_file} has no data rows" + + +def test_data_card_license_is_defined(): + """Example data should have a license.""" + license_file = Path("DATA_LICENSE_NOTICE.md") + assert license_file.exists() + text = license_file.read_text() + assert "CC0" in text or "CC BY" in text or "license" in text.lower() diff --git a/tests/test_leakage_sentinels.py b/tests/test_leakage_sentinels.py new file mode 100644 index 00000000..d8f3c728 --- /dev/null +++ b/tests/test_leakage_sentinels.py @@ -0,0 +1,22 @@ +"""Test leakage sentinel fixtures.""" +from pathlib import Path + + +def test_sentinel_amps_detected_in_references(): + """Known AMPs like magainin-2 should be detectable in reference sets. + This is expected — they are known sequences used as benchmarks.""" + known = "GIGKFLHSAKKFGKAFVGEIMNS" # magainin-2 + found = False + for csv_file in Path("examples").rglob("*.csv"): + text = csv_file.read_text() + if known in text: + found = True + break + assert found, "Magainin-2 should be findable in example data" + + +def test_sentinel_amps_are_valid(): + """Sentinel AMPs should be valid sequences.""" + for seq in ["GIGKFLHSAKKFGKAFVGEIMNS", "GIGAVLKVLTTGLPALISWIKRKRQQ"]: + for aa in seq: + assert aa in "ACDEFGHIKLMNPQRSTVWY", f"Invalid amino acid {aa} in {seq[:10]}..." diff --git a/tests/test_split_overlap.py b/tests/test_split_overlap.py new file mode 100644 index 00000000..cdabfaa7 --- /dev/null +++ b/tests/test_split_overlap.py @@ -0,0 +1,33 @@ +"""Test that dataset splits do not overlap.""" +from pathlib import Path + + +def test_amp_and_decoy_no_overlap(): + """AMP and decoy sets should not share sequences.""" + amps = set() + with open("examples/validation/known_amps_500.csv") as f: + for line in f.readlines()[1:]: + parts = line.strip().split(",") + if len(parts) >= 2: + amps.add(parts[1]) + + decoys = set() + with open("examples/validation/random_background_500.csv") as f: + for line in f.readlines()[1:]: + parts = line.strip().split(",") + if len(parts) >= 2: + decoys.add(parts[1]) + + overlap = amps & decoys + assert len(overlap) == 0, f"Found {len(overlap)} overlapping sequences between AMP and decoy sets" + + +def test_demo_candidates_have_unique_ids(): + """Demo candidates should have unique IDs.""" + ids = set() + with open("examples/sequences/demo_candidates.csv") as f: + for line in f.readlines()[1:]: + parts = line.strip().split(",") + if parts: + ids.add(parts[0]) + assert len(ids) >= 5, "Should have at least 5 unique demo candidate IDs"