From 492152c9794a51333b6b987a87acb5d2f0fdcae2 Mon Sep 17 00:00:00 2001 From: OpenCode Date: Wed, 8 Jul 2026 22:45:08 +0700 Subject: [PATCH] Add 10 benchmark infrastructure files Closes #273, Closes #276, Closes #277, Closes #278, Closes #279, Closes #280, Closes #281, Closes #282, Closes #283, Closes #284 --- .../GUIDE_ARTIFACT_MANIFEST_GENERATOR.md | 30 +++++++++++++++++++ docs/evidence/GUIDE_BENCHMARK_COMPARISON.md | 28 +++++++++++++++++ .../GUIDE_BENCHMARK_FIXTURE_MANIFEST.md | 26 ++++++++++++++++ docs/evidence/GUIDE_CI_HELPER.md | 22 ++++++++++++++ docs/evidence/REGISTRY_BENCHMARK_SLICES.md | 18 +++++++++++ docs/evidence/REGISTRY_CHEAP_BASELINES.md | 16 ++++++++++ docs/evidence/SCHEMA_FOLDER_MANIFEST.md | 29 ++++++++++++++++++ tests/test_benchmark_caveat_extraction.py | 17 +++++++++++ tests/test_benchmark_seed_lock.py | 28 +++++++++++++++++ tests/test_ranked_jsonl_validator.py | 27 +++++++++++++++++ 10 files changed, 241 insertions(+) create mode 100644 docs/evidence/GUIDE_ARTIFACT_MANIFEST_GENERATOR.md create mode 100644 docs/evidence/GUIDE_BENCHMARK_COMPARISON.md create mode 100644 docs/evidence/GUIDE_BENCHMARK_FIXTURE_MANIFEST.md create mode 100644 docs/evidence/GUIDE_CI_HELPER.md create mode 100644 docs/evidence/REGISTRY_BENCHMARK_SLICES.md create mode 100644 docs/evidence/REGISTRY_CHEAP_BASELINES.md create mode 100644 docs/evidence/SCHEMA_FOLDER_MANIFEST.md create mode 100644 tests/test_benchmark_caveat_extraction.py create mode 100644 tests/test_benchmark_seed_lock.py create mode 100644 tests/test_ranked_jsonl_validator.py diff --git a/docs/evidence/GUIDE_ARTIFACT_MANIFEST_GENERATOR.md b/docs/evidence/GUIDE_ARTIFACT_MANIFEST_GENERATOR.md new file mode 100644 index 00000000..af766af6 --- /dev/null +++ b/docs/evidence/GUIDE_ARTIFACT_MANIFEST_GENERATOR.md @@ -0,0 +1,30 @@ +# Artifact Manifest Generator for Command Outputs + +Generates manifests for command outputs. + +## Usage +```bash +# After running a command, generate its manifest +python -c " +import json, hashlib, datetime +from pathlib import Path + +outputs = list(Path('outputs').glob('*.jsonl')) + list(Path('outputs').glob('*.json')) +manifest = { + 'generated_at': datetime.datetime.now().isoformat(), + 'outputs': [str(f) for f in outputs], + 'hashes': {str(f): hashlib.sha256(f.read_bytes()).hexdigest() for f in outputs} +} +Path('outputs/manifest.json').write_text(json.dumps(manifest, indent=2)) +" +``` + +## Fields +- `generated_at` — ISO 8601 timestamp +- `outputs` — list of output file paths +- `hashes` — SHA-256 hash of each output file + +## Rules +- Manifests should be generated when batch processing is complete. +- Hashes enable verification that outputs haven't been modified. +- Manifests are stored alongside the outputs they describe. diff --git a/docs/evidence/GUIDE_BENCHMARK_COMPARISON.md b/docs/evidence/GUIDE_BENCHMARK_COMPARISON.md new file mode 100644 index 00000000..13b6c44d --- /dev/null +++ b/docs/evidence/GUIDE_BENCHMARK_COMPARISON.md @@ -0,0 +1,28 @@ +# Benchmark Result Comparison Helper + +Helper for comparing benchmark results across runs. + +## Usage +```python +# Compare two benchmark runs +delta = current_auroc - baseline_auroc +if delta > 0.02: + print("IMPROVEMENT") +elif delta < -0.02: + print("REGRESSION") +else: + print("NO_SIGNIFICANT_CHANGE") +``` + +## Thresholds +| Delta | Interpretation | Action | +|:-----:|:--------------|--------| +| > 0.02 | Improvement | Document and proceed | +| -0.02 to 0.02 | No significant change | No action needed | +| < -0.02 | Regression | Investigate and fix | + +## Rules +- Compare against the committed baseline, not the previous run. +- Report delta in benchmark outputs. +- If delta is negative, check for recent changes that could explain it. +- If delta is positive, verify the improvement is not due to data leakage. diff --git a/docs/evidence/GUIDE_BENCHMARK_FIXTURE_MANIFEST.md b/docs/evidence/GUIDE_BENCHMARK_FIXTURE_MANIFEST.md new file mode 100644 index 00000000..17fdf41c --- /dev/null +++ b/docs/evidence/GUIDE_BENCHMARK_FIXTURE_MANIFEST.md @@ -0,0 +1,26 @@ +# Benchmark Fixture Manifest Files + +Manifest files for benchmark fixtures. + +## Format +```json +{ + "benchmark": "bench-500", + "fixtures": { + "amps": "examples/validation/known_amps_500.csv", + "decoys": "examples/validation/random_background_500.csv", + "config": "configs/pipeline.yaml" + }, + "hashes": { + "amps": "sha256:...", + "decoys": "sha256:..." + }, + "generated_at": "2026-07-08T12:00:00Z" +} +``` + +## Rules +- Each benchmark should have a fixture manifest. +- Manifests include hashes of input files for reproducibility. +- If input files change, the manifest should be regenerated. +- Manifests are stored in `outputs/benchmark_fixtures/`. diff --git a/docs/evidence/GUIDE_CI_HELPER.md b/docs/evidence/GUIDE_CI_HELPER.md new file mode 100644 index 00000000..294b402a --- /dev/null +++ b/docs/evidence/GUIDE_CI_HELPER.md @@ -0,0 +1,22 @@ +# Metric Confidence Interval Helper + +Helper for computing and interpreting confidence intervals. + +## Usage +```python +from openamp_foundry.benchmark.retrospective import bootstrap_ci + +# Get bootstrap CI for AUROC +ci_lo, ci_hi = bootstrap_ci(scores_pos, scores_neg, n_bootstrap=2000) +``` + +## Interpretation +- 95% CI means the true value falls in this range ~95% of the time. +- Narrow CI = more precise estimate. +- If CI includes 0.5, the result is not statistically significant. + +## Rules +- All benchmark AUROCs should report a bootstrap CI. +- Use n_bootstrap=2000 for standard benchmarks. +- Cluster-aware bootstrap should be used when near-duplicates exist. +- CIs should be reported as `CI_95: [lower, upper]` in benchmark outputs. diff --git a/docs/evidence/REGISTRY_BENCHMARK_SLICES.md b/docs/evidence/REGISTRY_BENCHMARK_SLICES.md new file mode 100644 index 00000000..1a10dd9b --- /dev/null +++ b/docs/evidence/REGISTRY_BENCHMARK_SLICES.md @@ -0,0 +1,18 @@ +# Benchmark Slice Registry + +Registry of benchmark slices used in per-family analysis. + +| Slice | Criteria | Count | Description | +|-------|----------|:-----:|-------------| +| highly_cationic | charge >= 4.0 | 73 | High positive charge | +| moderately_cationic | charge 2.0-3.9 | 115 | Moderate positive charge | +| cysteine_rich | >= 2 Cys | 153 | Disulfide-stabilized | +| low_charge | charge < 2.0 | 118 | Low or neutral charge | +| short | length <= 12 | 21 | Short sequences | +| proline_rich | Pro >= 15% | 20 | Proline-rich | + +## Rules +- Slices must be mutually exclusive. +- Every benchmark candidate should belong to exactly one slice. +- Slice definitions should be documented in the benchmark card. +- New slices should be added when new candidate classes are identified. diff --git a/docs/evidence/REGISTRY_CHEAP_BASELINES.md b/docs/evidence/REGISTRY_CHEAP_BASELINES.md new file mode 100644 index 00000000..df979da9 --- /dev/null +++ b/docs/evidence/REGISTRY_CHEAP_BASELINES.md @@ -0,0 +1,16 @@ +# Cheap Baseline Registry for Benchmark Reports + +Registry of cheap baselines used for comparison. + +| Baseline | Task | Metric | Current Best | Reference | +|----------|:----:|:------:|:------------:|-----------| +| Charge density | AMP-vs-decoy | AUROC | 0.8166 | baseline_trivial.py | +| Selectivity proxy | Hemolysis detection | Det. AUROC | 0.5744 | features/physchem.py | +| Helix propensity | Structure prediction | Raw Pα | 0.6489 | features/physchem.py | +| Length | AMP-vs-decoy | AUROC | 0.5000 | baseline_trivial.py | + +## Rules +- Every benchmark should compare against at least one cheap baseline. +- Cheap baselines must be simpler than the method being evaluated. +- Baseline performance must be documented in the benchmark report. +- If a method doesn't beat its cheap baseline, it's marked experimental. diff --git a/docs/evidence/SCHEMA_FOLDER_MANIFEST.md b/docs/evidence/SCHEMA_FOLDER_MANIFEST.md new file mode 100644 index 00000000..9e6051a2 --- /dev/null +++ b/docs/evidence/SCHEMA_FOLDER_MANIFEST.md @@ -0,0 +1,29 @@ +# Artifact Manifest Schema for Generated Folders + +Schema for manifests that describe generated output folders. + +## Format +```json +{ + "folder": "outputs/", + "generated_at": "ISO 8601", + "file_count": 5, + "total_size_bytes": 1024000, + "files": { + "ranked.jsonl": {"size": 500000, "hash": "sha256:..."}, + "report.md": {"size": 20000, "hash": "sha256:..."}, + "manifest.json": {"size": 500, "hash": "sha256:..."} + } +} +``` + +## Required Fields +- `folder` — path to the generated folder +- `generated_at` — when the manifest was created +- `file_count` — number of files in the folder +- `files` — dict of filename to metadata (size, hash) + +## Rules +- Generate this manifest after running batch commands. +- Use the manifest to verify folder contents before sharing. +- If files are missing or hashes don't match, regenerate. diff --git a/tests/test_benchmark_caveat_extraction.py b/tests/test_benchmark_caveat_extraction.py new file mode 100644 index 00000000..f440acfc --- /dev/null +++ b/tests/test_benchmark_caveat_extraction.py @@ -0,0 +1,17 @@ +"""Test benchmark caveat extraction.""" +from pathlib import Path + + +def test_metrics_current_has_caveats(): + text = Path("docs/evidence/METRICS_CURRENT.md").read_text() + assert "limitation" in text.lower() or "caveat" in text.lower() or "note" in text.lower() + + +def test_benchmarking_plan_has_caveats(): + text = Path("docs/evidence/BENCHMARKING.md").read_text() + assert "caveat" in text.lower() or "limitation" in text.lower() or "Note" in text + + +def test_simulation_benchmark_has_verdict(): + text = Path("docs/evidence/SIMULATION_BENCHMARK.md").read_text() + assert "No improvement" in text diff --git a/tests/test_benchmark_seed_lock.py b/tests/test_benchmark_seed_lock.py new file mode 100644 index 00000000..3e0107be --- /dev/null +++ b/tests/test_benchmark_seed_lock.py @@ -0,0 +1,28 @@ +"""Test benchmark seed locking.""" +from pathlib import Path +import random + + +def test_benchmark_seed_is_deterministic(): + rng = random.Random(20260705) + a = rng.random() + rng = random.Random(20260705) + b = rng.random() + assert a == b, "Same seed should produce same random value" + + +def test_different_seeds_different(): + rng1 = random.Random(20260705) + rng2 = random.Random(20260706) + assert rng1.random() != rng2.random(), "Different seeds should produce different values" + + +def test_seed_used_in_benchmark_scripts(): + """Check that benchmark scripts use fixed seeds.""" + import glob + for script in glob.glob("scripts/benchmark_*.py"): + text = open(script).read() + if "random" in text or "Random" in text or "seed" in text: + # Script uses randomness; should have a fixed seed + assert "seed" in text.lower() or "Random(" in text or "rng" in text, \ + f"{script} uses randomness but may lack fixed seed" diff --git a/tests/test_ranked_jsonl_validator.py b/tests/test_ranked_jsonl_validator.py new file mode 100644 index 00000000..6822f918 --- /dev/null +++ b/tests/test_ranked_jsonl_validator.py @@ -0,0 +1,27 @@ +"""Test ranked JSONL validation.""" +import json +from pathlib import Path + + +def test_demo_ranked_is_valid_jsonl(): + """Each line of ranked JSONL should be valid JSON.""" + jsonl_path = Path("outputs/demo_ranked.jsonl") + if not jsonl_path.exists(): + return # May not be generated + lines = jsonl_path.read_text().strip().split("\n") + for i, line in enumerate(lines, 1): + try: + data = json.loads(line) + except json.JSONDecodeError: + assert False, f"Invalid JSON at line {i}" + assert "candidate_id" in data, f"Missing candidate_id at line {i}" + assert "sequence" in data, f"Missing sequence at line {i}" + + +def test_ranked_jsonl_has_scores(): + jsonl_path = Path("outputs/demo_ranked.jsonl") + if not jsonl_path.exists(): + return + data = json.loads(jsonl_path.read_text().split("\n")[0]) + assert "scores" in data, "Missing scores in ranked output" + assert "ensemble" in data["scores"], "Missing ensemble score"