diff --git a/.github/workflows/reproduce-artifacts.yml b/.github/workflows/reproduce-artifacts.yml new file mode 100644 index 0000000..5f7a5db --- /dev/null +++ b/.github/workflows/reproduce-artifacts.yml @@ -0,0 +1,70 @@ +name: reproduce-artifacts + +# AC1a (docs/PLAN.md section 5): download the three Releases, check every +# file against the SHA-256 in the committed manifests, rebuild analysis, +# Haiku latency, cost, figures and report offline, and require results/ and +# README.md to be byte-identical to the commit. No training, no API call, +# no secret. Not a required check: it downloads from GitHub Releases, so an +# outage there must not block merges. It runs weekly, on demand, and on pull +# requests that touch the files it exercises. +on: + schedule: + - cron: "41 4 * * 1" + workflow_dispatch: + pull_request: + paths: + - ".github/workflows/reproduce-artifacts.yml" + - "Makefile" + - "src/tinyrouter/reproduce.py" + - "src/tinyrouter/release.py" + - "src/tinyrouter/analysis*.py" + - "src/tinyrouter/selective.py" + - "src/tinyrouter/diagnostics.py" + - "src/tinyrouter/haiku.py" + - "src/tinyrouter/cost.py" + - "src/tinyrouter/figures.py" + - "src/tinyrouter/report.py" + - "src/tinyrouter/latency.py" + - "results/**" + - "README.md" + - "uv.lock" + +concurrency: + group: reproduce-artifacts-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: + contents: read + +env: + UV_VERSION: "0.10.2" + +jobs: + reproduce-artifacts: + name: reproduce-artifacts + # macOS arm64, the platform the committed figures were drawn on. On + # ubuntu-latest every JSON file, report.md and the README come out + # byte-identical, but the four PNGs do not (same matplotlib, different + # rasterized bytes; first seen on PR #19), and this check promises bytes. + runs-on: macos-15 + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Install uv + uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 + with: + version: ${{ env.UV_VERSION }} + enable-cache: true + + - name: make reproduce-artifacts + # Judged by the whole last line, not by the exit code alone. + run: | + make reproduce-artifacts 2>&1 | tee reproduce-artifacts.log + tail -n 1 reproduce-artifacts.log \ + | grep -Fx "completed reproduce-artifacts: results/ and README.md byte-identical to HEAD" + + - name: Show what differs (on failure) + if: failure() + run: git status --porcelain --untracked-files=all -- results README.md && git diff --stat -- results README.md diff --git a/.gitignore b/.gitignore index 0bc8d1b..cafad08 100644 --- a/.gitignore +++ b/.gitignore @@ -39,6 +39,12 @@ results/logits/ results/llm/*.jsonl results/llm/*.lock results/llm-smoke/ +# Reruns (`make reproduce`, `make reproduce-artifacts`) write only under +# reproduction/, a symlink into reproduction.nosync/ like checkpoints. +reproduction +reproduction/ +# Partial Release downloads (release.py moves a file into place once checked). +*.part *.npz *.tmp diff --git a/DEVLOG.md b/DEVLOG.md index d213f6a..003f723 100644 --- a/DEVLOG.md +++ b/DEVLOG.md @@ -4,6 +4,61 @@ --- +## 2026-09-29（深夜，七）：PR #19 審查修正（R1 到 R6），AC1b 實跑前固定比較範圍 + +### 本次工作 / 執行摘要 +- 狀態不變：RQ1 到 RQ5 完成；Tier 1 驗收未完成；AC1b 尚未實跑。 +- **R1**：續跑原本只比輸出的 SHA-256。審查實測同一個 `REPRO_ID` 多一個空 commit 後重跑，18 步有 17 步被跳過，verdict 仍 PASS。現在每一步記錄身分（HEAD、`uv.lock` 與 `configs/` 的 SHA-256、Python、torch、transformers 版本），目錄若由不同身分開始，preflight 判 `NOT STARTED`，訊息要求換新 id；`run_steps` 本身也會拒絕（`StaleStateError`）。comparison 的流程表附上每一步實際執行時的 commit，另列 `flow.commits`。 +- **R2**：比較範圍在實跑前補齊並寫進 PLAN §5.1（「實跑前補充」）：README router 表（LLM-only 3 列，ModernBERT k=10、k=100 的 small-only、hybrid 2% 與 5%、oracle 各 4 個指標，共 35 列）與 `ablation_comparison` 全部統計（22 列）要判定；延遲、k=100 訓練時間與峰值記憶體、成本模型只列不判（`LISTED, NOT JUDGED`）。 +- R3：AC1a workflow 的 PR paths 補上分析、成本、圖、報告、延遲模組、`results/**`、README、`uv.lock`。R4：`check-originals --snapshot` 比對執行前的快照（`results/` 所有檔案含 gitignored，加上 README）；AC1b 在第一步前拍快照，AC1a 在 verify-llm 後拍。R5：preflight 的 `git status` 或 `git rev-parse` 失敗時列為問題，不再當成乾淨。R6：README 註明 HF 快取是共用的，可用 `HF_HOME` 指到空目錄。 + +### 核心發現 / 數據 +- 已提交的結果與自己比較：router 35 列、消融 22 列都是 OK，仍是 0 個 `REVIEW REQUIRED`。 + +### Blockers / 遇到的問題 +- (無) + +### Next +- [ ] 合併後由協調者在乾淨 clone 實跑 AC1b + +### Files / Budget +- `src/tinyrouter/reproduce.py`、`comparison.py`；`tests/test_reproduce.py`、`test_comparison.py`；`.github/workflows/reproduce-artifacts.yml`；`README.md`、`docs/PLAN.md`、`DEVLOG.md` +- API 花費：US$0 + +--- + +## 2026-09-29（深夜，六）：AC1 拆成 AC1a／AC1b，`make reproduce-artifacts` 與 `make reproduce` + +### 本次工作 / 執行摘要 +- 狀態不變：RQ1 到 RQ5 完成；**Tier 1 驗收未完成**。這個 PR 只做程式與文件，AC1b 尚未實跑。 +- **PLAN §5**：AC1 原文保留，改稱 AC1b 並標「定義不變」；新增 AC1a（Release artifacts 驗證 SHA-256 後離線重建，不取代 AC1b）。新增 §5.1 決策紀錄：Drew 2026-09-29 的決定、凍結的六條 AC1b 通過標準、預算規則（原始 AC6 花費固定 US$3.19；AC1b 的 Haiku 是 reproduction-validation，獨立 US$5 上限，不相加）。其他 AC 與協定沒動。 +- **輸出位置可移動**：`load_config`、`load_protocol` 接受 `results_root`、`checkpoint_root`（兩者本來就是 `LOCATION_FIELDS`，不算進 run 的 identity）；`ac2`、`curves`、`pilots` 新增 `--results-root`、`--checkpoint-root`；Makefile 新增 `RESULTS_ROOT`、`CHECKPOINT_ROOT`、`README_OUT`，每個 target 都轉傳。預設為空，行為與先前相同。`report` 新增 `--readme-out`，而且 `--results-dir` 不是 `/results` 又沒給 `--readme-out` 時直接拒絕，避免用別的結果改寫 commit 裡的 README。 +- **`make reproduce-artifacts`（AC1a）**：`release.py` 從 commit 裡的 manifest 列出 76 個 Release 檔（3 個 AC2、72 個曲線、1 個 Haiku），下載到暫存名、比對 SHA-256 才換上；接著 verify-logits、verify-llm、analysis、llm-latency、cost、figures、report，最後 `check-originals` 要求 `results/` 與 `README.md` 與 HEAD 逐位元組相同。CPU 延遲不重量（隨機器變），沿用 commit 的檔案。新增 `reproduce-artifacts.yml`：每週、手動、以及改到相關檔案的 PR 會跑，不是必過檢查（依賴 GitHub Releases），actions 釘完整 SHA，`permissions: contents: read`。 +- **`make reproduce`（AC1b）**：順序照原本正式執行：AC2、pilot-lr、pilot-steps、baselines、兩條曲線、OOS 消融、verify-logits、Haiku（`MAX_USD=5`）、verify-llm、analysis、bench-cpu、llm-latency、cost、figures、report，最後檢查原始檔未被動過。全部寫進 `reproduction//`（results、checkpoints、原始 Haiku 預測、README、logs、steps.json、comparison）；`reproduction` 和 `checkpoints` 一樣是指向 `*.nosync` 的 symlink。Haiku 在新的 results root 開新 journal，所以 US$5 上限只涵蓋這次 reproduction-validation，也碰不到原始 journal 與花費紀錄。pilot 選出的值若不同，不改 `configs/curve.yaml`，照原 config 繼續，comparison 標 `REVIEW REQUIRED`。 +- 開跑前檢查（任一不過就不開始，寫 `preflight.json`，判定 `NOT STARTED`）：工作目錄乾淨、HEAD 已合併進 origin/main（記錄 SHA）、`uv sync --locked`、`ANTHROPIC_API_KEY` 存在（只記 true/false）、磁碟至少 8.8 GiB（估計峰值約 4.4 GB 的兩倍）。 +- 每一步以整行 completion 行加上離開碼 0 判定；缺一行就停，並寫出 comparison（FAIL）。可續跑：已通過且輸出 SHA-256 未變的步驟跳過，一旦有一步重跑，其後全部重跑（各 target 自己會跳過已完成的點）。 +- **comparison**（`comparison.py`）：流程每步狀態與 completion 行；AC2 三個 seed 對 95.7%；Haiku 8,600/8,600、identity、花費上限、逐列（split、index）不一致數、parse_failed；兩筆預算分開列；pilot 選值；README 首屏 14 個數字、兩個 encoder 每個 k 的 8 類準確率與 OOS recall（24 列）、所有門檻診斷統計（逐列）。規則：原始 std > 0 時看是否落在 mean ± std；std 為 0、null（只有一個 seed 可行）或單次執行（Haiku）時，任何差異或 seed 數不同都標 `REVIEW REQUIRED`；只在一邊出現也標。總判定只有流程或 AC2 失敗才 FAIL。 + +### 核心發現 / 數據 +- AC1a 在乾淨副本實跑：76/76 個 Release 檔 SHA-256 通過，重建後 `results/` 與 `README.md` 與 HEAD 逐位元組相同（約 2 分鐘，含下載）。 +- CI 第一次在 ubuntu-latest 跑 AC1a：所有 JSON、`report.md`、README 逐位元組相同，但四張 PNG 不同（同一版 matplotlib，點陣化結果的位元組不同）。workflow 改在 macOS arm64（產生 commit 裡圖檔的平台）執行後通過；README 註明這一點。 +- 已提交的結果和自己比較：0 個 `REVIEW REQUIRED`、PASS（測試）。 +- AC1b 時間估計（M4，依既有 run 記錄）：AC2 約 45 分、pilot-lr 約 90 分、pilot-steps 約 15 分、BERT 曲線約 55 分、ModernBERT 曲線約 130 分、消融約 60 分、Haiku 約 15 分，合計約 6 到 7 小時。 + +### Blockers / 遇到的問題 +- (無) + +### Next +- [ ] 合併後由協調者在乾淨 clone 實跑 `make setup && make reproduce`（AC1b），依 comparison 更新狀態 +- [ ] Tier 2（步驟 6） + +### Files / Budget +- 新增：`src/tinyrouter/release.py`、`reproduce.py`、`comparison.py`；`tests/test_release.py`、`test_reproduce.py`、`test_comparison.py`、`test_locations.py`；`.github/workflows/reproduce-artifacts.yml` +- 修改：`src/tinyrouter/config.py`、`protocol.py`、`ac2.py`、`curves.py`、`pilots.py`、`report.py`；`tests/test_ac2.py`、`test_curves.py`；`Makefile`、`.gitignore`、`README.md`、`docs/PLAN.md`、`docs/OPERATIONS.md`、`DEVLOG.md` +- API 花費：US$0 + +--- + ## 2026-09-29（深夜，五）：PR #18 審查修正（R1 到 R6） ### 本次工作 / 執行摘要 diff --git a/Makefile b/Makefile index f2c3498..ea2fc02 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,20 @@ .PHONY: setup lint format test test-network smoke train evaluate ac2 pilot-lr pilot-steps baselines \ curve oos-ablation verify-logits llm-smoke llm verify-llm analysis bench-cpu llm-latency cost figures report \ - clean-checkpoints + reproduce reproduce-artifacts clean-checkpoints CONFIG ?= configs/bert-base.yaml SEED ?= 42 +# Where output goes. Empty (the default) keeps each config's own paths, +# results/ and checkpoints/. `make reproduce` sets both to +# reproduction//... (and README_OUT for `make report`) so a rerun is +# written next to the committed results, never over them. +RESULTS_ROOT ?= +CHECKPOINT_ROOT ?= +README_OUT ?= +ROOT_ARGS := $(if $(RESULTS_ROOT),--results-root $(RESULTS_ROOT),) +LOCATION_ARGS := $(ROOT_ARGS) $(if $(CHECKPOINT_ROOT),--checkpoint-root $(CHECKPOINT_ROOT),) + # Load HF_TOKEN / ANTHROPIC_API_KEY from a gitignored .env if one exists. ENV_FILE := $(wildcard .env) UV_ENV := $(if $(ENV_FILE),--env-file $(ENV_FILE),) @@ -14,7 +24,7 @@ UV_ENV := $(if $(ENV_FILE),--env-file $(ENV_FILE),) # directories and the conventional names are symlinks to them. Without # this, iCloud tries to upload a multi-GB torch install and every # checkpoint, and can evict files mid-training. See README "iCloud". -NOSYNC_LINKS := .venv checkpoints +NOSYNC_LINKS := .venv checkpoints reproduction # No default target. On 2026-09-23 a zsh loop ran `make $t` with # t="curve MODEL=bert"; make got one argument, read it as a variable @@ -74,7 +84,7 @@ evaluate: # `make ac2` (its results already exist); getting them back takes FORCE=1, # which retrains all three seeds. ac2: - uv run $(UV_ENV) python -m tinyrouter.ac2 --config configs/bert-base.yaml $(if $(filter 1,$(FORCE)),--force,) + uv run $(UV_ENV) python -m tinyrouter.ac2 --config configs/bert-base.yaml $(LOCATION_ARGS) $(if $(filter 1,$(FORCE)),--force,) # Step 3 (docs/PLAN.md section 4, hyperparameter protocol). Order: # make pilot-lr then copy each model's selected lr into configs/curve.yaml @@ -92,26 +102,26 @@ ac2: # `completed 18/18 encoder points ()` (the ablation: # `completed 3/3 ablation points`), never just `completed`. pilot-lr: - uv run $(UV_ENV) python -m tinyrouter.pilots lr + uv run $(UV_ENV) python -m tinyrouter.pilots lr $(LOCATION_ARGS) pilot-steps: - uv run $(UV_ENV) python -m tinyrouter.pilots steps + uv run $(UV_ENV) python -m tinyrouter.pilots steps $(LOCATION_ARGS) # Majority-class and TF-IDF centroid baselines on every (k, seed) sample; seconds. baselines: - uv run python -m tinyrouter.baselines + uv run python -m tinyrouter.baselines $(ROOT_ARGS) # Checks configs/curve.yaml is filled in, then runs the baselines, then the curve. curve: @case "$(MODEL)" in bert|modernbert) ;; *) echo "usage: make curve MODEL=bert|modernbert"; exit 2;; esac - uv run $(UV_ENV) python -m tinyrouter.curves --model $(MODEL) + uv run $(UV_ENV) python -m tinyrouter.curves --model $(MODEL) $(LOCATION_ARGS) oos-ablation: - uv run $(UV_ENV) python -m tinyrouter.curves --ablation + uv run $(UV_ENV) python -m tinyrouter.curves --ablation $(LOCATION_ARGS) # Every archive listed in results/logits-manifest.json is present and matches its SHA-256. verify-logits: - uv run python -m tinyrouter.archive + uv run python -m tinyrouter.archive $(ROOT_ARGS) # Claude Haiku over CLINC150 in the 8-way routing space (docs/PLAN.md section 4, # AC6). Needs ANTHROPIC_API_KEY (in .env or exported); without it they exit 2. @@ -126,12 +136,12 @@ llm-smoke: uv run --group llm $(UV_ENV) python -m tinyrouter.llm_run --smoke $(if $(MAX_USD),--max-usd $(MAX_USD),) llm: - uv run --group llm $(UV_ENV) python -m tinyrouter.llm_run $(if $(MAX_USD),--max-usd $(MAX_USD),) + uv run --group llm $(UV_ENV) python -m tinyrouter.llm_run $(if $(MAX_USD),--max-usd $(MAX_USD),) $(ROOT_ARGS) # results/llm/haiku-8way.jsonl has every row once and matches its SHA-256 in # results/llm-manifest.json and results/llm/haiku-8way.json. verify-llm: - uv run python -m tinyrouter.llm_run --verify + uv run python -m tinyrouter.llm_run --verify $(ROOT_ARGS) # RQ2 to RQ4 from the stored logits and Haiku predictions (docs/PLAN.md sections 3 # and 4, AC3, AC4, AC6): no training, no API calls. Needs the archives in @@ -140,7 +150,7 @@ verify-llm: # Done means the whole last line `completed analysis (75/75 archives, 8600/8600 llm # rows, 25 groups)`. analysis: - uv run python -m tinyrouter.analysis_run --quiet + uv run python -m tinyrouter.analysis_run --quiet $(ROOT_ARGS) # AC5 latency. bench-cpu: both encoders on CPU, batch 1, validation rows # 0-499, 4 threads, the pretrained backbone with a 151-way head (latency @@ -149,25 +159,46 @@ analysis: # llm-latency: Haiku's per-call latency from results/llm/haiku-8way.jsonl # (Release; check it with verify-llm); writes results/efficiency/haiku_latency.json. bench-cpu: - uv run $(UV_ENV) python -m tinyrouter.latency cpu + uv run $(UV_ENV) python -m tinyrouter.latency cpu $(ROOT_ARGS) llm-latency: - uv run python -m tinyrouter.latency haiku + uv run python -m tinyrouter.latency haiku $(ROOT_ARGS) # RQ5 from committed JSON only: measured numbers and assumed prices kept # apart, break-even per scenario; writes results/cost/cost.json. cost: - uv run python -m tinyrouter.cost + uv run python -m tinyrouter.cost $(ROOT_ARGS) # README figures from results/analysis/*.json; writes results/figures/*.png. figures: - uv run --group figures python -m tinyrouter.figures + uv run --group figures python -m tinyrouter.figures $(ROOT_ARGS) # results/report.md and the README block between the BEGIN/END GENERATED # markers, from committed JSON. tests/test_report.py fails when either is stale. # Order after new results: analysis, bench-cpu, llm-latency, cost, figures, report. report: - uv run python -m tinyrouter.report + uv run python -m tinyrouter.report $(if $(RESULTS_ROOT),--results-dir $(RESULTS_ROOT),) $(if $(README_OUT),--readme-out $(README_OUT),) + +# AC1a (docs/PLAN.md section 5): download the three Releases into results/, +# check every file against the committed manifests, rebuild analysis, Haiku +# latency, cost, figures and report offline, and require results/ and +# README.md to be byte-identical to HEAD. No training, no API call, minutes. +# CPU latency is not rerun (machine-dependent); the committed file is used. +# Done means the last line `completed reproduce-artifacts: results/ and +# README.md byte-identical to HEAD`. +reproduce-artifacts: setup + uv run python -m tinyrouter.reproduce artifacts + +# AC1b (docs/PLAN.md section 5.1): the whole study again, about 6 to 7 hours +# on an Apple M4, Haiku about US$3 under its own US$5 reproduction-validation +# cap. Run it on a clean clone at a merged commit after `make setup`; it +# stops before anything else unless the tree is clean, HEAD is on +# origin/main, uv.lock is in sync, ANTHROPIC_API_KEY is set and 8.8 GiB are +# free. Everything goes under reproduction// (id = HEAD[:12] unless +# REPRO_ID is set); results/ and README.md are never written. Resumes. +# Verdict and differences: reproduction//comparison.{json,md}. +reproduce: + uv run $(UV_ENV) python -m tinyrouter.reproduce full $(if $(REPRO_ID),--id $(REPRO_ID),) # Removes every trained model. Disk is tight (see docs/PLAN.md section 6). clean-checkpoints: diff --git a/README.md b/README.md index eb90c9a..4a9b26f 100644 --- a/README.md +++ b/README.md @@ -138,9 +138,11 @@ Other targets: | `make cost` | RQ5 from committed JSON: measured training time, latency and Haiku spend, assumed prices, break-even per scenario; writes `results/cost/cost.json` | | `make figures` | the four README figures from `results/analysis/*.json`; writes `results/figures/*.png` (byte-identical on rerun with the same matplotlib) | | `make report` | `results/report.md` and the generated block of this README, from committed JSON; `tests/test_report.py` fails when either is stale | +| `make reproduce-artifacts` | AC1a: download the three Releases, check them against the committed manifests, rebuild everything offline, require `results/` and this README to be byte-identical to the commit (see Reproducing) | +| `make reproduce` | AC1b: the whole study again into `reproduction//`, then a comparison with the committed results (see Reproducing) | | `make clean-checkpoints` | delete all trained weights | -Run every target from the repository root: `checkpoint_root` and `results_root` in the configs are relative paths. +Run every target from the repository root: `checkpoint_root` and `results_root` in the configs are relative paths. `RESULTS_ROOT=...` and `CHECKPOINT_ROOT=...` (and `README_OUT=...` for `make report`) move a target's output elsewhere without changing the run's identity; `make reproduce` uses them. There is no default target: a bare `make`, or a quoted `make "curve MODEL=bert"` (one argument: GNU make 3.81, the macOS default, reads it as a variable assignment and used to run `make setup` and exit 0; make 4.x reads it as an unknown target), stops with an error. @@ -196,6 +198,9 @@ src/tinyrouter/ cost.py RQ5: measured costs, assumed prices, break-even figures.py README figures from results/analysis/*.json report.py results/*.json -> results/report.md and the README's generated block + release.py download the Release files and check them against the committed manifests + reproduce.py `make reproduce` (AC1b) and `make reproduce-artifacts` (AC1a) + comparison.py AC1b: a rerun against the committed results, verdict and REVIEW REQUIRED items smoke.py end-to-end wiring check tests/ pytest; `network` marker for Hub downloads ``` @@ -206,6 +211,34 @@ This repository sits in a folder synced by iCloud Drive. iCloud would otherwise Disk is also tight, so training keeps at most one checkpoint (`save_total_limit=1`) and deletes it once the final weights are saved. `make ac2` also deletes the weights of seeds 43 and 44 after their logits are archived; only seed 42's are kept. +## Reproducing + +There are two levels, and only the second is the acceptance criterion (docs/PLAN.md section 5). + +**AC1a, from the Release artifacts (minutes, no cost, no key).** + +```bash +make reproduce-artifacts +``` + +It downloads the logits archives and the Haiku predictions from the `ac2-bert-logits`, `curves-logits` and `haiku-predictions` Releases, checks each file against the SHA-256 in the committed `results/logits-manifest.json` and `results/llm-manifest.json`, rebuilds the analysis, Haiku latency, cost, figures and report, and fails unless `results/` and this README are byte-identical to the commit. CPU latency is machine-dependent, so the committed `results/efficiency/cpu_latency.json` is used, not remeasured. The last line is `completed reproduce-artifacts: results/ and README.md byte-identical to HEAD`. CI runs it weekly and on demand, on macOS arm64: on Linux every JSON file, the report and this README come out identical, but the PNG bytes of the figures differ. It checks that the published numbers follow from the stored predictions; it does not retrain anything. + +**AC1b, the whole study again.** + +```bash +git clone https://github.com/drewOrc/tinyrouter.git && cd tinyrouter +make setup +ANTHROPIC_API_KEY=... make reproduce +``` + +This trains and evaluates everything again in the original order (AC2, both pilots, baselines, both learning curves, the OOS ablation), reruns Haiku on all 8,600 queries, then analysis, latency, cost, figures and report. It takes about 6 to 7 hours on an Apple M4. Haiku costs about US$3; that run is a separate reproduction-validation run with its own US$5 cap, and its cost is not part of the original experiment's US$3.19. It needs `ANTHROPIC_API_KEY` (exported or in `.env`), a clean checkout at a commit merged into `main`, `uv.lock` in sync and 8.8 GiB free; it refuses to start otherwise. Everything it writes goes under `reproduction//` (`` is the first 12 characters of the commit), so `results/` and this README are never overwritten. It resumes after an interruption, but only under the same identity: each step records the commit, the SHA-256 of `uv.lock` and `configs/`, and the Python, torch and transformers versions, and a directory started under any other identity is refused (start a new `REPRO_ID`). The last step also checks that no file under `results/`, gitignored ones included, changed during the run. The verdict and every difference go to `reproduction//comparison.json` and `comparison.md`: FAIL only when a step fails or AC2 fails (any seed below 95.7%); a number outside the original mean ± std is marked `REVIEW REQUIRED` and explained, not tuned away. + +Training on Apple MPS is not bit-for-bit deterministic, so a rerun is not expected to reproduce every digit; the comparison uses the three-seed spread as the yardstick. Latency, training time, peak memory and the cost model depend on the machine: the comparison lists them with their differences but does not judge them. + +A clean clone is not a clean cache: the base models and the CLINC150 files come from the shared Hugging Face cache (`~/.cache/huggingface` by default). That does not affect correctness, since every model is pinned to a revision and every data file is checked against its SHA-256, but to download everything afresh point the cache at an empty directory, for example `HF_HOME=/tmp/tinyrouter-hf make reproduce`. + +**Status: AC1b has not been run yet.** Until it has passed, the status is RQ1 to RQ5 complete, Tier 1 acceptance not complete. + ## Reproducibility notes - Every direct dependency is pinned exactly in `pyproject.toml`; `uv.lock` pins the rest. diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 68c8d5f..4d4718d 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -12,8 +12,9 @@ TinyRouter is a research repository with no deployment target: nothing runs as a | `commit-hygiene` | rejects tool-attribution trailers in commit messages (patterns in `.github/disallowed-trailers.txt`) | yes | | `network` | `make test-network` and `make smoke` against the Hugging Face Hub | no | | `pr-text-hygiene` (`pr-text.yml`) | rejects the same patterns in the PR title and body, on open, edit, push and reopen | yes (added to the required checks once it is on `main`) | +| `reproduce-artifacts` (`reproduce-artifacts.yml`) | AC1a: `make reproduce-artifacts` (Release files checked against the committed manifests, analysis to README rebuilt offline, `results/` and `README.md` byte-identical to the commit); weekly, on demand, and on PRs that touch the Makefile, `reproduce.py`, `release.py` or the workflow | no | -`network` is not required because it depends on a third-party service. A Hub outage would otherwise block every merge, including ones that do not touch data loading, and a check people learn to override stops meaning anything. It still runs on every PR, and a red `network` on a PR that changes `data.py`, `smoke.py` or a config is a reason not to merge. The weekly run skips the cache, so it is the one that notices if a pinned upstream dataset or model disappears. +`network` and `reproduce-artifacts` are not required because they depend on a third-party service (the Hugging Face Hub, GitHub Releases). A Hub outage would otherwise block every merge, including ones that do not touch data loading, and a check people learn to override stops meaning anything. It still runs on every PR, and a red `network` on a PR that changes `data.py`, `smoke.py` or a config is a reason not to merge. The weekly run skips the cache, so it is the one that notices if a pinned upstream dataset or model disappears. `main` only takes squash merges, and a squash commit's title and message come from the PR title and body, not from the branch's commits. That is why `pr-text-hygiene` exists next to `commit-hygiene`; both use `.github/scripts/check-disallowed-text.sh` and the patterns in `.github/disallowed-trailers.txt`. `pr-text-hygiene` is its own workflow so that editing a PR description re-runs it without cancelling `ci.yml`. One gap is left: text typed by hand into the merge dialog is in no PR event, so CI cannot see it before the merge; the `commit-hygiene` run on the push to `main` finds it afterwards, and the fix is then a revert. diff --git a/docs/PLAN.md b/docs/PLAN.md index 332beee..b7f9709 100644 --- a/docs/PLAN.md +++ b/docs/PLAN.md @@ -92,7 +92,8 @@ TinyRouter 問三件事：**小模型要多少標註資料才夠？它知不知 | # | 條件 | 怎麼驗 | |---|---|---| -| AC1 | 乾淨 clone 後 `make setup && make reproduce` 跑完整流程（下載資料含 checksum → 訓練 → 評估 → 產表） | 在新目錄實跑一次 | +| AC1a | 從 Release artifacts（`ac2-bert-logits`、`curves-logits`、`haiku-predictions`）下載並驗證 SHA-256，離線重建分析、成本、圖、報告與 README。快速、零成本的日常驗證層，**不取代 AC1b** | `make reproduce-artifacts`：重建後 `results/` 與 `README.md` 與 commit 逐位元組相同；CI 每週與手動觸發 | +| AC1b | （原 AC1，**定義不變**）乾淨 clone 後 `make setup && make reproduce` 跑完整流程（下載資料含 checksum → 訓練 → 評估 → 產表） | 在新目錄實跑一次；通過標準見 §5.1 | | AC2 | **BERT 流程正確性檢查**：`bert-base-uncased` 在全量資料（k=100、OOS 250）訓練 151 類，150 類 in-scope 準確率在 3 seeds 下都 ≥ 95.7%。**這是從原論文 96.7% 推出的工程驗收門檻，不宣稱精確重現原論文**（實作、超參數、tokenizer、評估程式都可能與原論文不同）。第一次沒過先用 validation 調參，不下結論。BERT 是基準，不是主要結果 | `results/` JSON + report-check | | AC3 | 每一次訓練都把 val 與 test 的**逐筆 logits** 存檔（含 split、seed、k、模型 revision），之後所有 RQ2–RQ5 分析只讀存檔，不重跑模型 | 測試：分析函式只接受存檔格式 | | AC4 | **test 不參與任何調整**：T、門檻、聚合方式只由 validation 決定；有測試守住 | pytest（已有 `LeakageError`） | @@ -106,6 +107,30 @@ TinyRouter 問三件事：**小模型要多少標註資料才夠？它知不知 | AC12 | ONNX int8（主模型 ModernBERT；匯出不支援時改 BERT 並寫明）：8 類準確率相對 PyTorch 下降 ≤ 0.5pp；報 CPU batch 1 的 p50/p95 延遲、模型大小，並記錄硬體型號 | `make bench`，結果含硬體資訊 | | AC13 | 服務：`make serve` 起 FastAPI；Docker image 在 CI 建置並用極小模型打 `/healthz` 與一筆 `/route`；報本機壓測的 QPS 與 p95；Haiku fallback 預設關閉，沒有 API key 也能跑 | CI job + 壓測結果檔 | + +### 5.1 決策紀錄：AC1 拆成 AC1a 與 AC1b（Drew，2026-09-29） + +- **AC1 維持原定義，改稱 AC1b**，不弱化：乾淨 clone 後 `make setup && make reproduce` 跑完整流程（下載資料含 checksum、訓練、評估、產表），必須在乾淨 clone 實跑一次才算達成。 +- **新增 AC1a**：從 Release artifacts 驗證 SHA-256 並離線重建分析、成本、圖、報告與 README，作為快速、零成本的日常驗證層；**不取代 AC1b**。 +- AC1b 成功前，狀態只能寫成「RQ1 到 RQ5 完成；Tier 1 驗收未完成」。 + +**AC1b 通過標準（凍結，實跑前寫定，之後不改）：** + +1. 乾淨 clone 固定在已合併進 `main` 的 commit，使用 lockfile（`uv sync --locked`）、configs 裡的資料與模型 revision，以及 commit 裡的 Release artifact manifest（`results/logits-manifest.json`、`results/llm-manifest.json`）。 +2. 全流程成功完成：每一步的 completion 行（整行比對）、所有 SHA-256 檢查、analysis 與 report 的生成檢查都通過。 +3. AC2 重跑仍要求每個 seed 的 150 類 in-scope test 準確率 ≥ 95.7%。 +4. Haiku 完成 8,600/8,600 筆，journal、identity 與 cost 驗證通過；與原 run 的預測不一致筆數（逐列比對 split 與 index）、花費、parser fail 數都要報出，但不要求完全相同。 +5. 主要數字與原始結果的差異自動列成 comparison artifact（`reproduction//comparison.json` 與 `.md`）；偏離原始 mean ± std 的標記 `REVIEW REQUIRED`，需要解釋，但**不得調參重跑**。 +6. **只有完整流程或 AC2 失敗時，AC1b 才直接 FAIL。** 其餘偏離只標 `REVIEW REQUIRED`。 + +**實跑前補充（2026-09-29，PR #19 審查後、AC1b 第一次實跑前寫定）**：以下是比較範圍與續跑條件的釐清，不是放寬標準，第 1 到 6 條不變。 + +- 判定範圍（偏離原始 mean ± std 標 `REVIEW REQUIRED`）固定為：README 首屏數字；README router 表（LLM-only，以及 ModernBERT k=10 與 k=100 的 small-only、hybrid 目標 2% 與 5%、oracle，各報 8 類準確率、OOS recall、高信心 OOS 誤派、LLM 呼叫率）；學習曲線（兩個 encoder、每個 k 的 8 類準確率與 OOS recall）；`ablation_comparison`（OOS 250 對 0 的 AUROC、AUPRC 與 router 行為）；所有門檻診斷統計。 +- **只列不判**：延遲（`results/efficiency/*`）、k=100 的訓練時間與峰值記憶體、成本模型（`results/cost/cost.json`）隨機器變動。comparison 列出原始值、重跑值與差異，狀態固定為 `LISTED, NOT JUDGED`，不計入 `REVIEW REQUIRED`，也不影響判定。 +- 續跑只限同一身分：每一步記錄 HEAD、`uv.lock` 與 `configs/` 的 SHA-256、Python、torch 與 transformers 版本；同一個 reproduction 目錄若由不同身分開始，拒絕續跑，必須換新 id，所以一份 comparison 不會混到兩個 commit 的輸出（第 1 條的落實方式）。 + +**預算規則**：原始 AC6 實驗的 Haiku 花費固定是 US$3.19（完整 run US$3.18 加 smoke），這個數字不再變動。AC1b 的 Haiku 是另一次、明確標記為 reproduction-validation 的執行，有獨立的 US$5 上限，花費寫在 reproduction 目錄自己的紀錄裡；兩者不得混稱為原始實驗成本，也不相加。 + ## 6. 風險 | 風險 | 處理 | diff --git a/src/tinyrouter/ac2.py b/src/tinyrouter/ac2.py index 99eb66c..0b9bc0a 100644 --- a/src/tinyrouter/ac2.py +++ b/src/tinyrouter/ac2.py @@ -28,7 +28,7 @@ import json from pathlib import Path -from tinyrouter.config import RunConfig, load_config +from tinyrouter.config import RunConfig, add_location_arguments, load_config from tinyrouter.evaluate import RunPaths from tinyrouter.runs import ( EvaluateFn, @@ -167,8 +167,12 @@ def main(argv: list[str] | None = None) -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--config", default="configs/bert-base.yaml") parser.add_argument("--force", action="store_true", help="clear and rerun every seed") + add_location_arguments(parser) args = parser.parse_args(argv) - result = run_ac2(load_config(args.config), default_train, default_evaluate, args.force) + config = load_config( + args.config, results_root=args.results_root, checkpoint_root=args.checkpoint_root + ) + result = run_ac2(config, default_train, default_evaluate, args.force) print("seed val in-scope val OOS recall test in-scope verdict (test only)") for seed, entry in result["seeds"].items(): val, test = entry["validation"], entry["test"] diff --git a/src/tinyrouter/comparison.py b/src/tinyrouter/comparison.py new file mode 100644 index 0000000..3a46e47 --- /dev/null +++ b/src/tinyrouter/comparison.py @@ -0,0 +1,583 @@ +"""AC1b comparison: a rerun's outputs against the committed original ones (docs/PLAN.md 5.1). + +Pure functions over already-written JSON; nothing here trains, calls an +API or writes into ``results/``. ``reproduce.py`` calls ``build`` and +writes the result to ``reproduction//comparison.{json,md}``. + +The verdict follows the frozen AC1b criteria and nothing else: + +- ``FAIL`` when the flow did not finish (a step failed or never ran, or a + Haiku check failed) or when AC2 failed (any seed's 150-way in-scope test + accuracy below 95.7%); +- ``PASS`` otherwise, with the number of ``REVIEW REQUIRED`` items. A + number outside the original mean ± std needs an explanation, not a + retuned rerun, and never turns the verdict into FAIL. + +How one number is compared (``compare_stat``): + +- the original has a spread (a std above 0): the rerun's mean must lie in + [mean - std, mean + std], otherwise ``REVIEW REQUIRED``; +- the original has no spread (a single run such as Haiku, a std of 0, or a + std of null because only one seed had a feasible threshold): there is + nothing to scale a difference by, so any difference is + ``REVIEW REQUIRED``, and so is a different number of seeds behind it; +- present in only one of the two runs: ``REVIEW REQUIRED``; absent in both: ``OK``. + +Scope, fixed before the first AC1b run (docs/PLAN.md 5.1): the README's +first screen, its router table (k=10 and k=100, targets 2% and 5%), the +learning curves, the OOS ablation comparison and every threshold +diagnostic are judged as above. Latency, training time, peak memory and the +cost model (``results/efficiency/*``, the k=100 run records, +``results/cost/cost.json``) depend on the machine and are listed with their +differences but never judged (status ``LISTED, NOT JUDGED``). +""" + +from __future__ import annotations + +import json +from collections.abc import Iterator +from pathlib import Path + +from tinyrouter.ac2 import SEEDS as AC2_SEEDS +from tinyrouter.ac2 import THRESHOLD as AC2_THRESHOLD + +OK = "OK" +REVIEW = "REVIEW REQUIRED" +LISTED = "LISTED, NOT JUDGED" +PASS = "PASS" +FAIL = "FAIL" +EXACT_TOLERANCE = 1e-9 +EXPECTED_LLM_ROWS = 8600 +# docs/PLAN.md 5.1: the original AC6 spend is fixed; the rerun has its own cap. +ORIGINAL_AC6_USD = 3.19 +REPRODUCTION_CAP_USD = 5.0 +TARGET = "0.02" +CURVE_MODELS = ("modernbert", "bert") +CURVE_KS = (1, 5, 10, 25, 50, 100) +K100 = ("groups", "modernbert/k100", "result") +K10 = ("groups", "modernbert/k10", "result") +DIAG = (*K100, "diagnostics", TARGET) +K10_HYBRID = (*K10, "final", "fallback", TARGET, "hybrid", "test") +K100_HYBRID = (*K100, "final", "fallback", TARGET, "hybrid", "test") +# Every number on the README's first screen (tests/test_report.py FIRST_SCREEN). +HEADLINE: tuple[tuple[str, tuple[str, ...]], ...] = ( + ("ModernBERT k=100 small-only 8-way accuracy", (*K100, "final", "small_only", "accuracy_8")), + ("Haiku 4.5 8-way accuracy", ("llm_only_test", "accuracy_8")), + ("ModernBERT k=10 small-only 8-way accuracy", (*K10, "final", "small_only", "accuracy_8")), + ("ModernBERT k=10 hybrid 8-way accuracy", (*K10_HYBRID, "accuracy_8")), + ("ModernBERT k=10 hybrid Haiku call rate", (*K10_HYBRID, "llm_call_rate")), + ("k=100 threshold, test selective risk", (*DIAG, "test", "selective_risk")), + ("k=100 threshold, validation selective risk", (*DIAG, "validation", "selective_risk")), + ("validation OOS share", (*DIAG, "validation_oos_share")), + ("test OOS share", (*DIAG, "test_oos_share")), + ("share of the gap explained by OOS share", (*DIAG, "share_of_gap_explained_by_oos_share")), + ("k=100 kept-OOS error rate, test", (*DIAG, "test", "kept_oos_error_rate")), + ("k=100 kept-OOS error rate, validation", (*DIAG, "validation", "kept_oos_error_rate")), + ("ModernBERT k=100 hybrid Haiku call rate", (*K100_HYBRID, "llm_call_rate")), + ("ModernBERT k=100 hybrid 8-way accuracy", (*K100_HYBRID, "accuracy_8")), +) + + +def at(tree: object, path: tuple[str, ...]) -> object: + """The node at ``path``, or None when any key on the way is missing or null.""" + node = tree + for key in path: + if not isinstance(node, dict) or node.get(key) is None: + return None + node = node[key] + return node + + +def as_stat(value: object) -> dict | None: + """A ``{mean, std, n}`` view of a combined statistic or a single number; None if neither.""" + if isinstance(value, dict) and isinstance(value.get("mean"), int | float): + n = value.get("n", len(value["values"]) if isinstance(value.get("values"), list) else None) + return {"mean": float(value["mean"]), "std": value.get("std"), "n": n} + if isinstance(value, int | float) and not isinstance(value, bool): + return {"mean": float(value), "std": None, "n": 1} + return None + + +def compare_stat(label: str, original: object, reproduced: object) -> dict[str, object]: + """One row of the comparison; see the module docstring for the rule.""" + orig, rep = as_stat(original), as_stat(reproduced) + row: dict[str, object] = { + "metric": label, + "original_mean": None if orig is None else orig["mean"], + "original_std": None if orig is None else orig["std"], + "reproduced_mean": None if rep is None else rep["mean"], + "difference": None if orig is None or rep is None else rep["mean"] - orig["mean"], + } + if orig is None or rep is None: + both_absent = orig is None and rep is None + rule = "absent in both runs" if both_absent else "present in only one run" + return {**row, "status": OK if both_absent else REVIEW, "rule": rule} + difference = abs(rep["mean"] - orig["mean"]) + std = orig["std"] + if not isinstance(std, int | float) or std <= 0: + same = difference <= EXACT_TOLERANCE and rep["n"] == orig["n"] + rule = "original has no seed spread (single run, std 0 or null): any difference is reviewed" + return {**row, "status": OK if same else REVIEW, "rule": rule} + inside = difference <= std + EXACT_TOLERANCE + return {**row, "status": OK if inside else REVIEW, "rule": "within original mean ± std"} + + +ROUTER_METRICS = ("accuracy_8", "oos_recall", "high_conf_oos_misroute_rate", "llm_call_rate") +ROUTER_POINTS = ("modernbert/k10", "modernbert/k100") +TARGETS = ("0.02", "0.05") + + +def router_paths() -> list[tuple[str, tuple[str, ...]]]: + """The README router table: LLM-only, then small-only, hybrid per target and oracle.""" + llm = ("llm_only_test",) + paths = [ + ("LLM-only accuracy_8", (*llm, "accuracy_8")), + ("LLM-only oos_recall", (*llm, "oos", "recall")), + ("LLM-only llm_call_rate", (*llm, "llm_call_rate")), + ] + for point in ROUTER_POINTS: + final = ("groups", point, "result", "final") + routers = [("small-only", (*final, "small_only"))] + routers += [(f"hybrid {t}", (*final, "fallback", t, "hybrid", "test")) for t in TARGETS] + routers.append(("oracle", (*final, "oracle"))) + for name, base in routers: + paths += [(f"{point} {name} {m}", (*base, m)) for m in ROUTER_METRICS] + return paths + + +def router_rows(original: dict, reproduced: dict) -> list[dict]: + return [compare_stat(label, at(original, p), at(reproduced, p)) for label, p in router_paths()] + + +def ablation_rows(original: dict, reproduced: dict) -> list[dict]: + """Every statistic of ``ablation_comparison`` (OOS 250 against OOS 0).""" + base = ("ablation_comparison",) + paths = sorted(set(stat_leaves(at(original, base))) | set(stat_leaves(at(reproduced, base)))) + return [ + compare_stat("/".join(p), at(original, (*base, *p)), at(reproduced, (*base, *p))) + for p in paths + ] + + +def number_leaves(tree: object, prefix: tuple[str, ...] = ()) -> Iterator[tuple[str, ...]]: + """Paths of every number (not bool) in a JSON tree, list items by position.""" + if isinstance(tree, dict): + for key, value in tree.items(): + yield from number_leaves(value, (*prefix, str(key))) + elif isinstance(tree, list): + for i, value in enumerate(tree): + yield from number_leaves(value, (*prefix, str(i))) + elif isinstance(tree, int | float) and not isinstance(tree, bool): + yield prefix + + +def at_any(tree: object, path: tuple[str, ...]) -> object: + node = tree + for key in path: + if isinstance(node, list) and key.isdigit() and int(key) < len(node): + node = node[int(key)] + elif isinstance(node, dict) and key in node: + node = node[key] + else: + return None + return node + + +def listed_rows(name: str, original: object, reproduced: object) -> list[dict]: + """Machine-dependent numbers with original, rerun and difference; always LISTED.""" + paths = sorted(set(number_leaves(original)) | set(number_leaves(reproduced))) + rows = [] + for path in paths: + before, after = at_any(original, path), at_any(reproduced, path) + numeric = all( + isinstance(v, int | float) and not isinstance(v, bool) for v in (before, after) + ) + rows.append( + { + "metric": f"{name}:{'/'.join(path)}", + "original": before, + "reproduced": after, + "difference": after - before if numeric else None, # type: ignore[operator] + "status": LISTED, + } + ) + return rows + + +def k100_training(root: Path) -> dict | None: + """Training time and peak memory of each k=100 run, from the curve indexes and run JSONs.""" + out: dict[str, dict] = {} + for model in CURVE_MODELS: + index = read_json(root / "curves" / f"{model}.json") + if index is None: + return None + for point in index["points"]: + if point["k"] != 100: + continue + run = read_json(root / "runs" / f"{point['run_name']}.json") or {} + training = run.get("training", {}) + out[f"{model}/seed{point['seed']}"] = { + "train_wall_seconds": training.get("train_wall_seconds"), + "peak_memory": training.get("peak_memory"), + } + return out + + +def machine_dependent_rows(original_root: Path, reproduced_root: Path) -> list[dict]: + """Listed, not judged: latency, k=100 training time and peak memory, the cost model.""" + rows = [] + for name in ("efficiency/cpu_latency.json", "efficiency/haiku_latency.json", "cost/cost.json"): + rows += listed_rows( + name, read_json(original_root / name), read_json(reproduced_root / name) + ) + rows += listed_rows( + "k100 training", k100_training(original_root), k100_training(reproduced_root) + ) + return rows + + +def headline_rows(original: dict, reproduced: dict) -> list[dict]: + return [compare_stat(label, at(original, p), at(reproduced, p)) for label, p in HEADLINE] + + +def curve_rows(original: dict, reproduced: dict) -> list[dict]: + """Test 8-way accuracy and OOS recall of the small model at every k, both encoders.""" + rows = [] + for model in CURVE_MODELS: + for k in CURVE_KS: + small = ("groups", f"{model}/k{k}", "result", "final", "small_only") + for metric in ("accuracy_8", "oos_recall"): + path = (*small, metric) + label = f"{model} k={k} small-only {metric}" + rows.append(compare_stat(label, at(original, path), at(reproduced, path))) + return rows + + +def stat_leaves(tree: object, prefix: tuple[str, ...] = ()) -> Iterator[tuple[str, ...]]: + """Paths of every combined statistic (a dict with a ``mean``) under ``tree``.""" + if not isinstance(tree, dict): + return + if "mean" in tree: + yield prefix + return + for key, value in tree.items(): + yield from stat_leaves(value, (*prefix, key)) + + +def diagnostic_rows(original: dict, reproduced: dict) -> list[dict]: + """Every threshold-diagnostic statistic of every encoder point and target risk.""" + rows = [] + groups = sorted(set(original.get("groups", {})) | set(reproduced.get("groups", {}))) + for group in groups: + base = ("groups", group, "result", "diagnostics") + paths = sorted( + set(stat_leaves(at(original, base))) | set(stat_leaves(at(reproduced, base))) + ) + for path in paths: + full = (*base, *path) + label = f"{group} {'/'.join(path)}" + rows.append(compare_stat(label, at(original, full), at(reproduced, full))) + return rows + + +def read_jsonl(path: Path) -> list[dict]: + with path.open(encoding="utf-8") as handle: + return [json.loads(line) for line in handle if line.strip()] + + +def haiku_disagreements(original: list[dict], reproduced: list[dict]) -> dict[str, int]: + """Rows matched on (split, index): predicted agents that differ, rows in one run only.""" + before = {(r["split"], r["index"]): r["agent"] for r in original} + after = {(r["split"], r["index"]): r["agent"] for r in reproduced} + shared = before.keys() & after.keys() + return { + "compared_rows": len(shared), + "different_predictions": sum(before[key] != after[key] for key in shared), + "only_in_original": len(before.keys() - after.keys()), + "only_in_reproduction": len(after.keys() - before.keys()), + } + + +def haiku_section( + original_rows: list[dict] | None, + reproduced_rows: list[dict] | None, + reproduced_summary: dict | None, + original_summary: dict, +) -> dict[str, object]: + """Row count, prediction differences, parser failures and spend of the rerun.""" + if reproduced_rows is None or reproduced_summary is None: + return {"reached": False, "checks_passed": False, "problems": ["no reproduction run"]} + totals = reproduced_summary.get("totals", {}) + spent = float(totals.get("cost_usd", 0.0)) + problems = [] + if len(reproduced_rows) != EXPECTED_LLM_ROWS: + problems.append(f"{len(reproduced_rows)}/{EXPECTED_LLM_ROWS} rows") + if reproduced_summary.get("identity_sha256") != original_summary.get("identity_sha256"): + problems.append("identity differs from the original run (model, prompt or settings)") + if spent > REPRODUCTION_CAP_USD: + problems.append(f"spent US${spent:.4f}, above the US${REPRODUCTION_CAP_USD:g} cap") + body: dict[str, object] = { + "reached": True, + "rows": f"{len(reproduced_rows)}/{EXPECTED_LLM_ROWS}", + "parse_failed": sum(bool(r["parse_failed"]) for r in reproduced_rows), + "original_parse_failed": None, + "reproduction_cost_usd": round(spent, 6), + "checks_passed": not problems, + "problems": problems, + } + if original_rows is not None: + body["original_parse_failed"] = sum(bool(r["parse_failed"]) for r in original_rows) + body["predictions"] = haiku_disagreements(original_rows, reproduced_rows) + return body + + +def budget_section(reproduced_summary: dict | None) -> dict[str, object]: + """Two separate budgets; the rerun's spend is never added to the original experiment's.""" + spent = None + if reproduced_summary is not None: + spent = round(float(reproduced_summary.get("totals", {}).get("cost_usd", 0.0)), 6) + return { + "original_ac6_experiment": { + "usd": ORIGINAL_AC6_USD, + "note": "fixed: full run US$3.18 plus smoke; not changed by any rerun", + }, + "reproduction_validation": { + "usd": spent, + "cap_usd": REPRODUCTION_CAP_USD, + "note": "AC1b rerun only, its own journal and cap; not an experiment cost", + }, + "rule": "separate budgets; never summed or reported as one number", + } + + +def ac2_section(reproduced_ac2: dict | None, original_ac2: dict) -> dict[str, object]: + """Each seed's 150-way in-scope test accuracy against the 95.7% threshold.""" + seeds = {} + for seed in AC2_SEEDS: + entry = at(reproduced_ac2, ("seeds", str(seed), "test", "in_scope_accuracy_150")) + before = at(original_ac2, ("seeds", str(seed), "test", "in_scope_accuracy_150")) + value = float(entry) if isinstance(entry, int | float) else None + seeds[str(seed)] = { + "reproduced": value, + "original": before, + "passed": value is not None and value >= AC2_THRESHOLD, + } + passed = all(s["passed"] for s in seeds.values()) + return {"threshold": AC2_THRESHOLD, "seeds": seeds, "verdict": PASS if passed else FAIL} + + +def pilot_section(original: dict[str, dict | None], reproduced: dict[str, dict | None]) -> dict: + """Selected values; a different choice is reviewed, and configs/curve.yaml is not changed.""" + out = {} + for kind in ("lr", "steps"): + before = at(original.get(kind), ("selected",)) + after = at(reproduced.get(kind), ("selected",)) + same = after is not None and after == before + out[kind] = { + "original": before, + "reproduced": after, + "status": OK if same else REVIEW, + "note": "the rerun kept the committed configs/curve.yaml either way", + } + return out + + +def flow_passed(steps: list[dict], expected: list[str]) -> bool: + """Every expected step ran (or was resumed) and passed, in order.""" + passed = [s["name"] for s in steps if s.get("status") == PASS] + return passed == expected + + +def verdict(flow_ok: bool, haiku: dict, ac2: dict) -> str: + if not flow_ok or not haiku.get("checks_passed") or ac2["verdict"] != PASS: + return FAIL + return PASS + + +JUDGED_SECTIONS = ( + "headline", + "routers", + "learning_curves", + "ablation", + "threshold_diagnostics", +) + + +def review_count(sections: dict[str, object]) -> int: + """Rows and pilot choices marked REVIEW REQUIRED.""" + count = 0 + for name in JUDGED_SECTIONS: + rows = sections.get(name) or [] + count += sum(row["status"] == REVIEW for row in rows) # type: ignore[union-attr] + pilots = sections.get("pilots") or {} + count += sum(p["status"] == REVIEW for p in pilots.values()) # type: ignore[union-attr] + return count + + +def read_json(path: Path) -> dict | None: + return json.loads(path.read_text(encoding="utf-8")) if path.is_file() else None + + +def build( + *, + original_root: Path, + reproduced_root: Path, + original_haiku: Path, + steps: list[dict], + expected_steps: list[str], + context: dict[str, object], +) -> dict[str, object]: + """The whole comparison for one rerun; reads JSON only.""" + before = read_json(original_root / "analysis" / "summary.json") or {} + after = read_json(reproduced_root / "analysis" / "summary.json") + rep_jsonl = reproduced_root / "llm" / "haiku-8way.jsonl" + rep_summary = read_json(reproduced_root / "llm" / "haiku-8way.json") + haiku = haiku_section( + read_jsonl(original_haiku) if original_haiku.is_file() else None, + read_jsonl(rep_jsonl) if rep_jsonl.is_file() else None, + rep_summary, + read_json(original_root / "llm" / "haiku-8way.json") or {}, + ) + ac2 = ac2_section( + read_json(reproduced_root / "ac2.json"), read_json(original_root / "ac2.json") or {} + ) + sections: dict[str, object] = { + "pilots": pilot_section( + {k: read_json(original_root / "pilots" / f"{k}.json") for k in ("lr", "steps")}, + {k: read_json(reproduced_root / "pilots" / f"{k}.json") for k in ("lr", "steps")}, + ), + "headline": None if after is None else headline_rows(before, after), + "routers": None if after is None else router_rows(before, after), + "learning_curves": None if after is None else curve_rows(before, after), + "ablation": None if after is None else ablation_rows(before, after), + "threshold_diagnostics": None if after is None else diagnostic_rows(before, after), + "machine_dependent": machine_dependent_rows(original_root, reproduced_root), + } + flow_ok = flow_passed(steps, expected_steps) + return { + "format_version": 1, + "criteria": "docs/PLAN.md section 5.1 (frozen 2026-09-29)", + "verdict": verdict(flow_ok, haiku, ac2), + "review_required": review_count(sections), + "flow": { + "passed": flow_ok, + "expected_steps": expected_steps, + "steps": steps, + "commits": sorted({str((s.get("identity") or {}).get("head")) for s in steps}), + }, + "ac2": ac2, + "haiku": haiku, + "budget": budget_section(rep_summary), + **sections, + **context, + } + + +def fmt(value: object, scale: float = 100.0) -> str: + if not isinstance(value, int | float): + return "n/a" + return f"{scale * value:.2f}" + + +def stat_table(rows: list[dict], only_review: bool = False) -> list[str]: + lines = [ + "| metric | original mean (%) | original std (%) | rerun (%) | difference (pp) | status |", + "|---|---|---|---|---|---|", + ] + for row in rows: + if only_review and row["status"] != REVIEW: + continue + lines.append( + f"| {row['metric']} | {fmt(row['original_mean'])} | {fmt(row['original_std'])} | " + f"{fmt(row['reproduced_mean'])} | {fmt(row['difference'])} | {row['status']} |" + ) + return lines + + +def flow_lines(body: dict) -> list[str]: + lines = ["| step | status | commit | completion line(s) |", "|---|---|---|---|"] + for step in body["flow"]["steps"]: + found = "; ".join(f"`{line}`" for line in step.get("completion_found", [])) or "none" + status = step["status"] + (" (resumed)" if step.get("resumed") else "") + commit = str((step.get("identity") or {}).get("head", "unknown"))[:12] + lines.append(f"| {step['name']} | {status} | {commit} | {found} |") + return lines + + +def render_markdown(body: dict) -> str: + """comparison.md: the verdict first, then every section of comparison.json.""" + ac2, haiku, budget = body["ac2"], body["haiku"], body["budget"] + lines = [ + "# AC1b comparison", + "", + f"**Verdict: {body['verdict']}** ({body['review_required']} item(s) {REVIEW}). " + f"Criteria: {body['criteria']}. FAIL only when the flow or AC2 fails.", + "", + "## Flow", + "", + *flow_lines(body), + "", + f"## AC2 (threshold {100 * ac2['threshold']:.1f}%, every seed): {ac2['verdict']}", + "", + "| seed | rerun (%) | original (%) | passed |", + "|---|---|---|---|", + *( + f"| {seed} | {fmt(s['reproduced'])} | {fmt(s['original'])} | {s['passed']} |" + for seed, s in ac2["seeds"].items() + ), + "", + "## Haiku (reproduction-validation run)", + "", + "```json", + json.dumps(haiku, indent=2), + "```", + "", + "## Budget (separate, never summed)", + "", + f"- original AC6 experiment: US${budget['original_ac6_experiment']['usd']:.2f} (fixed)", + f"- reproduction-validation: US${budget['reproduction_validation']['usd']} " + f"(cap US${budget['reproduction_validation']['cap_usd']:g})", + "", + ] + lines += pilot_and_number_lines(body) + return "\n".join(lines) + "\n" + + +def pilot_and_number_lines(body: dict) -> list[str]: + lines = ["## Pilots", "", "| pilot | original | rerun | status |", "|---|---|---|---|"] + for kind, p in body["pilots"].items(): + lines.append(f"| {kind} | {p['original']} | {p['reproduced']} | {p['status']} |") + lines.append("") + if body["headline"] is None: + return [*lines, "Numbers: not reached (the analysis step did not finish).", ""] + diag = body["threshold_diagnostics"] + flagged = sum(row["status"] == REVIEW for row in diag) + listed = body["machine_dependent"] + return [ + *lines, + "## README first screen (test)", + "", + *stat_table(body["headline"]), + "", + "## README router table (test, 8-way)", + "", + *stat_table(body["routers"]), + "", + "## OOS ablation (OOS 250 against OOS 0)", + "", + *stat_table(body["ablation"]), + "", + f"## Machine-dependent numbers: {len(listed)} {LISTED} (all rows in the JSON)", + "", + "Latency, k=100 training time and peak memory, and the cost model vary with the " + "machine; they are listed with their differences and never marked REVIEW REQUIRED.", + "", + "## Learning curves (test, small model alone)", + "", + *stat_table(body["learning_curves"]), + "", + f"## Threshold diagnostics: {flagged} of {len(diag)} flagged (all rows in the JSON)", + "", + *stat_table(diag, only_review=True), + "", + ] diff --git a/src/tinyrouter/config.py b/src/tinyrouter/config.py index 5c45aff..deb6f62 100644 --- a/src/tinyrouter/config.py +++ b/src/tinyrouter/config.py @@ -2,6 +2,7 @@ from __future__ import annotations +import argparse from dataclasses import asdict, dataclass, fields, replace from pathlib import Path from typing import Literal @@ -109,7 +110,18 @@ def with_seed(self, seed: int) -> RunConfig: return replace(self, seed=seed) -def load_config(path: str | Path) -> RunConfig: +def load_config( + path: str | Path, + *, + results_root: str | Path | None = None, + checkpoint_root: str | Path | None = None, +) -> RunConfig: + """Read a config file; ``results_root`` and ``checkpoint_root`` override where output goes. + + The overrides are location fields (``LOCATION_FIELDS``), so the run's + identity does not change: ``make reproduce`` uses them to write a + rerun next to, never over, the committed ``results/``. + """ raw = yaml.safe_load(Path(path).read_text(encoding="utf-8")) or {} if not isinstance(raw, dict): raise ValueError(f"{path}: top level must be a mapping") @@ -117,4 +129,26 @@ def load_config(path: str | Path) -> RunConfig: unknown = sorted(set(raw) - allowed) if unknown: raise ValueError(f"{path}: unknown config keys {unknown}; allowed: {sorted(allowed)}") - return RunConfig(**raw) + return with_locations(RunConfig(**raw), results_root, checkpoint_root) + + +def with_locations( + config: RunConfig, + results_root: str | Path | None = None, + checkpoint_root: str | Path | None = None, +) -> RunConfig: + """The same run with its output moved; None keeps the config's own location.""" + moved: dict[str, str] = {} + if results_root is not None: + moved["results_root"] = str(results_root) + if checkpoint_root is not None: + moved["checkpoint_root"] = str(checkpoint_root) + return replace(config, **moved) + + +def add_location_arguments(parser: argparse.ArgumentParser) -> None: + """``--results-root`` and ``--checkpoint-root``; unset keeps each config's own paths.""" + parser.add_argument("--results-root", default=None, help="default: the config's results_root") + parser.add_argument( + "--checkpoint-root", default=None, help="default: the config's checkpoint_root" + ) diff --git a/src/tinyrouter/curves.py b/src/tinyrouter/curves.py index f9f4166..a4e9f49 100644 --- a/src/tinyrouter/curves.py +++ b/src/tinyrouter/curves.py @@ -57,7 +57,7 @@ check_points, publish_index, ) -from tinyrouter.config import RunConfig +from tinyrouter.config import RunConfig, add_location_arguments from tinyrouter.evaluate import RunPaths from tinyrouter.protocol import ( ABLATION_MODEL, @@ -240,8 +240,11 @@ def main(argv: list[str] | None = None) -> None: which.add_argument("--model", choices=MODELS) which.add_argument("--ablation", action="store_true", help="ModernBERT, k=100, 0 OOS rows") parser.add_argument("--protocol", default="configs/curve.yaml") + add_location_arguments(parser) args = parser.parse_args(argv) - protocol = load_protocol(args.protocol) + protocol = load_protocol( + args.protocol, results_root=args.results_root, checkpoint_root=args.checkpoint_root + ) # Build every config first: a protocol still missing a pilot value stops here, # before anything is computed or written. if args.ablation: diff --git a/src/tinyrouter/pilots.py b/src/tinyrouter/pilots.py index c76cd5e..ce237a5 100644 --- a/src/tinyrouter/pilots.py +++ b/src/tinyrouter/pilots.py @@ -37,7 +37,7 @@ from tinyrouter.archive import load_validation_logits from tinyrouter.calibrate import LeakageError, SplitLogits -from tinyrouter.config import RunConfig +from tinyrouter.config import RunConfig, add_location_arguments from tinyrouter.data import Split from tinyrouter.evaluate import RunPaths, read_training_summary from tinyrouter.labels import load_label_space @@ -317,14 +317,24 @@ def print_summary(body: dict) -> None: print(f"selected {body['pilot']}: {body['selected']} (validation only)") +def pilot_output(protocol: CurveProtocol, kind: str) -> Path: + """``/pilots/.json``, next to the AC2 run the lr pilot may reuse.""" + roots = {config.results_root for config in protocol.base_configs.values()} + if len(roots) != 1: + raise ValueError(f"the base configs write to different results roots: {sorted(roots)}") + return Path(roots.pop()) / "pilots" / f"{kind}.json" + + def main(argv: list[str] | None = None) -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("kind", choices=("lr", "steps")) parser.add_argument("--protocol", default="configs/curve.yaml") - parser.add_argument("--results-root", default="results") + add_location_arguments(parser) args = parser.parse_args(argv) - protocol = load_protocol(args.protocol) - out = Path(args.results_root) / "pilots" / f"{args.kind}.json" + protocol = load_protocol( + args.protocol, results_root=args.results_root, checkpoint_root=args.checkpoint_root + ) + out = pilot_output(protocol, args.kind) body = run_pilot(args.kind, protocol, out) print_summary(body) print(f"wrote {out}") diff --git a/src/tinyrouter/protocol.py b/src/tinyrouter/protocol.py index 1b53b28..2d0c4da 100644 --- a/src/tinyrouter/protocol.py +++ b/src/tinyrouter/protocol.py @@ -116,7 +116,13 @@ def _on_grid(value: object, grid: tuple, name: str) -> None: raise ProtocolError(f"{name} = {value!r} is not one of the protocol's values {grid}") -def load_protocol(path: str | Path = DEFAULT_PROTOCOL) -> CurveProtocol: +def load_protocol( + path: str | Path = DEFAULT_PROTOCOL, + *, + results_root: str | Path | None = None, + checkpoint_root: str | Path | None = None, +) -> CurveProtocol: + """Read ``configs/curve.yaml``; the two roots move every base config's output (config.py).""" raw = yaml.safe_load(Path(path).read_text(encoding="utf-8")) if not isinstance(raw, dict) or set(raw) != {"min_train_steps", "models"}: raise ProtocolError(f"{path}: top level must have exactly min_train_steps and models") @@ -137,5 +143,7 @@ def load_protocol(path: str | Path = DEFAULT_PROTOCOL) -> CurveProtocol: lr = entry["learning_rate"] lrs[model] = None if lr is None else float(lr) paths[model] = str(entry["config"]) - bases[model] = load_config(entry["config"]) + bases[model] = load_config( + entry["config"], results_root=results_root, checkpoint_root=checkpoint_root + ) return CurveProtocol(raw["min_train_steps"], lrs, bases, paths) diff --git a/src/tinyrouter/release.py b/src/tinyrouter/release.py new file mode 100644 index 0000000..f6e277c --- /dev/null +++ b/src/tinyrouter/release.py @@ -0,0 +1,131 @@ +"""Download the GitHub Release files the analysis reads, and check each against its SHA-256. + +Three Releases hold what git does not: ``ac2-bert-logits`` (the three AC2 +archives), ``curves-logits`` (every other logits archive) and +``haiku-predictions`` (``haiku-8way.jsonl``). The list of files and their +SHA-256 come from the committed manifests, ``results/logits-manifest.json`` +and ``results/llm-manifest.json``, never from the Release itself: a Release +asset that changed after the commit fails here instead of being trusted. + +A file already on disk with the right SHA-256 is kept. Anything else is +downloaded to a temporary name, checked, and only then moved into place, so +an interrupted download never leaves a file that looks complete. The last +line, printed only when every file checks out, is +``verified N/N release files``. +""" + +from __future__ import annotations + +import argparse +import json +import os +import time +import urllib.request +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path + +from tinyrouter.data import sha256_of + +REPOSITORY = "drewOrc/tinyrouter" +AC2_TAG = "ac2-bert-logits" +CURVES_TAG = "curves-logits" +HAIKU_TAG = "haiku-predictions" +AC2_ARCHIVES = frozenset(f"bert-base-uncased-full-seed{seed}.npz" for seed in (42, 43, 44)) +ATTEMPTS = 3 +TIMEOUT_SECONDS = 120 + +Fetch = Callable[[str, Path], None] + + +class ReleaseError(RuntimeError): + """A Release file is missing, or its bytes are not the ones the manifest records.""" + + +@dataclass(frozen=True) +class Asset: + tag: str + name: str + sha256: str + subdir: str + + @property + def url(self) -> str: + return f"https://github.com/{REPOSITORY}/releases/download/{self.tag}/{self.name}" + + +def manifest_files(path: Path) -> dict[str, dict]: + return dict(json.loads(path.read_text(encoding="utf-8"))["files"]) + + +def planned_assets(manifest_root: Path, only: str | None = None) -> list[Asset]: + """Every Release file the committed manifests list, with the Release that holds it.""" + assets: list[Asset] = [] + if only in (None, "logits"): + for name, entry in sorted(manifest_files(manifest_root / "logits-manifest.json").items()): + tag = AC2_TAG if name in AC2_ARCHIVES else CURVES_TAG + assets.append(Asset(tag, name, entry["sha256"], "logits")) + if only in (None, "llm"): + for name, entry in sorted(manifest_files(manifest_root / "llm-manifest.json").items()): + assets.append(Asset(HAIKU_TAG, name, entry["sha256"], "llm")) + return assets + + +def http_fetch(url: str, dest: Path) -> None: + last: Exception | None = None + for attempt in range(ATTEMPTS): + try: + with urllib.request.urlopen(url, timeout=TIMEOUT_SECONDS) as response: + dest.write_bytes(response.read()) + return + except OSError as exc: + last = exc + time.sleep(2**attempt) + raise ReleaseError(f"could not download {url} after {ATTEMPTS} attempts: {last}") + + +def ensure(asset: Asset, dest_root: Path, fetch: Fetch) -> bool: + """Put ``asset`` under ``dest_root`` with the recorded SHA-256; True if it was downloaded.""" + dest = dest_root / asset.subdir / asset.name + if dest.is_file() and sha256_of(dest) == asset.sha256: + return False + dest.parent.mkdir(parents=True, exist_ok=True) + partial = dest.with_name(dest.name + ".part") + fetch(asset.url, partial) + actual = sha256_of(partial) + if actual != asset.sha256: + partial.unlink() + raise ReleaseError( + f"{asset.tag}/{asset.name}: SHA-256 {actual} is not the manifest's {asset.sha256}; " + "the Release asset differs from the one this commit recorded" + ) + os.replace(partial, dest) + return True + + +def download( + manifest_root: Path, dest_root: Path, only: str | None = None, fetch: Fetch = http_fetch +) -> tuple[int, int]: + """(downloaded, total); raises ReleaseError on the first file that does not check out.""" + assets = planned_assets(manifest_root, only) + downloaded = sum(ensure(asset, dest_root, fetch) for asset in assets) + for asset in assets: + if sha256_of(dest_root / asset.subdir / asset.name) != asset.sha256: + raise ReleaseError(f"{asset.name} changed on disk after it was checked") + return downloaded, len(assets) + + +def main(argv: list[str] | None = None) -> None: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("action", choices=("download",)) + parser.add_argument("--manifest-root", default="results", help="where the manifests are") + parser.add_argument("--dest-root", default="results", help="logits/ and llm/ go under here") + parser.add_argument("--only", choices=("logits", "llm"), default=None) + args = parser.parse_args(argv) + downloaded, total = download(Path(args.manifest_root), Path(args.dest_root), args.only) + print(f"downloaded {downloaded}, kept {total - downloaded} already present") + print(f"verified {total}/{total} release files") + + +if __name__ == "__main__": + main() diff --git a/src/tinyrouter/report.py b/src/tinyrouter/report.py index effa130..1eacb8b 100644 --- a/src/tinyrouter/report.py +++ b/src/tinyrouter/report.py @@ -502,12 +502,17 @@ def replace_block(readme: str, block: str) -> str: return readme[:start] + block + readme[end + len(END) :] -def outputs(root: Path, repo: Path) -> dict[Path, str]: +def outputs(root: Path, repo: Path, readme_out: Path | None = None) -> dict[Path, str]: + """report.md and the README with its block rebuilt; ``readme_out`` keeps repo/README.md as is. + + ``make reproduce`` passes ``readme_out`` so a rerun writes its README next + to its own results instead of over the committed one. + """ r = Results.load(root) readme = repo / "README.md" return { root / "report.md": report_md(r), - readme: replace_block(readme.read_text(encoding="utf-8"), readme_block(r)), + readme_out or readme: replace_block(readme.read_text(encoding="utf-8"), readme_block(r)), } @@ -516,14 +521,23 @@ def main(argv: list[str] | None = None) -> None: parser.add_argument("--results-dir", default="results") parser.add_argument("--repo", default=".") parser.add_argument("--check", action="store_true", help="compare only; exit 1 if stale") + parser.add_argument("--readme-out", default=None, help="write the README here, not in --repo") args = parser.parse_args(argv) stale = [] - for path, text in outputs(Path(args.results_dir), Path(args.repo)).items(): + readme_out = Path(args.readme_out) if args.readme_out else None + moved = Path(args.results_dir).resolve() != (Path(args.repo) / "results").resolve() + if moved and readme_out is None and not args.check: + parser.error( + "--results-dir is not /results; pass --readme-out too, or the committed " + "README would be rewritten from other results" + ) + for path, text in outputs(Path(args.results_dir), Path(args.repo), readme_out).items(): current = path.read_text(encoding="utf-8") if path.is_file() else None if args.check: if current != text: stale.append(str(path)) else: + path.parent.mkdir(parents=True, exist_ok=True) path.write_text(text, encoding="utf-8") print(f"wrote {path}") if stale: diff --git a/src/tinyrouter/reproduce.py b/src/tinyrouter/reproduce.py new file mode 100644 index 0000000..e421002 --- /dev/null +++ b/src/tinyrouter/reproduce.py @@ -0,0 +1,659 @@ +"""``make reproduce`` (AC1b) and ``make reproduce-artifacts`` (AC1a), docs/PLAN.md section 5. + +AC1b, ``full``: the whole study again from a clean clone, in the order it +was first run: AC2, the two pilots, the baselines, both curves, the OOS +ablation, Haiku, then analysis, latency, cost, figures and report. Every +rerun output goes under ``reproduction//`` (```` defaults to the +first 12 characters of HEAD), never into the committed ``results/`` or +``README.md``: + +- ``results/``: what the make targets write, through ``RESULTS_ROOT``; +- ``checkpoints/``: weights while training, through ``CHECKPOINT_ROOT``; +- ``original/haiku-8way.jsonl``: the original Haiku predictions from the + Release, checked against the committed manifest, for the row comparison; +- ``README.md``: the rerun's README, through ``README_OUT``; +- ``logs/.log``, ``preflight.json``, ``steps.json`` and + ``comparison.{json,md}``. + +Haiku writes a fresh journal under the rerun's own results root, so its +US$5 cap (``MAX_USD=5``) covers the reproduction-validation run alone and +the original run's spend is neither reused nor touched. + +A step passes only when every one of its completion lines appears as a +whole line in its output, and its exit code is 0; the exit code alone never +counts as done. The first failing step stops the run, and the comparison is +still written, with verdict FAIL. Rerunning resumes: a step recorded as +passed whose output files still have the recorded SHA-256 is skipped, until +one step has to run again; from there every later step runs (each make +target resumes its own finished work, so that costs little). Every step +records the identity it ran under (HEAD, the SHA-256 of ``uv.lock`` and of +``configs/``, Python, torch and transformers); a directory started under +another identity is refused, not resumed, so one comparison never mixes +two commits. The last step checks that ``results/`` and ``README.md`` still +match HEAD and that no file under ``results/``, gitignored ones included, +differs from the snapshot taken before the first step. + +AC1a, ``artifacts``: download the three Releases into ``results/``, verify +them, rebuild analysis, Haiku latency, cost, figures and report, and check +that ``results/`` and ``README.md`` are byte-identical to HEAD. CPU latency +is not rerun (it depends on the machine); the committed file is used. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import platform +import shutil +import subprocess +import sys +from collections.abc import Callable +from dataclasses import dataclass, field +from importlib.metadata import PackageNotFoundError, version +from pathlib import Path + +from tinyrouter import comparison +from tinyrouter.archive import utc_now +from tinyrouter.data import DATASET_REVISION, sha256_of + +ORIGINAL_RESULTS = Path("results") +REPRODUCTION_DIR = Path("reproduction") +MAX_USD = 5 +KEY_ENV = "ANTHROPIC_API_KEY" +GIB = 1024**3 +# Peak extra disk while it runs: base models in the Hugging Face cache (about +# 1.1 GB), one ModernBERT run's final weights plus its one kept checkpoint +# with optimizer state (about 2.4 GB), AC2 seed 42's kept weights (0.44 GB), +# 75 logits archives (0.36 GB) and the Haiku journal. About 4.4 GB; twice +# that is required so a full disk cannot stop a six-hour run near the end. +REQUIRED_FREE_GIB = 8.8 +ORIGINALS = ("results", "README.md") +ORIGINALS_OK = "originals untouched: results/ and README.md match HEAD" +ARTIFACTS_SNAPSHOT = REPRODUCTION_DIR / "artifacts-originals-snapshot.json" +ANALYSIS_DONE = "completed analysis (75/75 archives, 8600/8600 llm rows, 25 groups)" +LLM_DONE = "completed 8600/8600 llm predictions" +FIGURES = ("learning_curves.png", "risk_coverage.png", "routers.png", "threshold_transfer.png") + +Executor = Callable[["Step", Path], tuple[int, list[str]]] +Git = Callable[[list[str]], tuple[int, str]] + + +class PathIsolationError(ValueError): + """A rerun would write into the committed results or README.""" + + +@dataclass(frozen=True) +class Step: + name: str + command: tuple[str, ...] + completion: tuple[str, ...] + outputs: tuple[str, ...] = () + + +@dataclass(frozen=True) +class Layout: + """Where one rerun writes; every path is relative to the repository root.""" + + run_id: str + base: Path = field(init=False) + + def __post_init__(self) -> None: + if not self.run_id or "/" in self.run_id or self.run_id.startswith("."): + raise PathIsolationError(f"bad reproduction id {self.run_id!r}") + object.__setattr__(self, "base", REPRODUCTION_DIR / self.run_id) + + @property + def results(self) -> Path: + return self.base / "results" + + @property + def checkpoints(self) -> Path: + return self.base / "checkpoints" + + @property + def original(self) -> Path: + return self.base / "original" + + @property + def readme(self) -> Path: + return self.base / "README.md" + + +def check_isolation(layout: Layout, repo: Path) -> None: + """Refuse any rerun path that is, or is inside, the committed results or README.""" + protected = [(repo / name).resolve() for name in ORIGINALS] + for path in (layout.results, layout.checkpoints, layout.original, layout.readme): + resolved = (repo / path).resolve() + for guard in protected: + if resolved == guard or guard in resolved.parents: + raise PathIsolationError(f"{path} resolves to {resolved}, inside {guard}") + + +def make(target: str, *variables: str) -> tuple[str, ...]: + return ("make", target, *variables) + + +def full_steps(layout: Layout) -> list[Step]: + """AC1b, in the order and with the settings of the original runs.""" + r = str(layout.results) + roots = (f"RESULTS_ROOT={r}", f"CHECKPOINT_ROOT={layout.checkpoints}") + return [ + Step( + "original-haiku", + ("uv", "run", "python", "-m", "tinyrouter.release", "download", "--only", "llm") + + ("--dest-root", str(layout.original)), + ("verified 1/1 release files",), + (f"{layout.original}/llm/haiku-8way.jsonl",), + ), + Step("ac2", make("ac2", *roots), (f"wrote {r}/ac2.json: PASS",), (f"{r}/ac2.json",)), + Step( + "pilot-lr", + make("pilot-lr", *roots), + (f"wrote {r}/pilots/lr.json",), + (f"{r}/pilots/lr.json",), + ), + Step( + "pilot-steps", + make("pilot-steps", *roots), + (f"wrote {r}/pilots/steps.json",), + (f"{r}/pilots/steps.json",), + ), + Step( + "baselines", + make("baselines", roots[0]), + ("completed 36/36 baseline points",), + (f"{r}/curves/baselines.json",), + ), + *curve_steps(r, roots), + Step( + "verify-logits", + make("verify-logits", roots[0]), + (f"OK 75 archive(s) match {r}/logits-manifest.json",), + (f"{r}/logits-manifest.json",), + ), + Step( + "llm", + make("llm", roots[0], f"MAX_USD={MAX_USD}"), + (LLM_DONE,), + (f"{r}/llm/haiku-8way.jsonl", f"{r}/llm/haiku-8way.json"), + ), + Step("verify-llm", make("verify-llm", roots[0]), (LLM_DONE,), (f"{r}/llm-manifest.json",)), + *rebuild_steps(r, (roots[0],), str(layout.readme), bench=True), + Step( + "originals-untouched", + module("check-originals", "--snapshot", str(layout.base / "originals-snapshot.json")), + (ORIGINALS_OK,), + ), + ] + + +def curve_steps(r: str, roots: tuple[str, str]) -> list[Step]: + steps = [ + Step( + f"curve-{model}", + make("curve", f"MODEL={model}", *roots), + (f"completed 18/18 encoder points ({model})",), + (f"{r}/curves/{model}.json",), + ) + for model in ("bert", "modernbert") + ] + ablation = Step( + "oos-ablation", + make("oos-ablation", *roots), + ("completed 3/3 ablation points",), + (f"{r}/curves/oos-ablation.json",), + ) + return [*steps, ablation] + + +def rebuild_steps(r: str, variables: tuple[str, ...], readme: str, bench: bool) -> list[Step]: + """analysis, latency, cost, figures and report from the logits and Haiku predictions. + + ``bench`` reruns the CPU latency benchmark (AC1b); AC1a keeps the committed + file, since latency depends on the machine. + """ + analysis_files = tuple(f"{r}/analysis/{n}.json" for n in ("summary", "curves", "haiku")) + latency = ("cpu_latency", "bench-cpu"), ("haiku_latency", "llm-latency") + steps = [Step("analysis", make("analysis", *variables), (ANALYSIS_DONE,), analysis_files)] + for stem, target in latency[0 if bench else 1 :]: + out = f"{r}/efficiency/{stem}.json" + steps.append(Step(target, make(target, *variables), (f"wrote {out}",), (out,))) + cost = f"{r}/cost/cost.json" + figures = tuple(f"{r}/figures/{n}" for n in FIGURES) + readme_var = () if readme == "README.md" else (f"README_OUT={readme}",) + return [ + *steps, + Step("cost", make("cost", *variables), (f"wrote {cost}",), (cost,)), + Step("figures", make("figures", *variables), tuple(f"wrote {f}" for f in figures), figures), + Step( + "report", + make("report", *variables, *readme_var), + (f"wrote {r}/report.md", f"wrote {readme}"), + (f"{r}/report.md", readme), + ), + ] + + +def artifact_steps() -> list[Step]: + """AC1a: Release files into results/, verified, then everything rebuilt offline.""" + r = str(ORIGINAL_RESULTS) + return [ + Step( + "release-download", + ("uv", "run", "python", "-m", "tinyrouter.release", "download"), + ("verified 76/76 release files",), + ), + Step( + "verify-logits", + make("verify-logits"), + (f"OK 75 archive(s) match {r}/logits-manifest.json",), + ), + Step("verify-llm", make("verify-llm"), (LLM_DONE,)), + Step( + "snapshot-originals", + module("snapshot", "--out", str(ARTIFACTS_SNAPSHOT)), + (f"wrote {ARTIFACTS_SNAPSHOT}",), + ), + *rebuild_steps(r, (), "README.md", bench=False), + Step( + "originals-untouched", + module("check-originals", "--snapshot", str(ARTIFACTS_SNAPSHOT)), + (ORIGINALS_OK,), + ), + ] + + +def module(action: str, *args: str) -> tuple[str, ...]: + return ("uv", "run", "python", "-m", "tinyrouter.reproduce", action, *args) + + +def run_command(step: Step, log_path: Path) -> tuple[int, list[str]]: + """Run ``step`` and stream its output to the terminal and to ``log_path``.""" + log_path.parent.mkdir(parents=True, exist_ok=True) + lines: list[str] = [] + with ( + log_path.open("a", encoding="utf-8") as log, + subprocess.Popen( + step.command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1 + ) as proc, + ): + log.write(f"$ {' '.join(step.command)}\n") + assert proc.stdout is not None + for line in proc.stdout: + sys.stdout.write(line) + log.write(line) + lines.append(line.rstrip("\n")) + return proc.returncode, lines + + +def judge_step(step: Step, code: int, lines: list[str]) -> dict[str, object]: + """PASS needs every completion line as a whole output line, and exit code 0.""" + seen = set(lines) + found = [line for line in step.completion if line in seen] + missing = [line for line in step.completion if line not in seen] + passed = not missing and code == 0 + return { + "name": step.name, + "status": comparison.PASS if passed else comparison.FAIL, + "exit_code": code, + "completion_found": found, + "completion_missing": missing, + } + + +def output_hashes(step: Step, repo: Path) -> dict[str, str | None]: + return { + path: sha256_of(repo / path) if (repo / path).is_file() else None for path in step.outputs + } + + +def resumable(step: Step, prior: dict | None, repo: Path) -> bool: + """Passed before, has outputs, and every output still has the recorded SHA-256.""" + if prior is None or prior.get("status") != comparison.PASS or not step.outputs: + return False + recorded = prior.get("outputs") + current = output_hashes(step, repo) + return recorded == current and None not in current.values() + + +def run_steps( + steps: list[Step], + repo: Path, + state_path: Path | None, + logs: Path, + executor: Executor, + identity: dict[str, object] | None = None, +) -> list[dict]: + """Run in order, resuming passed steps; stop at the first failure. + + With ``identity``, every recorded step must carry the same one + (``run_identity``), or nothing runs (``StaleStateError``). + """ + state = read_state(state_path) + refuse_stale(state, identity) + results: list[dict] = [] + rerun = False + for step in steps: + prior = state.get(step.name) + if not rerun and resumable(step, prior, repo): + print(f"[{step.name}] passed earlier and outputs unchanged, skipping", flush=True) + results.append({**prior, "resumed": True}) # type: ignore[dict-item] + continue + rerun = True + started = utc_now() + code, lines = executor(step, logs / f"{step.name}.log") + outcome = {**judge_step(step, code, lines), "started": started, "finished": utc_now()} + outcome["outputs"] = output_hashes(step, repo) + outcome["identity"] = identity + state[step.name] = outcome + write_state(state_path, state) + results.append(outcome) + if outcome["status"] != comparison.PASS: + print(f"[{step.name}] FAILED: missing {outcome['completion_missing']}", flush=True) + break + return results + + +def read_state(path: Path | None) -> dict[str, dict]: + if path is None or not path.is_file(): + return {} + return dict(json.loads(path.read_text(encoding="utf-8"))) + + +def write_state(path: Path | None, state: dict[str, dict]) -> None: + if path is None: + return + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_name(path.name + ".tmp") + tmp.write_text(json.dumps(state, indent=2) + "\n", encoding="utf-8") + os.replace(tmp, path) + + +def run_git(args: list[str]) -> tuple[int, str]: + done = subprocess.run(["git", *args], capture_output=True, text=True, check=False) + return done.returncode, done.stdout.rstrip() + + +def check_originals(repo: Path, snapshot: Path | None, git: Git = run_git) -> str: + """Tracked changes (git status) plus, with ``snapshot``, any file under results/ that differs. + + git status cannot see gitignored files (the logits archives, the Haiku + journal), so a snapshot of every file taken before the run is compared too. + """ + tracked = originals_changed(git) + problems = [tracked] if tracked else [] + if snapshot is not None: + before = json.loads(snapshot.read_text(encoding="utf-8")) + problems += snapshot_changes(before, snapshot_originals(repo)) + return "\n".join(problems) + + +def originals_changed(git: Git = run_git) -> str: + """``git status`` of results/ and README.md, untracked files included; empty when clean.""" + code, out = git(["status", "--porcelain", "--untracked-files=all", "--", *ORIGINALS]) + return out if code == 0 else f"git status failed ({code})" + + +class StaleStateError(RuntimeError): + """steps.json was written by another commit, lockfile, config set or library version.""" + + +def tree_sha256(root: Path) -> str: + """One SHA-256 over every file under ``root``: relative path and content hash, sorted.""" + lines = [ + f"{path.relative_to(root).as_posix()} {sha256_of(path)}" + for path in sorted(root.rglob("*")) + if path.is_file() + ] + return hashlib.sha256("\n".join(lines).encode("utf-8")).hexdigest() + + +def library_version(name: str) -> str | None: + try: + return version(name) + except PackageNotFoundError: + return None + + +def run_identity(repo: Path, git: Git = run_git) -> dict[str, object]: + """What a step's output depends on besides its inputs: commit, lockfile, configs, libraries.""" + code, head = git(["rev-parse", "HEAD"]) + if code != 0 or not head: + raise StaleStateError(f"git rev-parse HEAD failed (exit {code})") + return { + "head": head, + "uv_lock_sha256": sha256_of(repo / "uv.lock"), + "configs_sha256": tree_sha256(repo / "configs"), + "python": platform.python_version(), + "torch": library_version("torch"), + "transformers": library_version("transformers"), + } + + +def stale_steps(state: dict[str, dict], identity: dict[str, object]) -> list[str]: + """Recorded steps whose identity differs from ``identity``, with the fields that differ.""" + stale = [] + for name, entry in state.items(): + recorded = entry.get("identity") + if recorded == identity: + continue + if not isinstance(recorded, dict): + stale.append(f"{name} (no identity recorded)") + continue + fields = sorted(k for k in identity if recorded.get(k) != identity[k]) + stale.append(f"{name} ({', '.join(fields)} differ; recorded at {recorded.get('head')})") + return stale + + +def refuse_stale(state: dict[str, dict], identity: dict[str, object] | None) -> None: + if identity is None: + return + stale = stale_steps(state, identity) + if stale: + raise StaleStateError( + "this reproduction directory was started under another identity: " + f"{'; '.join(stale)}. Resuming would mix outputs of two setups; start a new " + "REPRO_ID (or delete that directory)." + ) + + +def snapshot_originals(repo: Path) -> dict[str, str]: + """SHA-256 of every file under results/ (gitignored ones too) and of README.md.""" + files = [p for p in sorted((repo / "results").rglob("*")) if p.is_file()] + files.append(repo / "README.md") + return {p.relative_to(repo).as_posix(): sha256_of(p) for p in files if p.is_file()} + + +def snapshot_changes(before: dict[str, str], after: dict[str, str]) -> list[str]: + changes = [f"removed {p}" for p in sorted(before.keys() - after.keys())] + changes += [f"added {p}" for p in sorted(after.keys() - before.keys())] + changes += [ + f"changed {p}" for p in sorted(before.keys() & after.keys()) if before[p] != after[p] + ] + return changes + + +@dataclass(frozen=True) +class Probes: + """What preflight looks at; replaced in tests.""" + + git: Git = run_git + free_bytes: Callable[[Path], int] = lambda path: shutil.disk_usage(path).free + key_present: Callable[[], bool] = lambda: bool(os.environ.get(KEY_ENV, "").strip()) + uv_sync: Callable[[], int] = lambda: ( + subprocess.run( + ["uv", "sync", "--locked", "--group", "llm", "--group", "figures"], check=False + ).returncode + ) + + +def preflight(repo: Path, probes: Probes) -> dict[str, object]: + """Clean tree, HEAD merged into origin/main, lockfile in sync, API key set, enough disk.""" + problems = [] + status_code, dirty = probes.git(["status", "--porcelain", "--untracked-files=all"]) + if status_code != 0: + problems.append(f"git status failed (exit {status_code}); cannot tell the tree is clean") + elif dirty: + problems.append("working tree is not clean (git status --porcelain is not empty)") + head_code, head = probes.git(["rev-parse", "HEAD"]) + if head_code != 0 or not head: + problems.append(f"git rev-parse HEAD failed (exit {head_code})") + fetched, _ = probes.git(["fetch", "--quiet", "origin", "main"]) + merged, _ = probes.git(["merge-base", "--is-ancestor", "HEAD", "FETCH_HEAD"]) + if fetched != 0 or merged != 0: + problems.append(f"HEAD {head} is not a commit merged into origin/main") + if probes.uv_sync() != 0: + problems.append("uv sync --locked failed: uv.lock does not match pyproject.toml") + key = probes.key_present() + if not key: + problems.append(f"{KEY_ENV} is not set (export it or put it in .env)") + free = probes.free_bytes(repo) / GIB + if free < REQUIRED_FREE_GIB: + problems.append(f"{free:.1f} GiB free, need {REQUIRED_FREE_GIB} GiB") + return { + "checked_at": utc_now(), + "head": head, + "api_key_present": key, + "free_gib": round(free, 1), + "required_free_gib": REQUIRED_FREE_GIB, + "problems": problems, + } + + +def pins(repo: Path) -> dict[str, object]: + """Lockfile, data and model revisions, and the Release manifests the rerun is held to.""" + revisions = {} + for name in ("bert-base.yaml", "modernbert-base.yaml"): + for line in (repo / "configs" / name).read_text(encoding="utf-8").splitlines(): + if line.startswith("model_revision:"): + revisions[name] = line.split(":", 1)[1].strip() + return { + "uv_lock_sha256": sha256_of(repo / "uv.lock"), + "dataset_revision": DATASET_REVISION, + "model_revisions": revisions, + "logits_manifest_sha256": sha256_of(repo / "results" / "logits-manifest.json"), + "llm_manifest_sha256": sha256_of(repo / "results" / "llm-manifest.json"), + } + + +def write_comparison(layout: Layout, repo: Path, steps: list[dict], context: dict) -> dict: + body = comparison.build( + original_root=repo / ORIGINAL_RESULTS, + reproduced_root=repo / layout.results, + original_haiku=repo / layout.original / "llm" / "haiku-8way.jsonl", + steps=steps, + expected_steps=[s.name for s in full_steps(layout)], + context=context, + ) + (repo / layout.base).mkdir(parents=True, exist_ok=True) + (repo / layout.base / "comparison.json").write_text( + json.dumps(body, indent=2) + "\n", encoding="utf-8" + ) + (repo / layout.base / "comparison.md").write_text( + comparison.render_markdown(body), encoding="utf-8" + ) + return body + + +def reproduce_full( + layout: Layout, repo: Path, probes: Probes | None = None, executor: Executor = run_command +) -> str: + """AC1b; returns the verdict (PASS, FAIL, or NOT STARTED when preflight refuses).""" + check_isolation(layout, repo) + probes = probes or Probes() + checks = preflight(repo, probes) + identity = None + if not checks["problems"]: + identity = run_identity(repo, probes.git) + stale = stale_steps(read_state(repo / layout.base / "steps.json"), identity) + if stale: + checks["problems"] = [ # type: ignore[index] + f"{layout.base} was started under another identity ({'; '.join(stale)}); " + "start a new REPRO_ID instead of resuming it" + ] + checks["identity"] = identity + (repo / layout.base).mkdir(parents=True, exist_ok=True) + (repo / layout.base / "preflight.json").write_text( + json.dumps(checks, indent=2) + "\n", encoding="utf-8" + ) + if checks["problems"]: + for problem in checks["problems"]: # type: ignore[attr-defined] + print(f"preflight: {problem}", file=sys.stderr) + return "NOT STARTED" + snapshot = repo / layout.base / "originals-snapshot.json" + if not snapshot.is_file(): + snapshot.write_text(json.dumps(snapshot_originals(repo), indent=2) + "\n", encoding="utf-8") + steps = run_steps( + full_steps(layout), + repo, + repo / layout.base / "steps.json", + repo / layout.base / "logs", + executor, + identity, + ) + context = {"reproduction_id": layout.run_id, "preflight": checks, "pins": pins(repo)} + body = write_comparison(layout, repo, steps, context) + return str(body["verdict"]) + + +def reproduce_artifacts(repo: Path, executor: Executor = run_command) -> bool: + """AC1a; True when every step passed and results/ and README.md match HEAD.""" + steps = artifact_steps() + results = run_steps(steps, repo, None, repo / REPRODUCTION_DIR / "artifacts-logs", executor) + return [r["name"] for r in results if r["status"] == comparison.PASS] == [s.name for s in steps] + + +def default_id(git: Git = run_git) -> str: + code, head = git(["rev-parse", "HEAD"]) + if code != 0 or not head: + raise SystemExit("error: not a git checkout; pass --id") + return head[:12] + + +def main(argv: list[str] | None = None) -> None: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "action", choices=("full", "artifacts", "check-originals", "snapshot", "compare") + ) + parser.add_argument("--id", default=None, help="reproduction/; default: HEAD[:12]") + parser.add_argument("--snapshot", default=None, help="check-originals: compare to this too") + parser.add_argument("--out", default=None, help="snapshot: where to write it") + args = parser.parse_args(argv) + repo = Path(".") + if args.action == "snapshot": + out = Path(args.out) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(snapshot_originals(repo), indent=2) + "\n", encoding="utf-8") + print(f"wrote {out}") + return + if args.action == "check-originals": + changed = check_originals(repo, Path(args.snapshot) if args.snapshot else None) + if changed: + print(f"changed since HEAD:\n{changed}") + raise SystemExit(1) + print(ORIGINALS_OK) + return + if args.action == "artifacts": + if not reproduce_artifacts(repo): + raise SystemExit(1) + print("completed reproduce-artifacts: results/ and README.md byte-identical to HEAD") + return + layout = Layout(args.id or default_id()) + if args.action == "compare": + state = read_state(repo / layout.base / "steps.json") + body = write_comparison( + layout, repo, list(state.values()), {"reproduction_id": layout.run_id} + ) + print(f"verdict {body['verdict']} ({body['review_required']} review required)") + return + verdict = reproduce_full(layout, repo) + record = "preflight.json" if verdict == "NOT STARTED" else "comparison.md" + print(f"AC1b verdict: {verdict}; see {layout.base}/{record}") + if verdict != comparison.PASS: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/tests/test_ac2.py b/tests/test_ac2.py index a3453eb..f2daed9 100644 --- a/tests/test_ac2.py +++ b/tests/test_ac2.py @@ -288,7 +288,7 @@ def test_fail_verdict_makes_the_command_exit_nonzero(base, monkeypatch, capsys): fake = FakePipeline(accuracy=0.9) monkeypatch.setattr(ac2, "default_train", fake.train) monkeypatch.setattr(ac2, "default_evaluate", fake.evaluate) - monkeypatch.setattr(ac2, "load_config", lambda _: base) + monkeypatch.setattr(ac2, "load_config", lambda _, **__: base) with pytest.raises(SystemExit) as exc: ac2.main(["--config", "unused.yaml"]) assert exc.value.code == 1 diff --git a/tests/test_comparison.py b/tests/test_comparison.py new file mode 100644 index 0000000..eb4fe81 --- /dev/null +++ b/tests/test_comparison.py @@ -0,0 +1,323 @@ +"""AC1b comparison verdict (docs/PLAN.md 5.1): only the flow or AC2 can FAIL it.""" + +import json +import shutil +from pathlib import Path + +import pytest + +from tinyrouter import comparison +from tinyrouter.comparison import ( + FAIL, + LISTED, + OK, + ORIGINAL_AC6_USD, + PASS, + REVIEW, + ac2_section, + budget_section, + build, + compare_stat, + haiku_disagreements, + haiku_section, + render_markdown, + verdict, +) + +RESULTS = Path(__file__).parent.parent / "results" +STEPS = ["ac2", "analysis"] +IDENTITY = json.loads((RESULTS / "llm" / "haiku-8way.json").read_text())["identity_sha256"] + + +def stat(mean, std, n=3): + return {"mean": mean, "std": std, "values": [mean] * n} + + +def test_a_rerun_inside_the_original_mean_plus_minus_std_is_ok(): + assert compare_stat("x", stat(0.90, 0.01), stat(0.909, 0.02))["status"] == OK + assert compare_stat("x", stat(0.90, 0.01), stat(0.891, 0.0))["status"] == OK + + +def test_a_rerun_outside_the_band_is_flagged_for_review(): + row = compare_stat("x", stat(0.90, 0.01), stat(0.911, 0.0)) + assert row["status"] == REVIEW + assert row["difference"] == pytest.approx(0.011) + + +def test_a_null_std_from_a_single_feasible_seed_flags_any_difference(): + single = {"mean": 0.5, "std": None, "n": 1, "values": [None, 0.5, None]} + assert compare_stat("x", single, dict(single))["status"] == OK + moved = {**single, "mean": 0.5001} + row = compare_stat("x", single, moved) + assert row["status"] == REVIEW + assert "no seed spread" in row["rule"] + + +def test_a_null_std_with_another_number_of_feasible_seeds_is_flagged_even_at_the_same_mean(): + single = {"mean": 0.5, "std": None, "n": 1, "values": [None, 0.5, None]} + assert compare_stat("x", single, stat(0.5, None))["status"] == REVIEW + + +def test_a_zero_std_or_a_single_run_number_needs_an_exact_match(): + assert compare_stat("share", stat(0.181818, 0.0), stat(0.181818, 0.0))["status"] == OK + assert compare_stat("haiku", 0.820727, 0.820727)["status"] == OK + assert compare_stat("haiku", 0.820727, 0.821)["status"] == REVIEW + + +def test_a_number_in_only_one_run_is_flagged_and_absent_in_both_is_ok(): + assert compare_stat("x", stat(0.5, 0.1), None)["status"] == REVIEW + assert compare_stat("x", None, stat(0.5, 0.1))["status"] == REVIEW + assert compare_stat("x", None, None)["status"] == OK + + +def ac2_json(*accuracies): + seeds = { + str(seed): {"test": {"in_scope_accuracy_150": acc}} + for seed, acc in zip((42, 43, 44), accuracies, strict=True) + } + return {"seeds": seeds} + + +def test_ac2_below_957_on_one_seed_fails_the_section(): + section = ac2_section(ac2_json(0.97, 0.9569, 0.97), ac2_json(0.97, 0.97, 0.97)) + assert section["verdict"] == FAIL + assert [s["passed"] for s in section["seeds"].values()] == [True, False, True] + assert ac2_section(ac2_json(0.957, 0.957, 0.957), {})["verdict"] == PASS + + +def test_ac2_missing_fails(): + assert ac2_section(None, {})["verdict"] == FAIL + + +def test_verdict_fails_only_on_flow_or_ac2(): + good_haiku, good_ac2 = {"checks_passed": True}, {"verdict": PASS} + assert verdict(True, good_haiku, good_ac2) == PASS + assert verdict(False, good_haiku, good_ac2) == FAIL + assert verdict(True, good_haiku, {"verdict": FAIL}) == FAIL + assert verdict(True, {"checks_passed": False}, good_ac2) == FAIL + + +def rows(agents, split="test", parse_failed=()): + return [ + {"split": split, "index": i, "agent": a, "parse_failed": i in parse_failed} + for i, a in enumerate(agents) + ] + + +def test_haiku_differences_are_counted_row_by_row_on_split_and_index(): + before = rows(["finance", "oos", "travel", "home"]) + after = rows(["finance", "travel", "travel"]) + after.append({"split": "validation", "index": 0, "agent": "oos", "parse_failed": False}) + counts = haiku_disagreements(before, after) + assert counts == { + "compared_rows": 3, + "different_predictions": 1, + "only_in_original": 1, + "only_in_reproduction": 1, + } + + +def test_the_same_agents_in_another_order_do_not_count_as_differences(): + before = rows(["finance", "oos", "travel"]) + assert haiku_disagreements(before, list(reversed(before)))["different_predictions"] == 0 + + +def summary(cost, identity=IDENTITY): + return {"identity_sha256": identity, "totals": {"cost_usd": cost}} + + +def test_haiku_section_reports_parse_failures_and_spend(): + full = rows(["oos"] * comparison.EXPECTED_LLM_ROWS, parse_failed={3, 7}) + body = haiku_section(full, full, summary(3.2), summary(3.18)) + assert body["checks_passed"] is True + assert body["rows"] == "8600/8600" + assert body["parse_failed"] == 2 + assert body["reproduction_cost_usd"] == 3.2 + assert body["predictions"]["different_predictions"] == 0 + + +def test_haiku_checks_fail_on_missing_rows_another_identity_or_spend_above_the_cap(): + short = rows(["oos"] * 10) + assert not haiku_section(short, short, summary(1.0), summary(1.0))["checks_passed"] + full = rows(["oos"] * comparison.EXPECTED_LLM_ROWS) + other = haiku_section(full, full, summary(1.0, "0" * 64), summary(1.0)) + assert "identity" in other["problems"][0] + over = haiku_section(full, full, summary(5.01), summary(1.0)) + assert not over["checks_passed"] + + +def test_the_two_budgets_stay_separate_and_the_original_is_fixed(): + budget = budget_section(summary(4.2)) + assert budget["original_ac6_experiment"]["usd"] == ORIGINAL_AC6_USD == 3.19 + assert budget["reproduction_validation"]["usd"] == 4.2 + assert budget["reproduction_validation"]["cap_usd"] == 5.0 + flat = json.dumps(budget) + assert "7.39" not in flat and "total" not in flat + + +@pytest.fixture +def roots(tmp_path): + """Committed results as the original, a copy as the rerun, and a Haiku journal for both.""" + original = tmp_path / "original" + names = ["analysis/summary.json", "ac2.json", "pilots/lr.json", "pilots/steps.json"] + names += ["efficiency/cpu_latency.json", "efficiency/haiku_latency.json", "cost/cost.json"] + names += [f"curves/{m}.json" for m in ("bert", "modernbert")] + names += [f"runs/{p.name}" for p in (RESULTS / "runs").glob("*k100-seed*.json")] + names += [f"runs/bert-base-uncased-full-seed{s}.json" for s in (42, 43, 44)] + for name in names: + (original / name).parent.mkdir(parents=True, exist_ok=True) + shutil.copy(RESULTS / name, original / name) + (original / "llm").mkdir() + shutil.copy(RESULTS / "llm" / "haiku-8way.json", original / "llm" / "haiku-8way.json") + rerun = tmp_path / "rerun" + shutil.copytree(original, rerun) + journal = "\n".join(json.dumps(r) for r in rows(["oos"] * 8600)) + "\n" + (rerun / "llm" / "haiku-8way.jsonl").write_text(journal) + (tmp_path / "haiku-original.jsonl").write_text(journal) + return original, rerun, tmp_path / "haiku-original.jsonl" + + +def run_build(roots, steps=None): + original, rerun, haiku = roots + steps = steps or [{"name": n, "status": PASS} for n in STEPS] + return build( + original_root=original, + reproduced_root=rerun, + original_haiku=haiku, + steps=steps, + expected_steps=STEPS, + context={}, + ) + + +def test_the_committed_results_against_themselves_pass_with_nothing_to_review(roots): + body = run_build(roots) + assert body["verdict"] == PASS + assert body["review_required"] == 0 + assert len(body["headline"]) == 14 + assert len(body["learning_curves"]) == 24 + assert len(body["threshold_diagnostics"]) > 100 + assert "Verdict: PASS" in render_markdown(body) + + +def edit(path: Path, change) -> None: + body = json.loads(path.read_text()) + change(body) + path.write_text(json.dumps(body)) + + +def test_a_number_off_by_more_than_its_std_is_reviewed_but_still_passes(roots): + _, rerun, _ = roots + + def shift(body): + stat = body["groups"]["modernbert/k100"]["result"]["final"]["small_only"]["accuracy_8"] + stat["mean"] += 0.05 + + edit(rerun / "analysis" / "summary.json", shift) + body = run_build(roots) + assert body["verdict"] == PASS + assert body["review_required"] >= 2 # first screen and the learning curve row + flagged = [r["metric"] for r in body["headline"] if r["status"] == REVIEW] + assert flagged == ["ModernBERT k=100 small-only 8-way accuracy"] + + +def test_a_different_pilot_choice_is_reviewed_not_failed(roots): + _, rerun, _ = roots + edit(rerun / "pilots" / "steps.json", lambda b: b.update(selected=200)) + body = run_build(roots) + assert body["verdict"] == PASS + assert body["pilots"]["steps"]["status"] == REVIEW + assert body["review_required"] == 1 + + +def test_an_ac2_seed_at_95_69_fails_the_whole_reproduction(roots): + _, rerun, _ = roots + edit( + rerun / "ac2.json", + lambda b: b["seeds"]["43"]["test"].update(in_scope_accuracy_150=0.9569), + ) + assert run_build(roots)["verdict"] == FAIL + + +def test_a_failed_step_fails_the_reproduction_even_when_every_number_matches(roots): + steps = [{"name": "ac2", "status": PASS}, {"name": "analysis", "status": FAIL}] + body = run_build(roots, steps) + assert body["verdict"] == FAIL + assert body["flow"]["passed"] is False + + +def test_a_step_that_never_ran_fails_the_reproduction(roots): + assert run_build(roots, [{"name": "ac2", "status": PASS}])["verdict"] == FAIL + + +def test_numbers_are_marked_not_reached_when_the_analysis_never_ran(roots): + _, rerun, _ = roots + (rerun / "analysis" / "summary.json").unlink() + body = run_build(roots, [{"name": "ac2", "status": FAIL}]) + assert body["headline"] is None + assert "not reached" in render_markdown(body) + + +def test_the_router_table_and_the_ablation_are_in_scope(roots): + body = run_build(roots) + labels = [r["metric"] for r in body["routers"]] + assert len(labels) == 3 + 2 * 4 * 4 + assert "modernbert/k10 hybrid 0.05 high_conf_oos_misroute_rate" in labels + assert "modernbert/k100 oracle llm_call_rate" in labels + assert "LLM-only oos_recall" in labels + ablation = {r["metric"]: r for r in body["ablation"]} + assert ablation["oos_250/oos_detection_test/auroc"]["original_mean"] == 0.983263 + assert ablation["oos_0/oos_detection_test/auroc"]["original_mean"] == 0.977988 + assert "oos_0/hybrid_test/0.02/llm_call_rate" in ablation + assert all(r["status"] == OK for r in body["routers"] + body["ablation"]) + + +def test_a_moved_router_or_ablation_number_is_reviewed(roots): + _, rerun, _ = roots + + def shift(body): + body["ablation_comparison"]["oos_0"]["oos_detection_test"]["auroc"]["mean"] += 0.01 + hybrid = body["groups"]["modernbert/k10"]["result"]["final"]["fallback"]["0.05"] + hybrid["hybrid"]["test"]["oos_recall"]["mean"] += 0.2 + + edit(rerun / "analysis" / "summary.json", shift) + body = run_build(roots) + assert [r["metric"] for r in body["ablation"] if r["status"] == REVIEW] == [ + "oos_0/oos_detection_test/auroc" + ] + assert [r["metric"] for r in body["routers"] if r["status"] == REVIEW] == [ + "modernbert/k10 hybrid 0.05 oos_recall" + ] + assert body["verdict"] == PASS and body["review_required"] == 2 + + +def test_machine_dependent_numbers_are_listed_and_never_judged(roots): + _, rerun, _ = roots + + def slower(body): + body["models"]["bert"]["end_to_end"]["p50_ms"] *= 10 + + edit(rerun / "efficiency" / "cpu_latency.json", slower) + edit(rerun / "cost" / "cost.json", lambda b: b.update(extra=1.0)) + run = next((rerun / "runs").glob("ModernBERT-base-k100-seed42.json")) + edit(run, lambda b: b["training"].update(train_wall_seconds=99999.0)) + body = run_build(roots) + listed = {r["metric"]: r for r in body["machine_dependent"]} + assert {r["status"] for r in listed.values()} == {LISTED} + p50 = listed["efficiency/cpu_latency.json:models/bert/end_to_end/p50_ms"] + assert p50["difference"] == pytest.approx(9 * p50["original"]) + assert listed["cost/cost.json:extra"]["original"] is None + wall = listed["k100 training:modernbert/seed42/train_wall_seconds"] + assert wall["reproduced"] == 99999.0 + assert any(m.startswith("k100 training:bert/seed44/peak_memory/") for m in listed) + assert body["verdict"] == PASS and body["review_required"] == 0 + + +def test_the_flow_records_the_commit_each_step_ran_at(roots): + steps = [ + {"name": n, "status": PASS, "identity": {"head": h * 40}} + for n, h in zip(STEPS, "ab", strict=True) + ] + body = run_build(roots, steps) + assert body["flow"]["commits"] == ["a" * 40, "b" * 40] + assert "| ac2 | PASS | aaaaaaaaaaaa |" in render_markdown(body) diff --git a/tests/test_curves.py b/tests/test_curves.py index 5f84090..0bb1bd6 100644 --- a/tests/test_curves.py +++ b/tests/test_curves.py @@ -165,7 +165,7 @@ def cli_fakes(monkeypatch, protocol) -> list[str]: calls: list[str] = [] fake = FakeRuns() - monkeypatch.setattr(curves, "load_protocol", lambda _: protocol) + monkeypatch.setattr(curves, "load_protocol", lambda _, **__: protocol) monkeypatch.setattr(curves, "run_baselines", lambda root: calls.append(f"baselines {root}")) def curve(name, configs, proto): diff --git a/tests/test_locations.py b/tests/test_locations.py new file mode 100644 index 0000000..a245187 --- /dev/null +++ b/tests/test_locations.py @@ -0,0 +1,138 @@ +"""Moved output roots: a rerun (``make reproduce``) writes beside ``results/``, never into it. + +The CLIs of AC2, the pilots and the curves run here with fake training and +evaluation in a scratch directory that holds a copy of ``configs/`` and a +``results/`` with one file in it. After all three, that ``results/`` must be +byte for byte what it was, no ``checkpoints/`` may exist, and every new file +must be under the moved roots. +""" + +import hashlib +import json +import shutil +from pathlib import Path + +import pytest + +from run_fakes import FakeRuns, fake_fingerprint +from tinyrouter import ac2, curves, pilots, report +from tinyrouter.ac2 import SetupError +from tinyrouter.config import load_config +from tinyrouter.protocol import load_protocol + +pytestmark = pytest.mark.filterwarnings("ignore::tinyrouter.calibrate.TemperatureBoundWarning") + +ROOT = Path(__file__).parent.parent +MOVED = ("--results-root", "repro/results", "--checkpoint-root", "repro/ckpt") + + +def tree(path: Path) -> dict[str, str]: + return { + str(p.relative_to(path)): hashlib.sha256(p.read_bytes()).hexdigest() + for p in sorted(path.rglob("*")) + if p.is_file() + } + + +def quiet(_: str) -> None: + pass + + +@pytest.fixture +def scratch(tmp_path, monkeypatch): + shutil.copytree(ROOT / "configs", tmp_path / "configs") + (tmp_path / "results").mkdir() + (tmp_path / "results" / "ac2.json").write_text('{"committed": true}\n') + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(curves, "expected_fingerprint", fake_fingerprint) + return tmp_path + + +def test_load_config_moves_only_the_two_location_fields(): + plain = load_config(ROOT / "configs" / "bert-base.yaml") + moved = load_config(ROOT / "configs" / "bert-base.yaml", results_root="r", checkpoint_root="c") + assert (moved.results_root, moved.checkpoint_root) == ("r", "c") + assert moved.identity() == plain.identity() + assert moved.run_name == plain.run_name + + +def test_load_protocol_moves_every_base_config_and_so_every_curve_and_pilot_run(): + protocol = load_protocol(ROOT / "configs" / "curve.yaml", results_root="r", checkpoint_root="c") + configs = [*protocol.curve_configs("bert"), *protocol.ablation_configs()] + configs += [p.config for p in pilots.lr_points(protocol) + pilots.steps_points(protocol)] + assert {c.results_root for c in configs} == {"r"} + assert {Path(c.checkpoint_root).parts[0] for c in configs} == {"c"} + assert pilots.pilot_output(protocol, "lr") == Path("r/pilots/lr.json") + + +def test_ac2_pilots_and_curves_with_moved_roots_leave_results_untouched(scratch, monkeypatch): + before = tree(scratch / "results") + fake = FakeRuns() + monkeypatch.setattr(ac2, "default_train", fake.train) + monkeypatch.setattr(ac2, "default_evaluate", fake.evaluate) + with pytest.raises(SetupError, match="test rows"): # fake splits are not 5,500 rows + ac2.main(["--config", "configs/bert-base.yaml", *MOVED]) + real_pilot = pilots.run_pilot + monkeypatch.setattr( + pilots, + "run_pilot", + lambda kind, protocol, out: real_pilot( + kind, protocol, out, fake.train, fake.validation, quiet + ), + ) + pilots.main(["lr", *MOVED]) + baseline_roots: list[Path] = [] + monkeypatch.setattr(curves, "run_baselines", baseline_roots.append) + real_curve = curves.run_curve + monkeypatch.setattr( + curves, + "run_curve", + lambda name, configs, proto: real_curve( + name, configs, proto, fake.train, fake.evaluate, quiet + ), + ) + curves.main(["--model", "bert", *MOVED]) + + assert tree(scratch / "results") == before + assert not (scratch / "checkpoints").exists() + assert baseline_roots == [Path("repro/results")] + outside = [p for p in tree(scratch) if not p.startswith(("repro/", "configs/", "results/"))] + assert outside == [] + moved = scratch / "repro" / "results" + assert len(list((moved / "runs").glob("bert-base-uncased-full-seed*.json"))) == 3 + lr = json.loads((moved / "pilots" / "lr.json").read_text()) + reused = [p for p in lr["points"] if p["source"].startswith("reused")] + assert [p["run_name"] for p in reused] == ["bert-base-uncased-k100-seed42"] + index = json.loads((moved / "curves" / "bert.json").read_text()) + k100 = [p for p in index["points"] if p["k"] == 100] + assert {p["reused_from"] for p in k100} == { + f"bert-base-uncased-full-seed{s}" for s in (42, 43, 44) + } + + +@pytest.fixture +def report_copy(tmp_path): + shutil.copytree( + ROOT / "results", tmp_path / "moved", ignore=shutil.ignore_patterns("logits", "*.jsonl") + ) + shutil.copy(ROOT / "README.md", tmp_path / "README.md") + return tmp_path + + +def test_report_with_readme_out_leaves_the_repo_readme_alone(report_copy): + before = (report_copy / "README.md").read_bytes() + out = report_copy / "rerun" / "README.md" + report.main( + ["--results-dir", str(report_copy / "moved"), "--repo", str(report_copy)] + + ["--readme-out", str(out)] + ) + assert (report_copy / "README.md").read_bytes() == before + assert out.read_bytes() == before # same JSON gives the same README + + +def test_report_refuses_moved_results_without_readme_out(report_copy): + before = (report_copy / "README.md").read_bytes() + with pytest.raises(SystemExit) as exc: + report.main(["--results-dir", str(report_copy / "moved"), "--repo", str(report_copy)]) + assert exc.value.code == 2 + assert (report_copy / "README.md").read_bytes() == before diff --git a/tests/test_release.py b/tests/test_release.py new file mode 100644 index 0000000..2d79041 --- /dev/null +++ b/tests/test_release.py @@ -0,0 +1,86 @@ +"""Release downloads are checked against the committed manifests, never trusted as given.""" + +import hashlib +from pathlib import Path + +import pytest + +from tinyrouter.release import ( + AC2_TAG, + CURVES_TAG, + HAIKU_TAG, + Asset, + ReleaseError, + download, + ensure, + planned_assets, +) + +RESULTS = Path(__file__).parent.parent / "results" + + +def test_the_committed_manifests_list_76_files_in_three_releases(): + assets = planned_assets(RESULTS) + by_tag = {tag: [a for a in assets if a.tag == tag] for tag in (AC2_TAG, CURVES_TAG, HAIKU_TAG)} + assert len(assets) == 76 + assert sorted(a.name for a in by_tag[AC2_TAG]) == [ + f"bert-base-uncased-full-seed{s}.npz" for s in (42, 43, 44) + ] + assert len(by_tag[CURVES_TAG]) == 72 + assert [a.name for a in by_tag[HAIKU_TAG]] == ["haiku-8way.jsonl"] + assert by_tag[HAIKU_TAG][0].subdir == "llm" + assert by_tag[HAIKU_TAG][0].url.endswith( + "/releases/download/haiku-predictions/haiku-8way.jsonl" + ) + assert len(planned_assets(RESULTS, only="llm")) == 1 + + +def asset_for(content: bytes) -> Asset: + return Asset("t", "f.npz", hashlib.sha256(content).hexdigest(), "logits") + + +def test_a_download_with_the_manifest_sha_is_moved_into_place(tmp_path): + asset = asset_for(b"right") + assert ensure(asset, tmp_path, lambda url, dest: dest.write_bytes(b"right")) is True + assert (tmp_path / "logits" / "f.npz").read_bytes() == b"right" + assert not list(tmp_path.rglob("*.part")) + + +def test_a_download_with_other_bytes_is_refused_and_leaves_nothing(tmp_path): + asset = asset_for(b"right") + with pytest.raises(ReleaseError, match="differs"): + ensure(asset, tmp_path, lambda url, dest: dest.write_bytes(b"tampered")) + assert not list(tmp_path.rglob("*.npz")) and not list(tmp_path.rglob("*.part")) + + +def test_a_file_already_present_and_correct_is_not_downloaded_again(tmp_path): + asset = asset_for(b"right") + (tmp_path / "logits").mkdir() + (tmp_path / "logits" / "f.npz").write_bytes(b"right") + + def refuse(url, dest): + raise AssertionError("should not download") + + assert ensure(asset, tmp_path, refuse) is False + + +def test_a_present_but_wrong_file_is_replaced(tmp_path): + asset = asset_for(b"right") + (tmp_path / "logits").mkdir() + (tmp_path / "logits" / "f.npz").write_bytes(b"stale") + assert ensure(asset, tmp_path, lambda url, dest: dest.write_bytes(b"right")) is True + assert (tmp_path / "logits" / "f.npz").read_bytes() == b"right" + + +def test_download_counts_every_file_of_the_manifest(tmp_path): + manifests = tmp_path / "m" + manifests.mkdir() + sha = hashlib.sha256(b"j").hexdigest() + (manifests / "llm-manifest.json").write_text( + f'{{"files": {{"haiku-8way.jsonl": {{"sha256": "{sha}"}}}}}}' + ) + downloaded, total = download( + manifests, tmp_path / "d", only="llm", fetch=lambda url, dest: dest.write_bytes(b"j") + ) + assert (downloaded, total) == (1, 1) + assert download(manifests, tmp_path / "d", only="llm", fetch=None) == (0, 1) # type: ignore[arg-type] diff --git a/tests/test_reproduce.py b/tests/test_reproduce.py new file mode 100644 index 0000000..8fa412b --- /dev/null +++ b/tests/test_reproduce.py @@ -0,0 +1,424 @@ +"""``make reproduce`` (AC1b) and ``make reproduce-artifacts`` (AC1a): driver behaviour. + +No test here trains, downloads or calls an API: steps run through a fake +executor, and preflight reads fake probes. +""" + +import json +import os +import subprocess +from pathlib import Path + +import pytest + +from tinyrouter import reproduce +from tinyrouter.comparison import FAIL, PASS +from tinyrouter.reproduce import ( + Layout, + PathIsolationError, + Probes, + Step, + artifact_steps, + check_isolation, + full_steps, + judge_step, + preflight, + reproduce_full, + run_steps, +) + +ROOT = Path(__file__).parent.parent +LAYOUT = Layout("abc123def456") + + +def test_every_rerun_output_is_under_reproduction_id_and_none_is_an_original(): + for step in full_steps(LAYOUT): + for path in step.outputs: + assert path.startswith("reproduction/abc123def456/"), (step.name, path) + commands = " ".join(" ".join(s.command) for s in full_steps(LAYOUT)) + assert "RESULTS_ROOT=reproduction/abc123def456/results" in commands + assert "README_OUT=reproduction/abc123def456/README.md" in commands + + +def test_training_steps_move_both_results_and_checkpoints(): + training = {"ac2", "pilot-lr", "pilot-steps", "curve-bert", "curve-modernbert", "oos-ablation"} + for step in full_steps(LAYOUT): + if step.name in training: + assert "CHECKPOINT_ROOT=reproduction/abc123def456/checkpoints" in step.command + assert "RESULTS_ROOT=reproduction/abc123def456/results" in step.command + + +def test_the_full_flow_keeps_the_original_order_and_haiku_gets_its_own_five_dollar_cap(): + names = [s.name for s in full_steps(LAYOUT)] + assert names == [ + "original-haiku", + "ac2", + "pilot-lr", + "pilot-steps", + "baselines", + "curve-bert", + "curve-modernbert", + "oos-ablation", + "verify-logits", + "llm", + "verify-llm", + "analysis", + "bench-cpu", + "llm-latency", + "cost", + "figures", + "report", + "originals-untouched", + ] + llm = next(s for s in full_steps(LAYOUT) if s.name == "llm") + assert "MAX_USD=5" in llm.command + + +def make_dry_run(*args: str) -> str: + env = {k: v for k, v in os.environ.items() if k not in {"MAKEFLAGS", "MFLAGS", "MAKELEVEL"}} + done = subprocess.run( + ["make", "-n", *args], cwd=ROOT, env=env, capture_output=True, text=True, check=True + ) + return done.stdout + + +def test_each_make_command_of_the_rerun_passes_the_moved_roots_to_python(): + """The Makefile, not only the step list, carries the roots (a dropped $(ROOT_ARGS) fails).""" + root = "reproduction/abc123def456/results" + for step in full_steps(LAYOUT): + if step.command[0] != "make": + continue + recipe = make_dry_run(*step.command[1:]) + python_lines = [line for line in recipe.splitlines() if "python -m tinyrouter" in line] + assert python_lines, step.name + for line in python_lines: + moved = f"--results-root {root}" in line or f"--results-dir {root}" in line + assert moved, (step.name, line) + if "CHECKPOINT_ROOT=reproduction/abc123def456/checkpoints" in step.command: + assert "--checkpoint-root reproduction/abc123def456/checkpoints" in line + if step.name == "report": + assert "--readme-out reproduction/abc123def456/README.md" in recipe + + +def test_without_roots_the_make_targets_keep_the_configs_paths(): + recipe = make_dry_run("ac2") + assert "--results-root" not in recipe and "--checkpoint-root" not in recipe + + +def test_a_reproduction_directory_pointing_into_results_is_refused(tmp_path): + (tmp_path / "results").mkdir() + (tmp_path / "reproduction").symlink_to(tmp_path / "results") + with pytest.raises(PathIsolationError): + check_isolation(LAYOUT, tmp_path) + + +def test_an_id_that_escapes_the_reproduction_directory_is_refused(): + for bad in ("", "../results", ".hidden", "a/b"): + with pytest.raises(PathIsolationError): + Layout(bad) + + +def test_a_normal_layout_passes_the_isolation_check(tmp_path): + (tmp_path / "results").mkdir() + check_isolation(LAYOUT, tmp_path) + + +STEP = Step("curve-bert", ("make", "curve"), ("completed 18/18 encoder points (bert)",)) + + +def test_a_step_passes_only_on_its_whole_completion_line_and_exit_0(): + good = ["training", "completed 18/18 encoder points (bert)"] + assert judge_step(STEP, 0, good)["status"] == PASS + assert judge_step(STEP, 1, good)["status"] == FAIL + assert judge_step(STEP, 0, ["completed 36/36 baseline points"])["status"] == FAIL + padded = ["completed 18/18 encoder points (bert) and more"] + assert judge_step(STEP, 0, padded)["status"] == FAIL + assert judge_step(STEP, 0, ["x completed 18/18 encoder points (bert)"])["status"] == FAIL + + +def test_a_step_with_several_completion_lines_needs_all_of_them(): + step = Step("report", ("make", "report"), ("wrote a", "wrote b")) + assert judge_step(step, 0, ["wrote a"])["status"] == FAIL + assert judge_step(step, 0, ["wrote b", "wrote a"])["status"] == PASS + + +class FakeExecutor: + """Prints each step's completion lines and writes its outputs, unless told to fail it.""" + + def __init__(self, repo: Path, fail: str | None = None, write: bool = True) -> None: + self.repo, self.fail, self.write, self.ran = repo, fail, write, [] + + def __call__(self, step: Step, log: Path) -> tuple[int, list[str]]: + self.ran.append(step.name) + if step.name == self.fail: + return 1, ["Traceback: boom"] + for path in step.outputs if self.write else (): + (self.repo / path).parent.mkdir(parents=True, exist_ok=True) + (self.repo / path).write_text(f"{step.name} output\n") + return 0, list(step.completion) + + +def three_steps() -> list[Step]: + return [ + Step(n, ("true",), (f"done {n}",), (f"out/{n}.txt",)) for n in ("first", "second", "third") + ] + + +def test_the_first_failing_step_stops_the_run_and_is_recorded(tmp_path): + executor = FakeExecutor(tmp_path, fail="second") + state = tmp_path / "steps.json" + results = run_steps(three_steps(), tmp_path, state, tmp_path / "logs", executor) + assert [r["status"] for r in results] == [PASS, FAIL] + assert executor.ran == ["first", "second"] + assert json.loads(state.read_text())["second"]["completion_missing"] == ["done second"] + + +def test_a_rerun_skips_passed_steps_whose_outputs_are_unchanged(tmp_path): + state = tmp_path / "steps.json" + run_steps(three_steps(), tmp_path, state, tmp_path / "logs", FakeExecutor(tmp_path, "third")) + executor = FakeExecutor(tmp_path) + results = run_steps(three_steps(), tmp_path, state, tmp_path / "logs", executor) + assert executor.ran == ["third"] + assert [r.get("resumed", False) for r in results] == [True, True, False] + assert all(r["status"] == PASS for r in results) + + +def test_a_changed_output_reruns_that_step_and_every_later_one(tmp_path): + state = tmp_path / "steps.json" + run_steps(three_steps(), tmp_path, state, tmp_path / "logs", FakeExecutor(tmp_path)) + (tmp_path / "out" / "second.txt").write_text("edited by hand\n") + executor = FakeExecutor(tmp_path) + run_steps(three_steps(), tmp_path, state, tmp_path / "logs", executor) + assert executor.ran == ["second", "third"] + + +def test_a_deleted_output_is_not_resumed(tmp_path): + state = tmp_path / "steps.json" + run_steps(three_steps(), tmp_path, state, tmp_path / "logs", FakeExecutor(tmp_path)) + (tmp_path / "out" / "first.txt").unlink() + executor = FakeExecutor(tmp_path) + run_steps(three_steps(), tmp_path, state, tmp_path / "logs", executor) + assert executor.ran == ["first", "second", "third"] + + +def probes( + dirty: str = "", + merged: int = 0, + key: bool = True, + free_gib: float = 50.0, + sync: int = 0, + head: str = "f" * 40, + status_code: int = 0, + head_code: int = 0, +) -> Probes: + def git(args: list[str]) -> tuple[int, str]: + if args[0] == "status": + return status_code, dirty + if args[0] == "rev-parse": + return head_code, "" if head_code else head + if args[0] == "merge-base": + return merged, "" + return 0, "" + + return Probes( + git=git, + free_bytes=lambda _: int(free_gib * reproduce.GIB), + key_present=lambda: key, + uv_sync=lambda: sync, + ) + + +def test_preflight_passes_on_a_clean_merged_synced_checkout_with_a_key_and_disk(tmp_path): + checks = preflight(tmp_path, probes()) + assert checks["problems"] == [] + assert checks["head"] == "f" * 40 + assert checks["api_key_present"] is True + + +@pytest.mark.parametrize( + ("change", "needle"), + [ + ({"dirty": " M src/x.py"}, "not clean"), + ({"merged": 1}, "not a commit merged"), + ({"key": False}, "ANTHROPIC_API_KEY"), + ({"free_gib": 8.0}, "GiB free"), + ({"sync": 1}, "uv.lock"), + ], +) +def test_preflight_names_each_problem(tmp_path, change, needle): + problems = preflight(tmp_path, probes(**change))["problems"] + assert len(problems) == 1 and needle in problems[0] + + +def test_preflight_never_writes_the_key_value(tmp_path, monkeypatch): + monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test-value-not-printed") + checks = preflight(tmp_path, Probes(git=probes().git, uv_sync=lambda: 0)) + assert "sk-test" not in json.dumps(checks) + + +def repo_files(path: Path) -> Path: + """What run_identity and the snapshot read: results/, README.md, uv.lock, configs/.""" + (path / "results").mkdir(exist_ok=True) + (path / "README.md").write_text("# x\n") + (path / "uv.lock").write_text("lock\n") + (path / "configs").mkdir(exist_ok=True) + (path / "configs" / "a.yaml").write_text("a: 1\n") + return path + + +def test_a_failed_preflight_runs_no_step_and_says_not_started(tmp_path): + repo_files(tmp_path) + executor = FakeExecutor(tmp_path) + verdict = reproduce_full(LAYOUT, tmp_path, probes(key=False), executor) + assert verdict == "NOT STARTED" + assert executor.ran == [] + written = json.loads((tmp_path / LAYOUT.base / "preflight.json").read_text()) + assert written["problems"] + + +def test_a_flow_with_every_line_but_no_real_outputs_fails_on_ac2(tmp_path, monkeypatch): + """Completion lines alone do not make a PASS: the comparison still needs AC2's JSON.""" + monkeypatch.setattr(reproduce, "pins", lambda repo: {}) + repo_files(tmp_path) + verdict = reproduce_full(LAYOUT, tmp_path, probes(), FakeExecutor(tmp_path, write=False)) + assert verdict == FAIL + body = json.loads((tmp_path / LAYOUT.base / "comparison.json").read_text()) + assert body["flow"]["passed"] is True + assert body["ac2"]["verdict"] == FAIL + + +def test_the_artifacts_flow_rebuilds_into_results_without_bench_cpu_and_ends_with_the_diff(): + steps = artifact_steps() + names = [s.name for s in steps] + assert names[0] == "release-download" and names[-1] == "originals-untouched" + assert "bench-cpu" not in names + assert steps[0].completion == ("verified 76/76 release files",) + for step in steps: + assert not any("RESULTS_ROOT" in part for part in step.command) + + +def test_check_originals_reports_a_change_to_results_or_readme(tmp_path): + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + (tmp_path / "results").mkdir() + (tmp_path / "results" / "a.json").write_text("{}\n") + (tmp_path / "README.md").write_text("# x\n") + (tmp_path / "other.txt").write_text("not checked\n") + env = {"GIT_AUTHOR_NAME": "t", "GIT_AUTHOR_EMAIL": "t@t", "GIT_COMMITTER_NAME": "t"} + env |= {"GIT_COMMITTER_EMAIL": "t@t", "PATH": os.environ["PATH"]} + subprocess.run(["git", "add", "."], cwd=tmp_path, check=True) + subprocess.run(["git", "commit", "-qm", "init"], cwd=tmp_path, check=True, env=env) + + def git(args): + done = subprocess.run(["git", *args], cwd=tmp_path, capture_output=True, text=True) + return done.returncode, done.stdout.strip() + + (tmp_path / "other.txt").write_text("changed\n") + assert reproduce.originals_changed(git) == "" + (tmp_path / "results" / "new.json").write_text("{}\n") + assert "results/new.json" in reproduce.originals_changed(git) + (tmp_path / "results" / "new.json").unlink() + (tmp_path / "README.md").write_text("# y\n") + assert "README.md" in reproduce.originals_changed(git) + + +@pytest.mark.parametrize( + ("change", "needle"), + [({"status_code": 128}, "git status failed"), ({"head_code": 128}, "rev-parse HEAD failed")], +) +def test_preflight_treats_a_failing_git_as_a_problem_not_as_clean(tmp_path, change, needle): + problems = preflight(tmp_path, probes(**change))["problems"] + assert any(needle in p for p in problems) + + +IDENTITY = { + "head": "a" * 40, + "uv_lock_sha256": "1" * 64, + "configs_sha256": "2" * 64, + "python": "3.12.12", + "torch": "2.14.0", + "transformers": "5.17.0", +} + + +def test_a_state_from_the_same_identity_resumes(tmp_path): + state = tmp_path / "steps.json" + run_steps(three_steps(), tmp_path, state, tmp_path / "logs", FakeExecutor(tmp_path), IDENTITY) + executor = FakeExecutor(tmp_path) + run_steps(three_steps(), tmp_path, state, tmp_path / "logs", executor, dict(IDENTITY)) + assert executor.ran == [] + + +@pytest.mark.parametrize( + "field", ["head", "uv_lock_sha256", "configs_sha256", "python", "torch", "transformers"] +) +def test_a_state_from_another_identity_is_refused_and_nothing_runs(tmp_path, field): + state = tmp_path / "steps.json" + run_steps(three_steps(), tmp_path, state, tmp_path / "logs", FakeExecutor(tmp_path), IDENTITY) + executor = FakeExecutor(tmp_path) + with pytest.raises(reproduce.StaleStateError, match=f"{field} differ.*new REPRO_ID"): + run_steps( + three_steps(), tmp_path, state, tmp_path / "logs", executor, {**IDENTITY, field: "x"} + ) + assert executor.ran == [] + + +def test_a_state_without_an_identity_is_refused(tmp_path): + state = tmp_path / "steps.json" + run_steps(three_steps(), tmp_path, state, tmp_path / "logs", FakeExecutor(tmp_path)) + with pytest.raises(reproduce.StaleStateError, match="no identity recorded"): + run_steps( + three_steps(), tmp_path, state, tmp_path / "logs", FakeExecutor(tmp_path), IDENTITY + ) + + +def test_run_identity_covers_head_lockfile_configs_and_libraries(tmp_path): + repo_files(tmp_path) + first = reproduce.run_identity(tmp_path, probes().git) + assert first["head"] == "f" * 40 and first["torch"] and first["python"] + (tmp_path / "configs" / "a.yaml").write_text("a: 2\n") + assert ( + reproduce.run_identity(tmp_path, probes().git)["configs_sha256"] != first["configs_sha256"] + ) + (tmp_path / "uv.lock").write_text("other\n") + assert ( + reproduce.run_identity(tmp_path, probes().git)["uv_lock_sha256"] != first["uv_lock_sha256"] + ) + + +def test_the_same_id_after_a_new_commit_is_refused_before_any_step(tmp_path, monkeypatch): + """The review's case: same REPRO_ID, one more (empty) commit, then make reproduce again.""" + monkeypatch.setattr(reproduce, "pins", lambda repo: {}) + repo_files(tmp_path) + reproduce_full(LAYOUT, tmp_path, probes(head="a" * 40), FakeExecutor(tmp_path, write=False)) + first = json.loads((tmp_path / LAYOUT.base / "steps.json").read_text()) + assert {s["identity"]["head"] for s in first.values()} == {"a" * 40} + executor = FakeExecutor(tmp_path) + verdict = reproduce_full(LAYOUT, tmp_path, probes(head="b" * 40), executor) + assert verdict == "NOT STARTED" + assert executor.ran == [] + problems = json.loads((tmp_path / LAYOUT.base / "preflight.json").read_text())["problems"] + assert "head differ" in problems[0] and "new REPRO_ID" in problems[0] + + +def test_check_originals_with_a_snapshot_sees_gitignored_files(tmp_path): + repo_files(tmp_path) + (tmp_path / "results" / "logits").mkdir() + (tmp_path / "results" / "logits" / "a.npz").write_bytes(b"archive") + clean = probes().git + snapshot = tmp_path / "snap.json" + snapshot.write_text(json.dumps(reproduce.snapshot_originals(tmp_path))) + assert reproduce.check_originals(tmp_path, snapshot, clean) == "" + (tmp_path / "results" / "logits" / "a.npz").write_bytes(b"overwritten") + (tmp_path / "results" / "logits" / "b.npz").write_bytes(b"new") + changed = reproduce.check_originals(tmp_path, snapshot, clean) + assert "changed results/logits/a.npz" in changed and "added results/logits/b.npz" in changed + (tmp_path / "README.md").write_text("# y\n") + assert "changed README.md" in reproduce.check_originals(tmp_path, snapshot, clean) + + +def test_both_flows_end_with_a_snapshot_check(): + full = full_steps(LAYOUT)[-1] + assert full.command[-2:] == ("--snapshot", "reproduction/abc123def456/originals-snapshot.json") + names = [s.name for s in artifact_steps()] + assert names.index("snapshot-originals") == names.index("verify-llm") + 1 + assert "--snapshot" in artifact_steps()[-1].command