Step 3 prep: k-shot sampling, validation-only pilots, curve runner, baselines, ModernBERT trial

.github/disallowed-trailers.txt

@@ -7,3 +7,7 @@
 co-authored-by:.*(anthropic|claude|openai|chatgpt|copilot|gemini|cursor)
 generated (with|by) \[?(claude|chatgpt|copilot|gemini|cursor)
 noreply@anthropic\.com
+# Canary: exercises the checks (for example in a throwaway draft PR)
+# without writing a real attribution line that would then stay in
+# GitHub's edit history.
+tinyrouter-hygiene-canary

.github/scripts/check-disallowed-text.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+# Reads text on stdin and fails if any line matches a pattern in
+# .github/disallowed-trailers.txt (extended regex, case-insensitive).
+# The one definition shared by the commit-hygiene job (ci.yml) and the
+# pr-text-hygiene job (pr-text.yml), so the two cannot drift apart.
+# Usage: <text> | check-disallowed-text.sh <what is being checked>
+# Exit: 0 clean, 1 a line matched, 2 the pattern file has no patterns.
+set -euo pipefail
+
+label="${1:-text}"
+patterns_file="${PATTERNS_FILE:-.github/disallowed-trailers.txt}"
+patterns="$(grep -vE '^[[:space:]]*(#|$)' "$patterns_file" | paste -sd'|' - || true)"
+if [ -z "$patterns" ]; then
+  echo "::error::$patterns_file has no patterns"
+  exit 2
+fi
+bad="$(grep -iE "$patterns" || true)"
+if [ -n "$bad" ]; then
+  echo "::error::$label must not carry tool-attribution lines:"
+  printf '%s\n' "$bad"
+  exit 1
+fi
+echo "clean: $label"

.github/workflows/ci.yml

@@ -87,7 +87,8 @@ jobs:
         # checkout on pull_request lands on GitHub's synthetic merge commit).
         # Pushes to main: the pushed range, which is where a squash merge
         # message edited in the web UI shows up. Patterns live in
-        # .github/disallowed-trailers.txt.
+        # .github/disallowed-trailers.txt; the check itself is
+        # .github/scripts/check-disallowed-text.sh, shared with pr-text.yml.
         env:
           EVENT: ${{ github.event_name }}
           PR_BASE: ${{ github.event.pull_request.base.sha }}
@@ -102,17 +103,8 @@ jobs:
           else
             range="$PUSH_BEFORE..$PUSH_AFTER"
           fi
-          patterns="$(grep -vE '^[[:space:]]*(#|$)' .github/disallowed-trailers.txt | paste -sd'|' -)"
-          if [ -z "$patterns" ]; then
-            echo "::error::.github/disallowed-trailers.txt has no patterns"
-            exit 1
-          fi
-          bad="$(git log "$range" --format='%h %s%n%b' | grep -iE "$patterns" || true)"
-          if [ -n "$bad" ]; then
-            echo "::error::commit messages must not carry tool-attribution trailers:"
-            echo "$bad"
-            exit 1
-          fi
+          git log "$range" --format='%h %s%n%b' \
+            | .github/scripts/check-disallowed-text.sh "commit messages in $range"
           echo "clean: $(git rev-list --count "$range") commit(s) in $range"
 
   network:

.github/workflows/pr-text.yml

@@ -0,0 +1,43 @@
+name: PR text
+
+# main only takes squash merges, and a squash commit's title and message
+# come from the PR title and body, not from the branch's commits. The
+# commit-hygiene job in ci.yml checks the branch's commits only, so this
+# job checks the PR title and body with the same patterns
+# (.github/disallowed-trailers.txt via .github/scripts/check-disallowed-text.sh).
+#
+# A separate workflow because it runs on `edited`: editing a PR description
+# must re-run this check without cancelling or re-running ci.yml's jobs.
+#
+# Not covered: text typed into the merge dialog by hand. No PR event carries
+# it; the push-to-main run of commit-hygiene sees it, after the fact.
+on:
+  pull_request:
+    types: [opened, edited, synchronize, reopened]
+
+concurrency:
+  group: pr-text-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  pr-text-hygiene:
+    name: pr-text-hygiene
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - name: Checkout (for the pattern file and the check script)
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - name: No tool-attribution lines in the PR title or body
+        # Title and body are attacker-controlled text. They reach the shell
+        # only as environment variables, never through ${{ }} inside `run`,
+        # so nothing in them can be executed.
+        env:
+          PR_TITLE: ${{ github.event.pull_request.title }}
+          PR_BODY: ${{ github.event.pull_request.body }}
+        run: |
+          printf '%s\n%s\n' "$PR_TITLE" "$PR_BODY" \
+            | .github/scripts/check-disallowed-text.sh "PR title and body"

DEVLOG.md

@@ -4,6 +4,82 @@
 
 ---
 
+## 2026-09-23（夜）：PR #7 審查修正
+
+### 本次工作 / 執行摘要
+- 新增 `pr-text.yml`（job `pr-text-hygiene`）：squash merge 的 commit 取自 PR 標題與內文，原本的 commit-hygiene 看不到。標題與內文只經 env 傳入。兩個 job 共用 `.github/scripts/check-disallowed-text.sh` 與同一份 pattern 檔。
+- pilot 重用 AC2 改為 validation-only：archive 整檔照算 SHA-256，但只讀 metadata 與 validation 陣列；結果 JSON 只取 `run_name`、`config`、`training`、`logits`、`environment`，不含 `metrics`。
+- 權重刪除前先確認結果 JSON、manifest、archive 三處一致；曲線索引對非重用點也驗 archive。
+- 曲線索引逐點比對「訓練時記下的抽樣指紋」與「現在重抽的 `curve_sample(k, seed)`」，numpy 升版造成樣本改變時會報錯；AC2 重用點補上現算的指紋並標註來源。測試裡另釘了 5 組抽樣指紋。
+- 等價判斷新增：torch 與 transformers 版本必須與目前安裝的相同（忽略 `+cpu` 這類 build 標籤）；schedule 層補獨立測試。
+- TF-IDF 的 100 倍：修正 docstring，PLAN §4.1 註明 RQ3 對 TF-IDF 只報溫度校準後的訊號與 margin，多數類排除在 RQ3 外。
+- scipy 列入 dev group。
+
+### 核心發現 / 數據
+- 審查實測（seed 42 val，k=100）：TF-IDF 未校準 MSP 的 AURC 在純餘弦為 0.0854、100 倍為 0.0533；entropy 為 0.1418 與 0.0539。倍數確實改變未校準訊號，所以 RQ3 不用它們。
+- 關於 test 洩漏：Drew 在 PR #6（AC2 結果）看過 BERT lr 5e-5 的 test 數字。lr pilot 的選擇規則是機械式的（只比 validation 的正確筆數，再比 validation OOS 正確筆數，再取較小 lr），看過 test 數字不會影響它選出什麼，所以不構成洩漏。
+
+### Blockers / 遇到的問題
+- (無)
+
+### Next
+- [ ] PR #7 合併後，由協調者把 `pr-text-hygiene` 加進 main 的 required checks
+- [ ] 其餘同上一則
+
+### Files / Budget
+- `.github/workflows/pr-text.yml`、`.github/scripts/check-disallowed-text.sh`、`tests/test_text_hygiene.py`
+- API 花費：US$0
+
+---
+
+## 2026-09-23（晚）：步驟 3 準備，抽樣、步數協定、pilot、執行器與 ModernBERT 相容性試跑
+
+### 本次工作 / 執行摘要
+- 超參數協定寫進 `docs/PLAN.md` §4.1：步數 = max(S_min, 5 epoch 步數)，S_min 與各 encoder 的 learning rate 都只用 validation pilot 選，選定值手動填進 `configs/curve.yaml`。
+- 新模組：`sampling.py`（k-shot 抽樣，OOS 用寫死對照表）、`steps.py`（步數規劃，訓練後核對實際步數）、`runs.py`（續跑規則與「等價 run」判斷）、`protocol.py`、`pilots.py`、`curves.py`、`baselines.py`；`metrics.wilson_interval`。
+- logits archive 升到 format 2：加三個 split parquet 的 SHA-256；讀 format 1（AC2）時只在 dataset revision 等於鎖定值才從 `SPLIT_FILES` 補上並標註來源，不符就報錯。
+- AC2 重用：BERT 若選中 lr 5e-5，曲線的 k=100 三個點與 lr pilot 的 5e-5 點都重用 AC2，前提是逐欄設定、訓練列數、步數與 seed 都對得上，且 archive 的 SHA-256 三處一致；任何一項不符就重訓。對照已提交的三個 AC2 結果 JSON 測過，S_min 取 100、200、400 或不設都判定等價。
+- 基準：多數類與 TF-IDF centroid 在每個 (k, seed) 的同一份抽樣上計算，存成與 encoder 相同的 archive 與結果 JSON。
+- ModernBERT 相容性試跑（`scripts/compat_trial.py`、`scripts/export_onnx.py`），結果在 `results/compat/`。只跑 50 步與一次 validation，不是正式 pilot 或曲線。
+
+### 核心發現 / 數據
+**ModernBERT 在 M4 MPS 上可以訓練。** 兩個模型同設定：batch 32、max_length 64、50 步、lr 5e-5（試跑用佔位值）、seed 42。
+
+| | ModernBERT-base | bert-base-uncased |
+|---|---:|---:|
+| 參數量 | 149.7M | 109.6M |
+| 每步時間（中位數，去掉前 5 步） | 0.234 s | 0.139 s |
+| 第一步 | 1.13 s | 0.51 s |
+| 峰值記憶體（Metal driver，採樣） | 4.80 GiB | 3.18 GiB |
+| 峰值記憶體（存活 tensor，採樣） | 2.30 GiB | 1.66 GiB |
+| loss（前 5 步 → 後 5 步） | 5.22 → 4.22 | 5.08 → 5.00 |
+| NaN | 無 | 無 |
+
+- attention 走 `sdpa`（MPS 上沒有 Flash Attention 與 unpadding），沒有報錯或退回警告。
+- 每步時間 ModernBERT 約為 BERT 的 1.69 倍。
+- **全量 5 epoch（2,385 步）預估：** 直接乘是 559 s，但同法估 BERT 得 332 s，而 AC2 實測是 822 到 886 s（每步約 0.36 s），短試跑低估約 2.6 倍（原因未查：可能是長時間執行的降頻、每個 epoch 存檔與每步記憶體採樣）。用 AC2 實測乘上 1.69 倍，**ModernBERT 全量每個 seed 約 24 分鐘**，以這個數字排時程。
+- 粗估整條曲線（S_min 以 400 計，每個 seed 約 5,400 步）：ModernBERT 18 點約 2.7 小時，OOS 消融 3 次約 1.2 小時；BERT 若重用 AC2，剩 15 點約 1 小時。
+- **ONNX 匯出成功，兩種 exporter 都可以。** `torch.onnx.export`（dynamo 與 TorchScript）加 ONNX Runtime 1.30 CPU，10 筆 validation（padding 到 14）最大絕對 logit 誤差 2.1e-5，argmax 10/10 一致；另換一批 3 筆、長度 25 的輸入也只差 1.1e-5，動態 batch 與長度可用。模型檔約 600 MB。TorchScript exporter 有 TracerWarning（masking 相關），實測沒出錯，但 RQ7 以 dynamo 為主。int8 量化還沒試，留給步驟 6。**RQ7 可以維持用 ModernBERT。**
+- TF-IDF centroid 若直接把餘弦（0 到 1）當 logit，validation 的溫度擬合在 k ≥ 5 撞到搜尋下界 T = 0.05；改用 CLIP 慣例的 100 倍餘弦，T 落在 2.9 到 7.2 之間。這是 seed 42、43 在 scratch 目錄的檢查，不是正式基準結果。
+
+### Blockers / 遇到的問題
+- 磁碟剩約 11 GB。試跑權重與兩個 ONNX 檔（共 1.7 GB）已刪除。
+- 短試跑的每步時間與長時間訓練差很多（見上），時程估計以 AC2 實測為準。
+- pilot 重用 AC2 時，判斷等價要讀 AC2 結果 JSON 的 `config` 與 `training` 欄位；那個檔案裡也有 test 數字，但程式只取這兩欄，validation 數字從 archive 只讀 validation 陣列。
+
+### Next
+- [ ] `make pilot-lr`（ModernBERT 3 次 + BERT 2 次，約 1.7 小時），把選定值填進 `configs/curve.yaml`
+- [ ] `make pilot-steps`（6 次 k=5 短訓練），填 S_min
+- [ ] `make curve MODEL=bert`、`make curve MODEL=modernbert`、`make oos-ablation`
+- [ ] logits 檔附到 GitHub Release
+
+### Files / Budget
+- 新增：`src/tinyrouter/{sampling,steps,runs,protocol,pilots,curves,baselines}.py`、`configs/{modernbert-base,curve}.yaml`、`scripts/{compat_trial,export_onnx}.py`、`results/compat/*.json`
+- 相依：optional group `onnx`（onnx 1.23.0、onnxruntime 1.30.0、onnxscript 0.7.2），CI 不安裝
+- API 花費：US$0
+
+---
+
 ## 2026-09-23：AC2 通過（BERT 流程正確性檢查）
 
 ### 本次工作 / 執行摘要

Makefile

@@ -1,4 +1,5 @@
-.PHONY: setup lint format test test-network smoke train evaluate ac2 verify-logits report clean-checkpoints
+.PHONY: setup lint format test test-network smoke train evaluate ac2 pilot-lr pilot-steps baselines \
+	curve oos-ablation verify-logits report clean-checkpoints
 
 CONFIG ?= configs/bert-base.yaml
 SEED ?= 42
@@ -64,6 +65,32 @@ evaluate:
 ac2:
 	uv run $(UV_ENV) python -m tinyrouter.ac2 --config configs/bert-base.yaml $(if $(filter 1,$(FORCE)),--force,)
 
+# Step 3 (docs/PLAN.md section 4, hyperparameter protocol). Order:
+#   make pilot-lr     then copy each model's selected lr into configs/curve.yaml
+#   make pilot-steps  then copy the selected S_min into configs/curve.yaml
+#   make curve MODEL=bert ; make curve MODEL=modernbert ; make oos-ablation
+# Pilots read and write validation numbers only (results/pilots/*.json) and
+# never edit configs/curve.yaml. Curves and the ablation resume like ac2 and
+# keep no weights, only logits. `curve` runs the cheap baselines first.
+# `make baselines` runs them alone.
+pilot-lr:
+	uv run $(UV_ENV) python -m tinyrouter.pilots lr
+
+pilot-steps:
+	uv run $(UV_ENV) python -m tinyrouter.pilots steps
+
+# Majority-class and TF-IDF centroid baselines on every (k, seed) sample; seconds.
+baselines:
+	uv run python -m tinyrouter.baselines
+
+# Checks configs/curve.yaml is filled in, then runs the baselines, then the curve.
+curve:
+	@case "$(MODEL)" in bert|modernbert) ;; *) echo "usage: make curve MODEL=bert|modernbert"; exit 2;; esac
+	uv run $(UV_ENV) python -m tinyrouter.curves --model $(MODEL)
+
+oos-ablation:
+	uv run $(UV_ENV) python -m tinyrouter.curves --ablation
+
 # Every archive listed in results/logits-manifest.json is present and matches its SHA-256.
 verify-logits:
 	uv run python -m tinyrouter.archive

README.md

@@ -29,6 +29,11 @@ Other targets:
 | `make train CONFIG=configs/bert-base.yaml SEED=42` | fine-tune; final weights in `checkpoints/<run>/final` |
 | `make evaluate CONFIG=... SEED=...` | score validation and test, write `results/runs/<run>.json` and the logits archive |
 | `make ac2` | bert-base-uncased, full data, seeds 42/43/44; writes `results/ac2.json`, exits 1 on FAIL; resumes only runs made with the current config, `FORCE=1` clears and reruns all three, including deleting seed 42's kept weights; after `make clean-checkpoints` only `FORCE=1` brings seed 42's weights back |
+| `make pilot-lr` | validation-only pilot: each encoder at k=100, seed 42, lr in {1e-5, 2e-5, 5e-5}; writes `results/pilots/lr.json` and prints the choice (BERT at 5e-5 reuses AC2 seed 42 when it is the same run) |
+| `make pilot-steps` | validation-only pilot: both encoders at k=5, seed 42, S_min in {100, 200, 400}; writes `results/pilots/steps.json`. Needs the learning rates in `configs/curve.yaml` |
+| `make baselines` | majority-class and TF-IDF centroid baselines on every (k, seed) sample, archived like an encoder run |
+| `make curve MODEL=bert\|modernbert` | baselines, then 6 values of k x 3 seeds with the lr and S_min from `configs/curve.yaml`; writes `results/curves/<model>.json`; keeps no weights |
+| `make oos-ablation` | ModernBERT, k=100, no OOS training rows, 3 seeds; writes `results/curves/oos-ablation.json` |
 | `make verify-logits` | check every archive in `results/logits/` against `results/logits-manifest.json` |
 | `make report` | build `results/summary.md` from `results/runs/*.json` |
 | `make clean-checkpoints` | delete all trained weights |
@@ -44,17 +49,19 @@ Put `HF_TOKEN` and `ANTHROPIC_API_KEY` in a `.env` (copy `.env.example`) if you
 - **Labels.** The model is trained on all 151 CLINC150 intents (150 plus `oos`). Each intent maps to one of 8 routing targets (7 agents plus `oos`) through `src/tinyrouter/resources/intent_to_agent.json`.
 - **Data.** `clinc/clinc_oos`, `plus` config, from the Hugging Face Hub at a pinned commit. Each parquet file is checked against its SHA-256 and row count (15,250 / 3,100 / 5,500), and the label names inside it must match the committed `intent_names.json`.
 - **Calibration.** Temperature scaling, fitted on validation logits only. `fit_temperature` raises `LeakageError` if given anything else, and a test checks that.
-- **Logits archive.** Every evaluation stores per-example validation and test logits (float32) with gold labels and metadata (model and revision, seed, k, OOS training rows, dataset revision, label-space hash, git commit, time) in `results/logits/<run>.npz`. Later analysis reads only these files. They are not committed (about 5 MB per bert-base run); their SHA-256 goes into the committed `results/logits-manifest.json`, and the files are attached to a GitHub Release.
+- **Logits archive.** Every evaluation stores per-example validation and test logits (float32) with gold labels and metadata (model and revision, seed, k, OOS training rows, dataset revision and the SHA-256 of the three split files, label-space hash, git commit, time) in `results/logits/<run>.npz`. Later analysis reads only these files. They are not committed (about 5 MB per bert-base run); their SHA-256 goes into the committed `results/logits-manifest.json`, and the files are attached to a GitHub Release.
 - **Training cost.** Each results JSON records parameter counts, training wall time, steps, device and peak memory. How memory is measured depends on the device (sampled Metal driver memory on MPS, the allocator peak on CUDA, process peak RSS on CPU) and is written next to the number; see `src/tinyrouter/efficiency.py`.
-- **Metrics.** 8-way accuracy, 150-way in-scope accuracy and OOS recall (as defined in Larson et al. 2019), ECE, and NLL. They are computed before and after calibration on both splits.
+- **Learning-curve protocol.** Each curve point trains for max(S_min, 5 epochs) steps. The learning rate of each encoder and the one S_min are chosen by validation-only pilots (`docs/PLAN.md` section 4.1) and written into `configs/curve.yaml` by hand. The pilot code raises `LeakageError` if handed the test split.
+- **Metrics.** 8-way accuracy, 150-way in-scope accuracy and OOS recall (as defined in Larson et al. 2019), ECE, and NLL. They are computed before and after calibration on both splits. `wilson_interval` gives the confidence interval for OOS recall.
 
 ## Layout
 
 ```
-configs/            run configs (bert-base.yaml, smoke.yaml); unknown keys are an error
+configs/            run configs (bert-base.yaml, modernbert-base.yaml, smoke.yaml; unknown keys
+                    are an error) and curve.yaml, the pilot-chosen lr and S_min
 docs/PLAN.md        research questions and acceptance criteria
 docs/OPERATIONS.md  CI jobs, branch protection, rollback
-scripts/            check_em_dash.py
+scripts/            check_em_dash.py; compat_trial.py and export_onnx.py (encoder trial runs)
 src/tinyrouter/
   data.py           pinned download, checksum, per-intent subsampling
   labels.py         151 intents <-> 8 routing targets
@@ -64,6 +71,13 @@ src/tinyrouter/
   evaluate.py       logits, temperature fit, metrics, results JSON
   archive.py        per-example logits archive (.npz) and SHA-256 manifest
   efficiency.py     parameter counts, wall time, peak memory per device
+  sampling.py       k-shot training samples (k per intent, ceil(2.5k) oos from a fixed table)
+  steps.py          training steps = max(S_min, epoch steps), checked after training
+  runs.py           resume rules and when an existing run counts as the same run
+  protocol.py       configs/curve.yaml: pilot-chosen lr per encoder and S_min
+  pilots.py         validation-only lr and S_min pilots
+  curves.py         learning curves and the OOS ablation (reuses AC2 at k=100 when equivalent)
+  baselines.py      majority-class and TF-IDF centroid baselines per curve point
   ac2.py            AC2 run over three seeds and PASS/FAIL verdict
   llm.py            Claude Haiku zero-shot router (baseline and fallback; not run yet)
   report.py         results/*.json -> results/summary.md

configs/curve.yaml

@@ -0,0 +1,15 @@
+# Learning-curve hyperparameters (docs/PLAN.md section 4, hyperparameter
+# protocol). Filled in by hand from the pilot outputs:
+#   make pilot-lr     -> results/pilots/lr.json    -> models.<model>.learning_rate
+#   make pilot-steps  -> results/pilots/steps.json -> min_train_steps
+# null means that pilot has not been run; `make curve` refuses to start.
+# Allowed values: learning_rate in {1.0e-5, 2.0e-5, 5.0e-5} (write the
+# ".0": YAML reads 5e-5 as a string), min_train_steps in {100, 200, 400}.
+min_train_steps: null
+models:
+  bert:
+    config: configs/bert-base.yaml
+    learning_rate: null
+  modernbert:
+    config: configs/modernbert-base.yaml
+    learning_rate: null