drewOrc · Sep 23, 2026 · Sep 23, 2026
diff --git a/DEVLOG.md b/DEVLOG.md
@@ -0,0 +1,38 @@
+# DEVLOG: TinyRouter
+
+實驗進度與觀察。最新條目在最上面。
+
+---
+
+## 2026-09-23：AC2 通過（BERT 流程正確性檢查）
+
+### 本次工作 / 執行摘要
+- `make ac2`：`bert-base-uncased` 在完整 OOS+ 訓練集（15,000 in-scope + 250 OOS）訓練 151 類，seeds 42 / 43 / 44，commit `b3a59e2`，工作目錄乾淨（`git_dirty: false`）。
+- 超參數是骨架的起始值，未調參：lr 5e-5、5 epochs、batch 32、max_length 64、warmup 10%、weight decay 0.01。
+- 開跑前 PR #5 經兩輪審查：續跑比對 config、三處 sha 一致才算完成、判定驗 seed 身分、過期的 `ac2.json` 開跑即刪。
+
+### 核心發現 / 數據
+| seed | val in-scope | val OOS recall | test in-scope | test OOS recall | 訓練時間 |
+|---:|---:|---:|---:|---:|---:|
+| 42 | 96.73% | 68.0% | 96.51% | 58.3% | 886 s |
+| 43 | 96.73% | 69.0% | 96.40% | 55.8% | 847 s |
+| 44 | 96.73% | 73.0% | 96.22% | 56.7% | 822 s |
+
+- AC2 門檻 95.7%（test in-scope，三個 seed 都要過）：**PASS**。test in-scope 平均 96.38%；Larson et al. 2019（BERT、OOS+、oos-train）為 96.7%，OOS recall 59.2%，本次 56.9%。這是工程驗收，不宣稱重現原論文。
+- 三個 seed 的 val in-scope 都是 2,902 / 3,000。已從 logits 檔直接重算確認不是同一份結果：三個檔 SHA-256 不同，val 上約 2.5% 的預測彼此不同，test 數字也不同；判定為巧合。
+- **OOS 是弱點，符合預期**：in-scope 96%，但 test 上約 43% 的 OOS 被分進某個 intent。這正是 RQ2 到 RQ4 要處理的問題。
+- 效率（M4、MPS）：參數 109.6M；每個 seed 約 14 分鐘（2,385 步）；峰值記憶體 driver 約 4.6 GB、存活 tensor 約 1.8 GB（MPS 沒有原生峰值 API，採樣值，量法見結果檔）。
+
+### Blockers / 遇到的問題
+- 磁碟可用空間從 15 GB 降到 11 GB。主要佔用是 uv 全域快取（33 GB，全機共用，非本專案）與 HF 模型快取（2.3 GB）；建議 Drew 執行 `uv cache prune`，本專案不代為刪除。
+- 背景執行時包了一層 `echo exit`，所以背景任務的離開碼永遠是 0，不能拿來判斷成功與否；要看 log 裡 `make` 的輸出與 `results/ac2.json`。
+
+### Next
+- [ ] logits 檔附到 GitHub Release（manifest 已記 SHA-256）
+- [ ] ModernBERT 相容性試跑（MPS 訓練、Optimum ONNX 匯出）
+- [ ] k-shot 抽樣模組（每個 intent k 筆、OOS ⌈2.5k⌉ 筆）與兩條學習曲線
+
+### Files / Budget
+- `results/ac2.json`、`results/runs/bert-base-uncased-full-seed{42,43,44}.json`、`results/logits-manifest.json`
+- logits 檔（不進 git）：`results/logits/bert-base-uncased-full-seed{42,43,44}.npz`，各約 4.8 MB
+- API 花費：US$0
diff --git a/results/ac2.json b/results/ac2.json
@@ -0,0 +1,81 @@
+{
+  "criterion": "AC2: test in_scope_accuracy_150 >= threshold for every seed",
+  "split": "test",
+  "threshold": 0.957,
+  "tuning_note": "tune on the validation numbers only; test is the verdict",
+  "seeds": {
+    "42": {
+      "run_name": "bert-base-uncased-full-seed42",
+      "logits_sha256": "673a4c11200dd61ddd6310f174ab9fb27580cf5b512c7ba3b5db19088958bab2",
+      "validation": {
+        "in_scope_accuracy_150": 0.9673333333333334,
+        "oos_recall_151": 0.68,
+        "accuracy_8": 0.9780645161290322,
+        "oos_recall_8": 0.68
+      },
+      "test": {
+        "in_scope_accuracy_150": 0.9651111111111111,
+        "oos_recall_151": 0.583,
+        "accuracy_8": 0.9158181818181819,
+        "oos_recall_8": 0.583
+      },
+      "passed": true
+    },
+    "43": {
+      "run_name": "bert-base-uncased-full-seed43",
+      "logits_sha256": "b6963da29f5886b915d2c5626391721fbd18c7f54a68b7720bf2d10d6cc2c58f",
+      "validation": {
+        "in_scope_accuracy_150": 0.9673333333333334,
+        "oos_recall_151": 0.69,
+        "accuracy_8": 0.9761290322580645,
+        "oos_recall_8": 0.69
+      },
+      "test": {
+        "in_scope_accuracy_150": 0.964,
+        "oos_recall_151": 0.558,
+        "accuracy_8": 0.9098181818181819,
+        "oos_recall_8": 0.558
+      },
+      "passed": true
+    },
+    "44": {
+      "run_name": "bert-base-uncased-full-seed44",
+      "logits_sha256": "0324b20d4736ca9ad913b97269100b398475ab06a2bd130f653af55de239c277",
+      "validation": {
+        "in_scope_accuracy_150": 0.9673333333333334,
+        "oos_recall_151": 0.73,
+        "accuracy_8": 0.9790322580645161,
+        "oos_recall_8": 0.73
+      },
+      "test": {
+        "in_scope_accuracy_150": 0.9622222222222222,
+        "oos_recall_151": 0.567,
+        "accuracy_8": 0.9105454545454545,
+        "oos_recall_8": 0.567
+      },
+      "passed": true
+    }
+  },
+  "verdict": "PASS",
+  "config": {
+    "model_name": "google-bert/bert-base-uncased",
+    "model_revision": "86b5e0934494bd15c9632b12f734a8a67f723594",
+    "per_intent": null,
+    "max_length": 64,
+    "learning_rate": 5e-05,
+    "weight_decay": 0.01,
+    "warmup_ratio": 0.1,
+    "num_train_epochs": 5,
+    "max_steps": -1,
+    "train_batch_size": 32,
+    "eval_batch_size": 128,
+    "device": "auto",
+    "replace_classifier_head": false,
+    "eval_per_intent": null
+  },
+  "weights_kept": {
+    "42": true,
+    "43": false,
+    "44": false
+  }
+}
diff --git a/results/logits-manifest.json b/results/logits-manifest.json
@@ -0,0 +1,38 @@
+{
+  "format_version": 1,
+  "files": {
+    "bert-base-uncased-full-seed42.npz": {
+      "sha256": "673a4c11200dd61ddd6310f174ab9fb27580cf5b512c7ba3b5db19088958bab2",
+      "bytes": 4824207,
+      "run_name": "bert-base-uncased-full-seed42",
+      "git_commit": "b3a59e2debe8bf210ba749ec6d34033b9be6cab8",
+      "created_at": "2026-09-23T03:37:16+00:00",
+      "rows": {
+        "validation": 3100,
+        "test": 5500
+      }
+    },
+    "bert-base-uncased-full-seed43.npz": {
+      "sha256": "b6963da29f5886b915d2c5626391721fbd18c7f54a68b7720bf2d10d6cc2c58f",
+      "bytes": 4822394,
+      "run_name": "bert-base-uncased-full-seed43",
+      "git_commit": "b3a59e2debe8bf210ba749ec6d34033b9be6cab8",
+      "created_at": "2026-09-23T03:51:50+00:00",
+      "rows": {
+        "validation": 3100,
+        "test": 5500
+      }
+    },
+    "bert-base-uncased-full-seed44.npz": {
+      "sha256": "0324b20d4736ca9ad913b97269100b398475ab06a2bd130f653af55de239c277",
+      "bytes": 4823526,
+      "run_name": "bert-base-uncased-full-seed44",
+      "git_commit": "b3a59e2debe8bf210ba749ec6d34033b9be6cab8",
+      "created_at": "2026-09-23T04:05:58+00:00",
+      "rows": {
+        "validation": 3100,
+        "test": 5500
+      }
+    }
+  }
+}
diff --git a/results/runs/bert-base-uncased-full-seed42.json b/results/runs/bert-base-uncased-full-seed42.json
@@ -0,0 +1,135 @@
+{
+  "run_name": "bert-base-uncased-full-seed42",
+  "config": {
+    "model_name": "google-bert/bert-base-uncased",
+    "model_revision": "86b5e0934494bd15c9632b12f734a8a67f723594",
+    "seed": 42,
+    "per_intent": null,
+    "max_length": 64,
+    "learning_rate": 5e-05,
+    "weight_decay": 0.01,
+    "warmup_ratio": 0.1,
+    "num_train_epochs": 5,
+    "max_steps": -1,
+    "train_batch_size": 32,
+    "eval_batch_size": 128,
+    "device": "auto",
+    "replace_classifier_head": false,
+    "checkpoint_root": "checkpoints",
+    "results_root": "results",
+    "eval_per_intent": null
+  },
+  "environment": {
+    "python": "3.12.12",
+    "torch": "2.14.0",
+    "transformers": "5.17.0",
+    "platform": "macOS-26.6.2-arm64-arm-64bit",
+    "device": "mps",
+    "device_name": "Apple M4",
+    "deterministic_algorithms": false,
+    "dataset_revision": "155b9c710419136e17307b80d0a13e68cd46b4ec"
+  },
+  "training": {
+    "run_name": "bert-base-uncased-full-seed42",
+    "seed": 42,
+    "config": {
+      "model_name": "google-bert/bert-base-uncased",
+      "model_revision": "86b5e0934494bd15c9632b12f734a8a67f723594",
+      "seed": 42,
+      "per_intent": null,
+      "max_length": 64,
+      "learning_rate": 5e-05,
+      "weight_decay": 0.01,
+      "warmup_ratio": 0.1,
+      "num_train_epochs": 5,
+      "max_steps": -1,
+      "train_batch_size": 32,
+      "eval_batch_size": 128,
+      "device": "auto",
+      "replace_classifier_head": false,
+      "checkpoint_root": "checkpoints",
+      "results_root": "results",
+      "eval_per_intent": null
+    },
+    "git_commit": "b3a59e2debe8bf210ba749ec6d34033b9be6cab8",
+    "git_dirty": false,
+    "device": "mps",
+    "train_rows": 15250,
+    "oos_train_rows": 250,
+    "per_intent": null,
+    "global_step": 2385,
+    "train_loss": 0.8498310221066265,
+    "parameters": {
+      "total": 109598359,
+      "trainable": 109598359
+    },
+    "train_wall_seconds": 886.184,
+    "peak_memory": {
+      "device": "mps",
+      "measures": "sampled after each backward and optimizer step: torch.mps.driver_allocated_memory (Metal driver, includes allocator cache; headline, usually at or above the tensor peak but not guaranteed) and current_allocated_memory (live tensors; lower bound)",
+      "samples": 4772,
+      "driver_allocated_bytes": 4607033344,
+      "tensor_allocated_bytes": 1791083008
+    }
+  },
+  "logits": {
+    "file": "bert-base-uncased-full-seed42.npz",
+    "sha256": "673a4c11200dd61ddd6310f174ab9fb27580cf5b512c7ba3b5db19088958bab2",
+    "bytes": 4824207
+  },
+  "metrics": {
+    "temperature": 1.0109612433324868,
+    "validation": {
+      "raw": {
+        "accuracy_8": 0.9780645161290322,
+        "oos_recall_8": 0.68,
+        "accuracy_8_summed": 0.9764516129032258,
+        "oos_recall_8_summed": 0.64,
+        "in_scope_accuracy_150": 0.9673333333333334,
+        "oos_recall_151": 0.68,
+        "ece_151": 0.013230313917851991,
+        "ece_8_summed": 0.00762503365895657,
+        "nll_151": 0.2252614577481346,
+        "n": 3100
+      },
+      "calibrated": {
+        "accuracy_8": 0.9780645161290322,
+        "oos_recall_8": 0.68,
+        "accuracy_8_summed": 0.9764516129032258,
+        "oos_recall_8_summed": 0.64,
+        "in_scope_accuracy_150": 0.9673333333333334,
+        "oos_recall_151": 0.68,
+        "ece_151": 0.0150340170966494,
+        "ece_8_summed": 0.00832392567182635,
+        "nll_151": 0.22519446791111825,
+        "n": 3100
+      }
+    },
+    "test": {
+      "raw": {
+        "accuracy_8": 0.9158181818181819,
+        "oos_recall_8": 0.583,
+        "accuracy_8_summed": 0.912,
+        "oos_recall_8_summed": 0.562,
+        "in_scope_accuracy_150": 0.9651111111111111,
+        "oos_recall_151": 0.583,
+        "ece_151": 0.035210789312504855,
+        "ece_8_summed": 0.03976045968822517,
+        "nll_151": 0.5683634595779032,
+        "n": 5500
+      },
+      "calibrated": {
+        "accuracy_8": 0.9158181818181819,
+        "oos_recall_8": 0.583,
+        "accuracy_8_summed": 0.9118181818181819,
+        "oos_recall_8_summed": 0.561,
+        "in_scope_accuracy_150": 0.9651111111111111,
+        "oos_recall_151": 0.583,
+        "ece_151": 0.033049239642836514,
+        "ece_8_summed": 0.0381867926146527,
+        "nll_151": 0.5663161034014556,
+        "n": 5500
+      }
+    }
+  }
+}