diff --git a/DEVLOG.md b/DEVLOG.md new file mode 100644 index 0000000..0dfd2f8 --- /dev/null +++ b/DEVLOG.md @@ -0,0 +1,38 @@ +# DEVLOG: TinyRouter + +實驗進度與觀察。最新條目在最上面。 + +--- + +## 2026-09-23:AC2 通過(BERT 流程正確性檢查) + +### 本次工作 / 執行摘要 +- `make ac2`:`bert-base-uncased` 在完整 OOS+ 訓練集(15,000 in-scope + 250 OOS)訓練 151 類,seeds 42 / 43 / 44,commit `b3a59e2`,工作目錄乾淨(`git_dirty: false`)。 +- 超參數是骨架的起始值,未調參:lr 5e-5、5 epochs、batch 32、max_length 64、warmup 10%、weight decay 0.01。 +- 開跑前 PR #5 經兩輪審查:續跑比對 config、三處 sha 一致才算完成、判定驗 seed 身分、過期的 `ac2.json` 開跑即刪。 + +### 核心發現 / 數據 +| seed | val in-scope | val OOS recall | test in-scope | test OOS recall | 訓練時間 | +|---:|---:|---:|---:|---:|---:| +| 42 | 96.73% | 68.0% | 96.51% | 58.3% | 886 s | +| 43 | 96.73% | 69.0% | 96.40% | 55.8% | 847 s | +| 44 | 96.73% | 73.0% | 96.22% | 56.7% | 822 s | + +- AC2 門檻 95.7%(test in-scope,三個 seed 都要過):**PASS**。test in-scope 平均 96.38%;Larson et al. 2019(BERT、OOS+、oos-train)為 96.7%,OOS recall 59.2%,本次 56.9%。這是工程驗收,不宣稱重現原論文。 +- 三個 seed 的 val in-scope 都是 2,902 / 3,000。已從 logits 檔直接重算確認不是同一份結果:三個檔 SHA-256 不同,val 上約 2.5% 的預測彼此不同,test 數字也不同;判定為巧合。 +- **OOS 是弱點,符合預期**:in-scope 96%,但 test 上約 43% 的 OOS 被分進某個 intent。這正是 RQ2 到 RQ4 要處理的問題。 +- 效率(M4、MPS):參數 109.6M;每個 seed 約 14 分鐘(2,385 步);峰值記憶體 driver 約 4.6 GB、存活 tensor 約 1.8 GB(MPS 沒有原生峰值 API,採樣值,量法見結果檔)。 + +### Blockers / 遇到的問題 +- 磁碟可用空間從 15 GB 降到 11 GB。主要佔用是 uv 全域快取(33 GB,全機共用,非本專案)與 HF 模型快取(2.3 GB);建議 Drew 執行 `uv cache prune`,本專案不代為刪除。 +- 背景執行時包了一層 `echo exit`,所以背景任務的離開碼永遠是 0,不能拿來判斷成功與否;要看 log 裡 `make` 的輸出與 `results/ac2.json`。 + +### Next +- [ ] logits 檔附到 GitHub Release(manifest 已記 SHA-256) +- [ ] ModernBERT 相容性試跑(MPS 訓練、Optimum ONNX 匯出) +- [ ] k-shot 抽樣模組(每個 intent k 筆、OOS ⌈2.5k⌉ 筆)與兩條學習曲線 + +### Files / Budget +- `results/ac2.json`、`results/runs/bert-base-uncased-full-seed{42,43,44}.json`、`results/logits-manifest.json` +- logits 檔(不進 git):`results/logits/bert-base-uncased-full-seed{42,43,44}.npz`,各約 4.8 MB +- API 花費:US$0 diff --git a/results/ac2.json b/results/ac2.json new file mode 100644 index 0000000..9d31d5f --- /dev/null +++ b/results/ac2.json @@ -0,0 +1,81 @@ +{ + "criterion": "AC2: test in_scope_accuracy_150 >= threshold for every seed", + "split": "test", + "threshold": 0.957, + "tuning_note": "tune on the validation numbers only; test is the verdict", + "seeds": { + "42": { + "run_name": "bert-base-uncased-full-seed42", + "logits_sha256": "673a4c11200dd61ddd6310f174ab9fb27580cf5b512c7ba3b5db19088958bab2", + "validation": { + "in_scope_accuracy_150": 0.9673333333333334, + "oos_recall_151": 0.68, + "accuracy_8": 0.9780645161290322, + "oos_recall_8": 0.68 + }, + "test": { + "in_scope_accuracy_150": 0.9651111111111111, + "oos_recall_151": 0.583, + "accuracy_8": 0.9158181818181819, + "oos_recall_8": 0.583 + }, + "passed": true + }, + "43": { + "run_name": "bert-base-uncased-full-seed43", + "logits_sha256": "b6963da29f5886b915d2c5626391721fbd18c7f54a68b7720bf2d10d6cc2c58f", + "validation": { + "in_scope_accuracy_150": 0.9673333333333334, + "oos_recall_151": 0.69, + "accuracy_8": 0.9761290322580645, + "oos_recall_8": 0.69 + }, + "test": { + "in_scope_accuracy_150": 0.964, + "oos_recall_151": 0.558, + "accuracy_8": 0.9098181818181819, + "oos_recall_8": 0.558 + }, + "passed": true + }, + "44": { + "run_name": "bert-base-uncased-full-seed44", + "logits_sha256": "0324b20d4736ca9ad913b97269100b398475ab06a2bd130f653af55de239c277", + "validation": { + "in_scope_accuracy_150": 0.9673333333333334, + "oos_recall_151": 0.73, + "accuracy_8": 0.9790322580645161, + "oos_recall_8": 0.73 + }, + "test": { + "in_scope_accuracy_150": 0.9622222222222222, + "oos_recall_151": 0.567, + "accuracy_8": 0.9105454545454545, + "oos_recall_8": 0.567 + }, + "passed": true + } + }, + "verdict": "PASS", + "config": { + "model_name": "google-bert/bert-base-uncased", + "model_revision": "86b5e0934494bd15c9632b12f734a8a67f723594", + "per_intent": null, + "max_length": 64, + "learning_rate": 5e-05, + "weight_decay": 0.01, + "warmup_ratio": 0.1, + "num_train_epochs": 5, + "max_steps": -1, + "train_batch_size": 32, + "eval_batch_size": 128, + "device": "auto", + "replace_classifier_head": false, + "eval_per_intent": null + }, + "weights_kept": { + "42": true, + "43": false, + "44": false + } +} diff --git a/results/logits-manifest.json b/results/logits-manifest.json new file mode 100644 index 0000000..b96459e --- /dev/null +++ b/results/logits-manifest.json @@ -0,0 +1,38 @@ +{ + "format_version": 1, + "files": { + "bert-base-uncased-full-seed42.npz": { + "sha256": "673a4c11200dd61ddd6310f174ab9fb27580cf5b512c7ba3b5db19088958bab2", + "bytes": 4824207, + "run_name": "bert-base-uncased-full-seed42", + "git_commit": "b3a59e2debe8bf210ba749ec6d34033b9be6cab8", + "created_at": "2026-09-23T03:37:16+00:00", + "rows": { + "validation": 3100, + "test": 5500 + } + }, + "bert-base-uncased-full-seed43.npz": { + "sha256": "b6963da29f5886b915d2c5626391721fbd18c7f54a68b7720bf2d10d6cc2c58f", + "bytes": 4822394, + "run_name": "bert-base-uncased-full-seed43", + "git_commit": "b3a59e2debe8bf210ba749ec6d34033b9be6cab8", + "created_at": "2026-09-23T03:51:50+00:00", + "rows": { + "validation": 3100, + "test": 5500 + } + }, + "bert-base-uncased-full-seed44.npz": { + "sha256": "0324b20d4736ca9ad913b97269100b398475ab06a2bd130f653af55de239c277", + "bytes": 4823526, + "run_name": "bert-base-uncased-full-seed44", + "git_commit": "b3a59e2debe8bf210ba749ec6d34033b9be6cab8", + "created_at": "2026-09-23T04:05:58+00:00", + "rows": { + "validation": 3100, + "test": 5500 + } + } + } +} diff --git a/results/runs/bert-base-uncased-full-seed42.json b/results/runs/bert-base-uncased-full-seed42.json new file mode 100644 index 0000000..50e4c1c --- /dev/null +++ b/results/runs/bert-base-uncased-full-seed42.json @@ -0,0 +1,135 @@ +{ + "run_name": "bert-base-uncased-full-seed42", + "config": { + "model_name": "google-bert/bert-base-uncased", + "model_revision": "86b5e0934494bd15c9632b12f734a8a67f723594", + "seed": 42, + "per_intent": null, + "max_length": 64, + "learning_rate": 5e-05, + "weight_decay": 0.01, + "warmup_ratio": 0.1, + "num_train_epochs": 5, + "max_steps": -1, + "train_batch_size": 32, + "eval_batch_size": 128, + "device": "auto", + "replace_classifier_head": false, + "checkpoint_root": "checkpoints", + "results_root": "results", + "eval_per_intent": null + }, + "environment": { + "python": "3.12.12", + "torch": "2.14.0", + "transformers": "5.17.0", + "platform": "macOS-26.6.2-arm64-arm-64bit", + "device": "mps", + "device_name": "Apple M4", + "deterministic_algorithms": false, + "dataset_revision": "155b9c710419136e17307b80d0a13e68cd46b4ec" + }, + "training": { + "run_name": "bert-base-uncased-full-seed42", + "seed": 42, + "config": { + "model_name": "google-bert/bert-base-uncased", + "model_revision": "86b5e0934494bd15c9632b12f734a8a67f723594", + "seed": 42, + "per_intent": null, + "max_length": 64, + "learning_rate": 5e-05, + "weight_decay": 0.01, + "warmup_ratio": 0.1, + "num_train_epochs": 5, + "max_steps": -1, + "train_batch_size": 32, + "eval_batch_size": 128, + "device": "auto", + "replace_classifier_head": false, + "checkpoint_root": "checkpoints", + "results_root": "results", + "eval_per_intent": null + }, + "git_commit": "b3a59e2debe8bf210ba749ec6d34033b9be6cab8", + "git_dirty": false, + "device": "mps", + "train_rows": 15250, + "oos_train_rows": 250, + "per_intent": null, + "global_step": 2385, + "train_loss": 0.8498310221066265, + "parameters": { + "total": 109598359, + "trainable": 109598359 + }, + "train_wall_seconds": 886.184, + "peak_memory": { + "device": "mps", + "measures": "sampled after each backward and optimizer step: torch.mps.driver_allocated_memory (Metal driver, includes allocator cache; headline, usually at or above the tensor peak but not guaranteed) and current_allocated_memory (live tensors; lower bound)", + "samples": 4772, + "driver_allocated_bytes": 4607033344, + "tensor_allocated_bytes": 1791083008 + } + }, + "logits": { + "file": "bert-base-uncased-full-seed42.npz", + "sha256": "673a4c11200dd61ddd6310f174ab9fb27580cf5b512c7ba3b5db19088958bab2", + "bytes": 4824207 + }, + "metrics": { + "temperature": 1.0109612433324868, + "validation": { + "raw": { + "accuracy_8": 0.9780645161290322, + "oos_recall_8": 0.68, + "accuracy_8_summed": 0.9764516129032258, + "oos_recall_8_summed": 0.64, + "in_scope_accuracy_150": 0.9673333333333334, + "oos_recall_151": 0.68, + "ece_151": 0.013230313917851991, + "ece_8_summed": 0.00762503365895657, + "nll_151": 0.2252614577481346, + "n": 3100 + }, + "calibrated": { + "accuracy_8": 0.9780645161290322, + "oos_recall_8": 0.68, + "accuracy_8_summed": 0.9764516129032258, + "oos_recall_8_summed": 0.64, + "in_scope_accuracy_150": 0.9673333333333334, + "oos_recall_151": 0.68, + "ece_151": 0.0150340170966494, + "ece_8_summed": 0.00832392567182635, + "nll_151": 0.22519446791111825, + "n": 3100 + } + }, + "test": { + "raw": { + "accuracy_8": 0.9158181818181819, + "oos_recall_8": 0.583, + "accuracy_8_summed": 0.912, + "oos_recall_8_summed": 0.562, + "in_scope_accuracy_150": 0.9651111111111111, + "oos_recall_151": 0.583, + "ece_151": 0.035210789312504855, + "ece_8_summed": 0.03976045968822517, + "nll_151": 0.5683634595779032, + "n": 5500 + }, + "calibrated": { + "accuracy_8": 0.9158181818181819, + "oos_recall_8": 0.583, + "accuracy_8_summed": 0.9118181818181819, + "oos_recall_8_summed": 0.561, + "in_scope_accuracy_150": 0.9651111111111111, + "oos_recall_151": 0.583, + "ece_151": 0.033049239642836514, + "ece_8_summed": 0.0381867926146527, + "nll_151": 0.5663161034014556, + "n": 5500 + } + } + } +} diff --git a/results/runs/bert-base-uncased-full-seed43.json b/results/runs/bert-base-uncased-full-seed43.json new file mode 100644 index 0000000..8524948 --- /dev/null +++ b/results/runs/bert-base-uncased-full-seed43.json @@ -0,0 +1,135 @@ +{ + "run_name": "bert-base-uncased-full-seed43", + "config": { + "model_name": "google-bert/bert-base-uncased", + "model_revision": "86b5e0934494bd15c9632b12f734a8a67f723594", + "seed": 43, + "per_intent": null, + "max_length": 64, + "learning_rate": 5e-05, + "weight_decay": 0.01, + "warmup_ratio": 0.1, + "num_train_epochs": 5, + "max_steps": -1, + "train_batch_size": 32, + "eval_batch_size": 128, + "device": "auto", + "replace_classifier_head": false, + "checkpoint_root": "checkpoints", + "results_root": "results", + "eval_per_intent": null + }, + "environment": { + "python": "3.12.12", + "torch": "2.14.0", + "transformers": "5.17.0", + "platform": "macOS-26.6.2-arm64-arm-64bit", + "device": "mps", + "device_name": "Apple M4", + "deterministic_algorithms": false, + "dataset_revision": "155b9c710419136e17307b80d0a13e68cd46b4ec" + }, + "training": { + "run_name": "bert-base-uncased-full-seed43", + "seed": 43, + "config": { + "model_name": "google-bert/bert-base-uncased", + "model_revision": "86b5e0934494bd15c9632b12f734a8a67f723594", + "seed": 43, + "per_intent": null, + "max_length": 64, + "learning_rate": 5e-05, + "weight_decay": 0.01, + "warmup_ratio": 0.1, + "num_train_epochs": 5, + "max_steps": -1, + "train_batch_size": 32, + "eval_batch_size": 128, + "device": "auto", + "replace_classifier_head": false, + "checkpoint_root": "checkpoints", + "results_root": "results", + "eval_per_intent": null + }, + "git_commit": "b3a59e2debe8bf210ba749ec6d34033b9be6cab8", + "git_dirty": false, + "device": "mps", + "train_rows": 15250, + "oos_train_rows": 250, + "per_intent": null, + "global_step": 2385, + "train_loss": 0.8336240428548689, + "parameters": { + "total": 109598359, + "trainable": 109598359 + }, + "train_wall_seconds": 846.954, + "peak_memory": { + "device": "mps", + "measures": "sampled after each backward and optimizer step: torch.mps.driver_allocated_memory (Metal driver, includes allocator cache; headline, usually at or above the tensor peak but not guaranteed) and current_allocated_memory (live tensors; lower bound)", + "samples": 4772, + "driver_allocated_bytes": 4598808576, + "tensor_allocated_bytes": 1791410688 + } + }, + "logits": { + "file": "bert-base-uncased-full-seed43.npz", + "sha256": "b6963da29f5886b915d2c5626391721fbd18c7f54a68b7720bf2d10d6cc2c58f", + "bytes": 4822394 + }, + "metrics": { + "temperature": 1.0008284006317338, + "validation": { + "raw": { + "accuracy_8": 0.9761290322580645, + "oos_recall_8": 0.69, + "accuracy_8_summed": 0.9764516129032258, + "oos_recall_8_summed": 0.69, + "in_scope_accuracy_150": 0.9673333333333334, + "oos_recall_151": 0.69, + "ece_151": 0.012097747935842097, + "ece_8_summed": 0.010009780011393682, + "nll_151": 0.21022689886551754, + "n": 3100 + }, + "calibrated": { + "accuracy_8": 0.9761290322580645, + "oos_recall_8": 0.69, + "accuracy_8_summed": 0.9764516129032258, + "oos_recall_8_summed": 0.69, + "in_scope_accuracy_150": 0.9673333333333334, + "oos_recall_151": 0.69, + "ece_151": 0.012845696440683318, + "ece_8_summed": 0.010012789975572572, + "nll_151": 0.21022648824776508, + "n": 3100 + } + }, + "test": { + "raw": { + "accuracy_8": 0.9098181818181819, + "oos_recall_8": 0.558, + "accuracy_8_summed": 0.9054545454545454, + "oos_recall_8_summed": 0.533, + "in_scope_accuracy_150": 0.964, + "oos_recall_151": 0.558, + "ece_151": 0.04177787788265683, + "ece_8_summed": 0.046941955372483246, + "nll_151": 0.602609068105558, + "n": 5500 + }, + "calibrated": { + "accuracy_8": 0.9098181818181819, + "oos_recall_8": 0.558, + "accuracy_8_summed": 0.9052727272727272, + "oos_recall_8_summed": 0.532, + "in_scope_accuracy_150": 0.964, + "oos_recall_151": 0.558, + "ece_151": 0.04163421676177682, + "ece_8_summed": 0.047010190192239724, + "nll_151": 0.6024169366824238, + "n": 5500 + } + } + } +} diff --git a/results/runs/bert-base-uncased-full-seed44.json b/results/runs/bert-base-uncased-full-seed44.json new file mode 100644 index 0000000..c96b16a --- /dev/null +++ b/results/runs/bert-base-uncased-full-seed44.json @@ -0,0 +1,135 @@ +{ + "run_name": "bert-base-uncased-full-seed44", + "config": { + "model_name": "google-bert/bert-base-uncased", + "model_revision": "86b5e0934494bd15c9632b12f734a8a67f723594", + "seed": 44, + "per_intent": null, + "max_length": 64, + "learning_rate": 5e-05, + "weight_decay": 0.01, + "warmup_ratio": 0.1, + "num_train_epochs": 5, + "max_steps": -1, + "train_batch_size": 32, + "eval_batch_size": 128, + "device": "auto", + "replace_classifier_head": false, + "checkpoint_root": "checkpoints", + "results_root": "results", + "eval_per_intent": null + }, + "environment": { + "python": "3.12.12", + "torch": "2.14.0", + "transformers": "5.17.0", + "platform": "macOS-26.6.2-arm64-arm-64bit", + "device": "mps", + "device_name": "Apple M4", + "deterministic_algorithms": false, + "dataset_revision": "155b9c710419136e17307b80d0a13e68cd46b4ec" + }, + "training": { + "run_name": "bert-base-uncased-full-seed44", + "seed": 44, + "config": { + "model_name": "google-bert/bert-base-uncased", + "model_revision": "86b5e0934494bd15c9632b12f734a8a67f723594", + "seed": 44, + "per_intent": null, + "max_length": 64, + "learning_rate": 5e-05, + "weight_decay": 0.01, + "warmup_ratio": 0.1, + "num_train_epochs": 5, + "max_steps": -1, + "train_batch_size": 32, + "eval_batch_size": 128, + "device": "auto", + "replace_classifier_head": false, + "checkpoint_root": "checkpoints", + "results_root": "results", + "eval_per_intent": null + }, + "git_commit": "b3a59e2debe8bf210ba749ec6d34033b9be6cab8", + "git_dirty": false, + "device": "mps", + "train_rows": 15250, + "oos_train_rows": 250, + "per_intent": null, + "global_step": 2385, + "train_loss": 0.8535237958096358, + "parameters": { + "total": 109598359, + "trainable": 109598359 + }, + "train_wall_seconds": 821.605, + "peak_memory": { + "device": "mps", + "measures": "sampled after each backward and optimizer step: torch.mps.driver_allocated_memory (Metal driver, includes allocator cache; headline, usually at or above the tensor peak but not guaranteed) and current_allocated_memory (live tensors; lower bound)", + "samples": 4772, + "driver_allocated_bytes": 4665753600, + "tensor_allocated_bytes": 1794621952 + } + }, + "logits": { + "file": "bert-base-uncased-full-seed44.npz", + "sha256": "0324b20d4736ca9ad913b97269100b398475ab06a2bd130f653af55de239c277", + "bytes": 4823526 + }, + "metrics": { + "temperature": 1.0037511342083603, + "validation": { + "raw": { + "accuracy_8": 0.9790322580645161, + "oos_recall_8": 0.73, + "accuracy_8_summed": 0.9783870967741936, + "oos_recall_8_summed": 0.7, + "in_scope_accuracy_150": 0.9673333333333334, + "oos_recall_151": 0.73, + "ece_151": 0.013541854142624253, + "ece_8_summed": 0.008075282913590446, + "nll_151": 0.2151283633162338, + "n": 3100 + }, + "calibrated": { + "accuracy_8": 0.9790322580645161, + "oos_recall_8": 0.73, + "accuracy_8_summed": 0.9783870967741936, + "oos_recall_8_summed": 0.7, + "in_scope_accuracy_150": 0.9673333333333334, + "oos_recall_151": 0.73, + "ece_151": 0.014578263743353697, + "ece_8_summed": 0.008104776476679537, + "nll_151": 0.21512074166895853, + "n": 3100 + } + }, + "test": { + "raw": { + "accuracy_8": 0.9105454545454545, + "oos_recall_8": 0.567, + "accuracy_8_summed": 0.9072727272727272, + "oos_recall_8_summed": 0.549, + "in_scope_accuracy_150": 0.9622222222222222, + "oos_recall_151": 0.567, + "ece_151": 0.0398886793658524, + "ece_8_summed": 0.045877351850559384, + "nll_151": 0.6166574293291137, + "n": 5500 + }, + "calibrated": { + "accuracy_8": 0.9105454545454545, + "oos_recall_8": 0.567, + "accuracy_8_summed": 0.9072727272727272, + "oos_recall_8_summed": 0.549, + "in_scope_accuracy_150": 0.9622222222222222, + "oos_recall_151": 0.567, + "ece_151": 0.03919760853721055, + "ece_8_summed": 0.0453676554517786, + "nll_151": 0.6157523386856693, + "n": 5500 + } + } + } +}