diff --git a/.gitignore b/.gitignore index 49d6395..814fcff 100644 --- a/.gitignore +++ b/.gitignore @@ -19,8 +19,10 @@ codecov_comment.md DerivedData build-engine -# P0.5 eval harness: Lman's own text must never enter this repo. Any file -# matching *.private.* (corpus candidates, generated eval sets derived from -# personal vault/Slack text) stays under ~/Dev/mixime-private/ only. +# P0.5 eval harness: nobody's personal text may enter this repo. Corpus +# candidates and the eval TSVs generated from them are derived from private +# notes/chat logs, so keep them outside the repo (point BOPOMIX_EVAL_CORPUS +# at them) or under the ignored eval-corpus/ directory. # See tools/eval/README.md. -*.private.* \ No newline at end of file +*.private.* +eval-corpus/ \ No newline at end of file diff --git a/AGENTS.md b/AGENTS.md index 219bd06..38a6eb5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -11,8 +11,7 @@ Bopomix (混打注音輸入法) is a fork of OpenVanilla McBopomofo (小麥注 **Runtime:** macOS 10.15 (Catalina) or later **Development:** -- macOS 14.7 or later -- Xcode 15.3 or later +- macOS 26 or later, Xcode 26 or later (Swift 6.3) -- what the maintainer builds with; upstream McBopomofo's stated floor was macOS 14.7 / Xcode 15.3 and has not been re-verified for this fork - Python 3.9 (for dictionary data generation) ## Building and Running diff --git a/BopomixTests/KeyHandlerBopomofoTests.swift b/BopomixTests/KeyHandlerBopomofoTests.swift index 55ab005..d417d1c 100644 --- a/BopomixTests/KeyHandlerBopomofoTests.swift +++ b/BopomixTests/KeyHandlerBopomofoTests.swift @@ -35,10 +35,10 @@ class KeyHandlerBopomofoTests: XCTestCase { var handler = KeyHandler() // P3 fix #5 (XCTest/machine-preference isolation, see - // ~/.claude/plans/zhuyin-ime-personal.md's P3 fix #5): these upstream + // the design notes' P3 fix #5): these upstream // tests know nothing about mixedScript and never save/restore its // preferences themselves, so on a machine that already has - // MixedScriptEnabled=1 in its real plist (Lman dogfoods a master + // MixedScriptEnabled=1 in its real plist (the maintainer dogfoods a master // build) they would otherwise inherit that value and see mixedScript // rewrite readings it never used to (e.g. testInvalidBpmf's "ni4" // becomes a Rule-A literal run instead of the upstream diff --git a/BopomixTests/LatinCompletionKeyHandlerTests.swift b/BopomixTests/LatinCompletionKeyHandlerTests.swift index 3ac1c08..95949e5 100644 --- a/BopomixTests/LatinCompletionKeyHandlerTests.swift +++ b/BopomixTests/LatinCompletionKeyHandlerTests.swift @@ -26,7 +26,7 @@ import XCTest @testable import Bopomix /// KeyHandler-level integration tests for P3 English prediction + Tab -/// completion (see ~/.claude/plans/zhuyin-ime-personal.md's F3 scope). +/// completion (see the design notes' F3 scope). /// Mirrors MixedScriptKeyHandlerTests' approach (drive the real KeyHandler /// FSM, not the engine in isolation) for the same reason P1's review found: /// every interesting interaction here is with candidate-window/Tab/grid @@ -353,7 +353,7 @@ class LatinCompletionKeyHandlerTests: XCTestCase { XCTAssertNil(predictedCompletion, "\(state)") } - /// P3 fix #3 (see ~/.claude/plans/zhuyin-ime-personal.md's P3 fix #3 + /// P3 fix #3 (see the design notes' P3 fix #3 /// and KeyHandler's _offeredCompletionFor:lexicon:): once a pending run is /// itself already a recognized word ranked at least as well as the /// best longer completion sharing its prefix, that counts as "the @@ -1503,9 +1503,11 @@ class LatinCompletionKeyHandlerTests: XCTestCase { } /// How many letters of an English token does the user actually have - /// to type before Tab would complete it, on eval200 (the only - /// Traditional-Mandarin-plus-real-project-vocabulary corpus this repo - /// has) -- measured two ways (P3 fix #4): "no history" evaluates + /// to type before Tab would complete it, on the corpus named by + /// `BOPOMIX_EVAL_CORPUS` (a Traditional-Mandarin-plus-real-project- + /// vocabulary TSV, kept outside this repo -- build one from your own + /// text with tools/eval/build_corpus.py) -- measured two ways + /// (P3 fix #4): "no history" evaluates /// every token cold, against just the built-in dictionary/tech seed; /// "with history" replays the corpus in row order, and after /// evaluating each row's tokens, types and commits every eligible one @@ -1514,14 +1516,23 @@ class LatinCompletionKeyHandlerTests: XCTestCase { /// enough to be confirmed becomes more completable later in the /// corpus. This is a measurement/report, not a /// correctness gate (except the pure-Chinese control below): skipped - /// outright when the private corpus is not present, and neither table + /// outright when `BOPOMIX_EVAL_CORPUS` is unset, and neither table /// asserts thresholds the way testEval200ThroughKeyHandler does -- /// there is no prior baseline to hold either to yet, this run creates /// one for both. func testEval200LatinCompletion() throws { - let corpusPath = NSHomeDirectory() + "/Dev/mixime-private/eval200.tsv" + guard let corpusPath = ProcessInfo.processInfo.environment["BOPOMIX_EVAL_CORPUS"], + !corpusPath.isEmpty + else { + throw XCTSkip( + "BOPOMIX_EVAL_CORPUS is not set; nothing to measure. Point it at a " + + "corpus TSV built from your own text with tools/eval/build_corpus.py.") + } guard FileManager.default.fileExists(atPath: corpusPath) else { - throw XCTSkip("corpus not present at \(corpusPath); nothing to measure") + throw XCTSkip( + "BOPOMIX_EVAL_CORPUS points at \(corpusPath), which does not exist; " + + "nothing to measure. Build a corpus TSV from your own text with " + + "tools/eval/build_corpus.py.") } let rows = try loadEvalRows(at: corpusPath) XCTAssertFalse(rows.isEmpty) diff --git a/BopomixTests/MixedScriptKeyHandlerTests.swift b/BopomixTests/MixedScriptKeyHandlerTests.swift index 16281d6..3bb7ea2 100644 --- a/BopomixTests/MixedScriptKeyHandlerTests.swift +++ b/BopomixTests/MixedScriptKeyHandlerTests.swift @@ -26,7 +26,7 @@ import XCTest @testable import Bopomix /// KeyHandler-level integration tests for P1 zh/en mixed typing (see -/// ~/.claude/plans/zhuyin-ime-personal.md). The engine-level tests under +/// the design notes). The engine-level tests under /// `Source/Engine/MixedScript/` cover the rules in isolation; everything /// here drives the real `KeyHandler` FSM the way the input method does, /// because every blocking defect found in the first review round @@ -791,10 +791,18 @@ class MixedScriptKeyHandlerTests: XCTestCase { /// dropping a whole run). This types the same corpus through the real /// `KeyHandler` instead and is the number that counts. func testEval200ThroughKeyHandler() throws { - let corpusPath = - NSHomeDirectory() + "/Dev/mixime-private/eval200.tsv" + guard let corpusPath = ProcessInfo.processInfo.environment["BOPOMIX_EVAL_CORPUS"], + !corpusPath.isEmpty + else { + throw XCTSkip( + "BOPOMIX_EVAL_CORPUS is not set; nothing to measure. Point it at a " + + "corpus TSV built from your own text with tools/eval/build_corpus.py.") + } guard FileManager.default.fileExists(atPath: corpusPath) else { - throw XCTSkip("corpus not present at \(corpusPath); nothing to measure") + throw XCTSkip( + "BOPOMIX_EVAL_CORPUS points at \(corpusPath), which does not exist; " + + "nothing to measure. Build a corpus TSV from your own text with " + + "tools/eval/build_corpus.py.") } let rows = try loadCorpus(at: corpusPath) XCTAssertFalse(rows.isEmpty) @@ -946,7 +954,7 @@ class MixedScriptKeyHandlerTests: XCTestCase { /// R5: a rule-A run has a single candidate. Tab-cycling over it is not a /// choice between Chinese and English and must not touch latin-user.txt. /// - /// P3 (see ~/.claude/plans/zhuyin-ime-personal.md's F3 scope) gives Tab + /// P3 (see the design notes' F3 scope) gives Tab /// a new, deliberate job on a pending rule-A run -- completing it into /// a longer dictionary word -- but P3 fix #3 also gives it a deliberate /// *non*-job here: "acer" is tech-seed ranked ahead of every longer @@ -973,7 +981,7 @@ class MixedScriptKeyHandlerTests: XCTestCase { XCTAssertEqual(after, before, "Tab over a rule-A run must not remember the word") } - /// P3 fix #3 (see ~/.claude/plans/zhuyin-ime-personal.md's P3 fix #3 + /// P3 fix #3 (see the design notes' P3 fix #3 /// and KeyHandler's _offeredCompletionFor:lexicon:): Tab must not silently /// grow an already-finished word into a longer dictionary entry just /// because one happens to share its prefix. Companion to diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..34df54c --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,55 @@ +# Changelog + +混打注音相對於上游小麥注音（McBopomofo）的變更。上游本身的版本紀錄見 [McBopomofo releases](https://github.com/openvanilla/McBopomofo/releases)。 + +## 0.1.0（2026-09-14） + +第一個公開版。基底是 McBopomofo `f5ba010`（2026-09）。 + +### 新增 + +**中英混打，不切換輸入法**（`MixedScriptEnabled`，預設開） + +- 字母序列一旦拼不出合法注音（例如 `th`，或 `acer` 打到第二個字母），整段轉成英文，組字區直接顯示原始字母；英文後面的空白鍵就是真的空白。 +- 拼得出注音、又剛好是英文單字的序列（`app`、`ell`、`all`、`full`）維持中文，英文形式放在候選窗第二列，按一次 Tab 切過去。用 Tab 或候選窗選過的英文詞會寫進個人詞庫，之後同一個詞加空白就直接判成英文（`MixedScriptLatinOnSpaceForUserWords`，預設開）。 +- 永不靜默丟字：任何情況下組字區都顯示你打的原始字母。 +- 只在標準（大千）配置與「注音」模式下作用；關掉時所有程式路徑短路，行為與上游完全相同（有測試釘住）。 +- 數字（維護者自己 200 句中英混打語料，走 app 的真實按鍵路徑）：英文 token 保留率 0% → 90.6%，純中文句子與上游逐字元相同（95.3% 同音字正確率不變）；混打開與關的每句延遲差異約 0.3 ms（`tools/eval/BASELINE.md` 純中文 on／off 對照）。 + +**英文預測與 Tab 補全**（`LatinCompletionEnabled`，預設開；需要混打也開著） + +- 英文打到 4 個字母，組字區下方提示完整的字加 ⇥；Tab 接受（游標停在字尾，可以繼續打），Shift+Tab 開候選窗列出多個，用平常的選字鍵挑；候選窗開著繼續打字母會更新清單，Esc 回到原始字母。 +- 4 個字母的門檻是實測來的：用 17 個每天會打的詞共 100 個按鍵測，門檻 2 時螢幕上出現 33 次錯的提示、只省 7 個按鍵；門檻 4 錯誤提示降到 11 次、省下的按鍵只少 1 個。Tab 與 Shift+Tab 從 2 個字母起就能用，只是不主動提示。 +- 已經是完整的字、而且常用度不比更長的補全差時（`acer` 相對 `acerbic`），視為打完了，不提示也不補。 +- 排序：個人詞庫（用得多的優先）＞ 手工科技詞表 ＞ 基礎詞典（依常用度分級）＞ 字母序。基礎詞典來自 SCOWL/ESDB size ≤ 70 的美式拼法詞表，141,697 個詞、含變化形，分級是 5 個粗桶。 +- 數字（同一份語料）：英文 token 在 4 個字母內可補全 23%（有個人詞庫歷史時 28.8%），每個 token 省 0.39 到 0.46 個按鍵。不可補全的 47% 裡，81 個是排序沒排到、55 個不在詞典、18 個還沒被判成英文、32 個低於提示門檻。 + +**從打過的英文學習**（`LatinLearnTypedWords`，預設開） + +- 打完一段純英文並以 Enter、空白或標點提交，就計入個人詞庫，不只 Tab 接受的才算。 +- 一次提交加 1 分，2 分才影響排序與建議；詞典裡沒有的字（人名、產品名）第一次以 1 分寫入但不會被建議，第二次提交才會。兩次不必在同一次開機內。 +- 只記提交出去的英文，Esc 或 Backspace 取消的不記，注音與中文從不記錄。Shift 強制大寫的路徑（上游既有）也不記。 +- 寫入採先讀現有檔再合併、暫存檔改名落盤；換使用者詞庫資料夾時一起重新載入，另一台機器同步回來的 `latin-user.txt` 不會被蓋掉。 +- 檔案：`~/Library/Application Support/Bopomix/latin-user.txt`，格式 `詞次數`，沒有次數的行視為 1。手動刪掉的字要重啟輸入法才會忘記。 + +**改名為 Bopomix 混打注音，並自動接手小麥注音的資料** + +- Bundle id `io.github.lmanchu.inputmethod.bopomix`（macOS 要求輸入法的 bundle id 含 `inputmethod` 這一段，否則註冊會被靜默忽略）。 +- 第一次啟動時把 `org.openvanilla.inputmethod.McBopomofo` 的偏好複製到新身分（只補沒有的鍵；自訂詞庫位置與指向舊 app 的 hook 路徑除外），並把 `~/Library/Application Support/McBopomofo/` 逐行合併到 `Bopomix/`（缺的檔複製，其餘只補目的地沒有的行；`latin-user.txt` 以字為單位合併並保留較大的次數）。舊資料只讀不寫，新資料不覆蓋，可以重試。失敗或自訂位置暫時無法讀取時下次啟動再試，並顯示一次通知。 +- 引擎的 C++ 命名空間、`McBopomofoLM` 與所有上游檔頭刻意保留，方便跟上游同步。 + +### 開發與測試 + +- 引擎的 headless 評測工具 `bopomix-eval`（cmake），可以不裝輸入法就量測「按鍵序列 → 組句」。語料由 `tools/eval/build_corpus.py` 從你自己的文字產生，不隨 repo 附上。 +- XCTest 走真實的 `KeyHandler`，評測數字以 app 路徑為準（CLI 的數字曾經誤導過一輪）。 +- 測試不得碰真實的偏好與使用者詞庫資料夾：`tools/eval/check_plist_unchanged.sh` 把整套測試包起來跑，前後比對零差異；測試套件必須序列執行。 +- 每一輪實作都有獨立的 fresh-context 審查與複驗記錄，原樣放在 `docs/`。 + +### 已知問題 + +- 拼得出注音的英文字（`mac` = ㄇㄚ）第一次一定要按 Tab；`ai`、`api`、`app`、`agent`、`acer`、`chat`、`check`、`draft`、`free` 這類高頻詞都屬此類。 +- 補全候選窗前三名之外是字母序（詞典只有 5 個常用度分桶）。下一步是換成真實詞頻。 +- 補全候選窗開著時按小寫字母會關窗。 +- 英文後連按兩個空白會跳出空白的候選窗。 +- 許氏、倚天 26 配置沒有混打。 +- 整句 AI 選字（本機小模型重排候選）研究過但未實作。 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..62362f4 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,67 @@ +# 參與混打注音 / Contributing to Bopomix + +問題回報、想法、PR 都歡迎。這份文件講怎麼把專案建起來、怎麼跑測試、PR 需要附什麼。 +細節（架構、狀態機、詞庫資料）在 [AGENTS.md](AGENTS.md) 與 [algorithm.md](algorithm.md)， +那兩份也是給 AI 助手讀的，內容以它們為準。 + +## 建置 + +需要 macOS 26、Xcode 26（Swift 6.3），引擎測試另外需要 CMake 與 GoogleTest（`brew install cmake googletest`）。 + +```sh +git clone https://github.com/lmanchu/bopomix.git +cd bopomix +xcodebuild -project Bopomix.xcodeproj -scheme BopomixInstaller -configuration Debug build +``` + +建好的 `BopomixInstaller.app` 執行一次就會把輸入法裝進 `~/Library/Input Methods/` 並重啟它。 +第一次要到系統設定 ▸ 鍵盤 ▸ 輸入方式加入「混打注音」（英文介面顯示 Bopomix）。 +之後每次改完程式重跑安裝程式即可；macOS 對同一次登入能 kill 輸入法的次數有限制， +裝了幾次沒反應就登出再登入。 + +## 跑測試 + +兩套：Swift 測試（走真正的 `KeyHandler`）與 C++ 引擎測試。 + +```sh +tools/eval/check_plist_unchanged.sh \ + xcodebuild -project Bopomix.xcodeproj -scheme Bopomix \ + -configuration Debug -derivedDataPath build \ + CODE_SIGN_IDENTITY="-" CODE_SIGNING_REQUIRED=NO DEVELOPMENT_TEAM="" test +``` + +```sh +cmake -S Source/Engine -B build-engine -DENABLE_TEST=ON && cmake --build build-engine && ctest --test-dir build-engine +``` + +三件事一定要照做： + +- **Swift 測試必須序列跑。** 不要加 `-parallel-testing-enabled YES`。偏好設定是一個所有 runner 共用的檔案，並行會互相踩到，出現與你的修改無關的失敗。 +- **測試不得碰你自己的輸入法設定與詞庫。** 上面那個 `check_plist_unchanged.sh` 把測試包起來跑，前後比對 `io.github.lmanchu.inputmethod.bopomix` 這個 domain 與 `~/Library/Application Support/Bopomix/`，有任何差異就報 `PLIST_CHANGED` / `DATA_FOLDER_CHANGED`。新測試若動到偏好，`setUpWithError()` 第一句要 `PreferenceSandbox.install(on: self)`；動到詞庫資料夾要用 `LanguageModelManager.dataFolderOverrideForTesting`。這條規則是血淚換來的：曾經有測試把暫存資料夾的路徑寫進維護者正在用的輸入法。 +- **測試跑的時候不要在已安裝的混打注音裡改設定。** 跑完 teardown 會整包還原到測試開始時的快照。 + +## 評測語料 + +兩個評測測試（`testEval200ThroughKeyHandler`、`testEval200LatinCompletion`）讀環境變數 `BOPOMIX_EVAL_CORPUS` +指向的 TSV；沒設就 skip。語料是用你**自己的**文字產生的，永遠不進 repo（`.gitignore` 已擋 `eval-corpus/` 與 `*.private.*`）： + +```sh +python3 tools/eval/build_corpus.py --candidates <你的句子檔> --output eval-corpus/corpus.tsv --cli build-engine/tools/eval/bopomix-eval --data build/Build/Products/Debug/Bopomix.app/Contents/Resources +BOPOMIX_EVAL_CORPUS="$PWD/eval-corpus/corpus.tsv" tools/eval/check_plist_unchanged.sh xcodebuild ... test +``` + +用法細節與 headless 的 `bopomix-eval` CLI 見 [tools/eval/README.md](tools/eval/README.md)。 +評測數字以 XCTest 走真實 `KeyHandler` 的那份為準，CLI 的數字只量引擎；兩者曾經差過一輪，別再被騙。 + +## 送 PR + +- Commit 用 Conventional Commits（`feat(mixedscript): …`、`fix(completion): …`）。 +- 動到中英判斷、補全、學習的 PR，請附 eval 前後的數字（`tools/eval/BASELINE.md` 那幾行）和你用的語料規模。 +- 改動集中在新檔案與少量插入點。引擎的 `McBopomofo` 命名空間、`McBopomofoLM`、上游檔頭都刻意保留，方便跟上游同步；不要順手改名。 +- 打錯字的句子是最好的回報：Issue 裡貼你打的按鍵序列（例如 `ji3slack cj04`）、你期望看到的、實際看到的。 + +## 我們怎麼做事 + +這個專案是「人跟 AI 一起 build」的練習：每一輪實作都有獨立的審查與複驗，記錄原樣放在 [docs/](docs/)。 +三個從那裡學到、寫進規則的教訓：評測要走 app 的真實路徑；測試不得碰真實設定； +任何「其實不需要做」的判斷都要查證，審查者的建議也要被複驗。 diff --git a/README.markdown b/README.markdown deleted file mode 100644 index ee6fc5d..0000000 --- a/README.markdown +++ /dev/null @@ -1,137 +0,0 @@ -# Bopomix 混打注音輸入法（fork of OpenVanilla McBopomofo 小麥注音） - -## mixime：中英混打（P1，預設關閉） - -這個 fork 多了「不切換輸入法直接打英文」的功能（見 -`~/.claude/plans/zhuyin-ime-personal.md` 的 F1／P1 節）。P1 期間**預設關閉**， -要 dogfood 請自己打開： - -```sh -defaults write io.github.lmanchu.inputmethod.bopomix MixedScriptEnabled -bool true -# 關掉： -defaults write io.github.lmanchu.inputmethod.bopomix MixedScriptEnabled -bool false -``` - -⚠️ 2026-09-10 之前的建置預設是**開啟**的，而且 `Preferences.populateDefaults()` -會把預設值寫進 plist —— 也就是說裝過舊版的機器即使升級也還是開著。這種機器要 -先跑一次 `defaults delete io.github.lmanchu.inputmethod.bopomix MixedScriptEnabled` -才會回到「預設關閉」。 - -關閉時所有 mixedScript 的程式路徑都會短路，行為與上游 McBopomofo 完全相同 -（`McBopomofoTests/MixedScriptKeyHandlerTests.swift` 的 -`testB7_DisabledBehavesLikeUpstream` 釘住這件事）。 - -打開之後： - -- 字母打到「注音上不可能組成音節」時（例如 `th`、`acer` 的第二個字母）自動轉成 - 英文，組字區直接顯示原始字母；空白鍵接在英文後面就是一個真正的空白。 -- 打得出合法注音、又剛好是英文單字時（例如 `ell`／`app`）**維持中文**，英文形式 - 放在候選窗第二列，按一次 Tab 就切過去。用 Tab 或候選窗選過的英文詞會寫進 - `latin-user.txt`，之後再打同一個詞加空白就會自動判成英文。 -- 只支援標準（大千）鍵盤配置與「注音」輸入模式，「傳統注音」不受影響。 - -另一個開關 `MixedScriptLatinOnSpaceForUserWords`（預設開）控制上面最後那條 -「個人詞庫 + 空白 → 英文」的自動行為。 - -### 英文即時預測與 Tab 補全（P3） - -在 mixedScript 打開的前提下（見上面），組字區顯示英文字母且長度達 **4 個字母**以上 -時，會在組字區下方顯示預測的完整詞加上 ⇥ 符號（例如打 `thro` 會看到 -`throughput ⇥`）： - -> 4 個字母這個門檻只管「主動顯示」。**Tab／Shift+Tab 從 2 個字母就能用**， -> 只是輸入法不會自己先跳出來猜。門檻從 2 提到 4 是 2026-09-12 實測的結果： -> 用 17 個每天會打的詞、共 100 個按鍵測，門檻 2 的時候有 33 次螢幕上顯示的是 -> **錯的**字（例如打 `the` 打到第 2 個字母就跳 `throughput`），總共只省下 7 個 -> 按鍵；提到 4 之後錯誤畫面掉到 11 次，而省下的按鍵只少 1 個。 - -- **Tab**：如果目前正在打的英文字母序列已能唯一判定成英文（例如上面提到的 - `th`／`acer`），且有比目前打的字更長的字典詞，Tab 直接把整個詞補完，游標 - 停在詞尾，可以繼續往後打字。如果目前顯示的是中文（例如 `ell`／`app` 這種還 - 沒切到英文的情況），Tab 維持原本「切到英文」的行為；切過去之後如果那個英文 - 詞還有更長的補全，再按一次 Tab 就會繼續補完。沒有任何補全時 Tab 完全不動作 - （行為與關掉這個功能時相同）——**已經打完的完整詞也算「沒有補全」**：如果 - 目前打的字母序列本身就是字典詞，而且常用度不比任何更長的補全差（例如 - `acer` 是手工科技詞表詞，排序遠優於 `acerbic`），就視為使用者已經打完了， - 不顯示預測、Tab 也不會把它拉長成別的詞。 -- **Shift+Tab**：開一個候選窗列出多個補全（數量跟目前設定的選字鍵數一樣多）， - 用平常選字的按鍵（預設 `123456789`）挑一個；候選窗開著的時候繼續打字母會直接 - 重新查詢、更新候選清單，不會像一般候選窗一樣被字母關掉；按 Esc 關掉候選窗、 - 回到還沒補完的原始字母。 -- 用 Tab 或候選窗接受的補全詞會計入個人詞庫 `latin-user.txt`（格式從純字改成 - `詞次數`，舊檔案沒有次數的行視為 1 次），下次同樣的前綴會優先排到最前 - 面；只有「接受」補全才會計次，單純看到預測 tooltip 不會。 -- 這個功能自己的開關是 `LatinCompletionEnabled`（**預設開**，但實際上要 - `MixedScriptEnabled` 也開著才會生效）： - - ```sh - defaults write io.github.lmanchu.inputmethod.bopomix LatinCompletionEnabled -bool false - ``` - -- 基礎詞典（`Source/Data/latin-words.txt`）本體與常用度分級 2026-09-11 起 - 全部來自 SCOWL/ESDB（見 `ACKNOWLEDGEMENTS.md`），排序優先序是「個人詞庫 - （用過次數多的優先）＞ 手工科技詞表 `latin-tech-seed.txt`（依整理順序）＞ - 基礎詞典（依常用度）＞ 字母序」。這是粗粒度分級（5 個 size 分桶），不是 - 逐字精確頻率；改用 SCOWL 當詞表本體後，每個詞都有分級（不再有落到字母序 - 的未分級詞），而且比舊版（macOS 內建 `/usr/share/dict/words`，1934 年 - Webster's Second，只拿 SCOWL 做分級）多了大量變化形（`games`／`typing`／ - `comments` 這類舊版查不到的詞現在都在）。 -- **從實際打過的英文學習**：`LatinLearnTypedWords`（**預設開**）控制是否把 - 「打完一段純英文字母、以 Enter／空白／標點結束」也計入個人詞庫，不只是 - Tab／候選窗接受的補全才算——也就是說，就算完全不用 Tab 補全，只要正常打 - 英文單字，之後同一個詞也會排到比較前面。隱私：只在英文段真的被提交後才寫 - （按 Esc／Backspace 取消打到一半的英文不會寫），從不記錄任何注音或中文， - Shift 強制大寫那條路徑（原版既有行為，跟 mixedScript 無關）也不會被這個 - 功能記錄；寫入的檔案一樣是本機的 `latin-user.txt`，關掉不想被記錄： - - ```sh - defaults write io.github.lmanchu.inputmethod.bopomix LatinLearnTypedWords -bool false - ``` - - **打一次記一次，打兩次才算數。** 每次提交只加 1 分，要到 2 分才會影響排序； - 字典裡沒有的詞（人名、產品名、內部代號）**第一次就會以 1 分寫進 - `latin-user.txt`**，但 1 分的非字典詞完全不會出現在預測、Tab 補全或候選窗裡 - ——它只是一張「我看過這個字一次」的紀錄，等第二次提交升到 2 分才變成建議。 - 這代表： - - - 檔案裡本來就會有一些你從沒確認過的 1 分詞，包括打錯的字、以及英文後面 - 黏到注音鍵的那種怪字串（`acersu` 之類）。它們不會被建議，不用管它。 - - 兩次不必在同一次開機／同一個輸入法 process 裡湊到（2026-09-12 之前是 - 這樣，所以真正需要學的新詞幾乎永遠學不起來）。 - - 要「忘掉」某個詞：直接編輯 `latin-user.txt` 砍掉那一行，**然後重新啟動 - 輸入法**（登出登入、或切到別的輸入法再切回來）。輸入法在記憶體裡還留著 - 那個詞的時候，下一次寫入會把它合併回檔案裡。 - -- 使用者詞庫檔案的位置跟中文使用者詞庫同一個資料夾（預設 - `~/Library/Application Support/Bopomix/`，偏好設定裡改過就跟著走）。 - 換資料夾時 Latin 詞庫會一起重新載入，寫入採「先讀現有檔案再合併」＋暫存檔 - 改名落盤，所以新資料夾原本就有的 `latin-user.txt`（例如 Dropbox 從另一台 - 機器同步回來的）不會被蓋掉，舊資料夾也不會被動到。 - -## 系統需求 - -小麥注音輸入法可以在 macOS 13 以上版本運作。如果您要自行編譯小麥注音輸入法，或參與開發，您需要： - -- macOS 26 或更高版本 -- Xcode 26 或更高版本 -- Python 3.9 (使用 Xcode 安裝後內附的就可以，也可使用 homebrew 等方式安裝) - -## 開發流程 - -用 Xcode 開啟 `Bopomix.xcodeproj`，選 "BopomixInstaller" target，build 完之後直接執行該安裝程式，就可以安裝小麥注音。 - -第一次安裝完，日後程式碼或詞庫有任何修改，只要重複上述流程，再次安裝小麥注音即可。 - -要注意的是 macOS 可能會限制同一次 login session 能 kill 同一個輸入法 process 的次數（安裝程式透過 kill input method process 來讓新版的輸入法生效）。如果安裝若干次後，發現程式修改的結果並沒有出現，或甚至輸入法已無法再選用，只要登出目前帳號再重新登入即可。 - -## 社群公約 - -歡迎小麥注音用戶回報問題與指教，也歡迎大家參與小麥注音開發。 - -首先，請參考我們在「[常見問題](https://github.com/openvanilla/McBopomofo/wiki/常見問題)」中所提「[我可以怎麼參與小麥注音？](https://github.com/openvanilla/McBopomofo/wiki/常見問題#我可以怎麼參與小麥注音)」一節的說明。 - -我們採用了 GitHub 的[通用社群公約](https://github.com/openvanilla/McBopomofo/blob/master/CODE_OF_CONDUCT.md)。公約的中文版請參考[這裡的翻譯](https://www.contributor-covenant.org/zh-tw/version/1/4/code-of-conduct/)。 - -## 軟體授權 - -本專案採用 MIT License 釋出，使用者可自由使用、散播本軟體，惟散播時必須完整保留版權聲明及軟體授權（[詳全文](https://github.com/openvanilla/McBopomofo/blob/master/LICENSE.txt)）。 diff --git a/README.md b/README.md new file mode 100644 index 0000000..247e3e2 --- /dev/null +++ b/README.md @@ -0,0 +1,98 @@ +# Bopomix 混打注音 + +macOS 注音輸入法。打中文的時候直接打英文，不用切換輸入法。 + +[English](#english) + +你打「我在 slack 上看到那個 issue 了」，從頭打到尾不按 Shift、不切輸入法。 +打到 `sl` 的時候輸入法就知道這是英文，打到 `iss` 它會提示 `issue ⇥`，按 Tab 補完。 + +混打注音做的是兩件事：不切換就能打英文，以及英文的預測與 Tab 補全。 +注音本身的打法、選字、自訂詞都跟你習慣的一樣；底層引擎與中文詞庫來自[小麥注音（McBopomofo）](https://github.com/openvanilla/McBopomofo)。 + +## 安裝 + +需求：macOS 13 以上，Apple Silicon（Intel 版之後補）。 + +1. 到 [Releases](https://github.com/lmanchu/bopomix/releases) 下載最新的 `Bopomix-*.dmg`，打開。 +2. 雙擊裡面的「安裝混打注音」，等它說安裝完成。 +3. 到系統設定 ▸ 鍵盤 ▸ 輸入方式 ▸ 編輯 ▸ 按「＋」▸ 繁體中文 ▸ 加入「混打注音」（英文介面顯示為 Bopomix）。之後從選單列的輸入法選單就能切換。清單裡找不到的話，登出再登入一次，macOS 在登入時才會重新掃描新裝的輸入法。 + +安裝程式和輸入法都有 Apple 簽章與公證，不用改任何安全設定。第 3 步只有第一次需要：macOS 要求新的輸入法由使用者親自加入一次。要移除的話，到系統設定 ▸ 鍵盤 ▸ 輸入方式把它拿掉，再刪除 `~/Library/Input Methods/Bopomix.app`。 + +原本用小麥注音？第一次啟動時，你的設定和自訂詞會自動複製過來，原來那一份不會被動到。 + +## 它會做的事 + +- **打英文不用切換**：字母一旦拼不出注音（例如 `th`、`acer` 打到第二個字母）， + 輸入法就當它是英文，組字區直接顯示字母，後面接空白就是真的空白。 +- **拼得出注音的英文字也不會卡住**：像 `app`、`ell` 這種兩邊都說得通的，先當中文， + 英文放在候選窗第二列，按一次 Tab 就切過去。選過一次，下次打同一個字加空白就直接是英文。 +- **英文預測與 Tab 補全**：英文打到 4 個字母會在下方提示完整的字，Tab 接受， + Shift+Tab 列出更多選擇。你常打的字會排前面。 +- **會學你的字**：Tab 接受過或正常打完的英文字都會記進本機的個人詞庫，人名、產品名打過兩次就會開始被建議。 + +只支援標準（大千）注音配置與「注音」模式；「傳統注音」模式與許氏、倚天 26 配置不受影響，但也沒有混打。 + +## 設定 + +都是開著的，要關掉在「終端機」執行： + +| 要關掉的功能 | 指令 | +|---|---| +| 中英混打（整組） | `defaults write io.github.lmanchu.inputmethod.bopomix MixedScriptEnabled -bool false` | +| 英文預測與 Tab 補全 | `defaults write io.github.lmanchu.inputmethod.bopomix LatinCompletionEnabled -bool false` | +| 從打過的英文學習 | `defaults write io.github.lmanchu.inputmethod.bopomix LatinLearnTypedWords -bool false` | +| 個人詞庫的字加空白直接變英文 | `defaults write io.github.lmanchu.inputmethod.bopomix MixedScriptLatinOnSpaceForUserWords -bool false` | + +把 `false` 換成 `true` 就是打開。改完切到別的輸入法再切回來。 + +## 隱私 + +- 所有事情都在你的電腦上發生。輸入法唯一會連線的是每天一次的版本檢查（到 GitHub），可以在偏好設定裡關掉。 +- 學到的英文字存在 `~/Library/Application Support/Bopomix/latin-user.txt`，一行一個字，你可以打開看、可以刪。 + 只記提交出去的英文字，從不記注音或中文；打到一半按 Esc 或 Backspace 取消的不會記。 +- 中文自訂詞也在同一個資料夾，都是純文字檔。 + +## 目前的限制 + +- 中英判斷是規則，不是 AI。拼得出注音的英文字（`mac` 也是「ㄇㄚ」）第一次一定要按 Tab。 +- 補全候選窗前三名之外是字母順序，因為詞典只有粗粒度的常用度分級。真實詞頻是下一個要做的事。 +- 從 `latin-user.txt` 手動刪掉的字，要切換輸入法重啟才會真的忘記。 +- 補全候選窗開著的時候按小寫字母會把窗關掉。 +- 整句的 AI 選字（用本機小模型重排候選）研究過但還沒做。 + +## 從原始碼建置、回報問題、參與 + +- 建置與測試方式見 [CONTRIBUTING.md](CONTRIBUTING.md)。 +- 每一版改了什麼見 [CHANGELOG.md](CHANGELOG.md)。開發過程中每一輪的審查與複驗記錄在 [docs/](docs/)，原樣公開。 +- 問題與想法請開 [Issue](https://github.com/lmanchu/bopomix/issues)。打錯字的句子是最好的回報：把你打的按鍵和你期望的結果貼上來。 + +## 致謝與授權 + +混打注音衍生自 OpenVanilla 的 [小麥注音 McBopomofo](https://github.com/openvanilla/McBopomofo)（MIT License，Copyright 2011-2026 Mengjuei Hsieh et al.）。 +英文詞典來自 [SCOWL](https://github.com/en-wl/wordlist)。完整清單見 [ACKNOWLEDGEMENTS.md](ACKNOWLEDGEMENTS.md)。 + +本專案採用 MIT License，見 [LICENSE.txt](LICENSE.txt)。 + +--- + +## English + +**Bopomix** is a Zhuyin (Bopomofo) input method for macOS that lets you type English in the middle of Chinese without switching input sources. + +Type 我在 slack 上看到那個 issue 了 straight through. By `sl` the input method has decided it is English; by `iss` it offers `issue ⇥`, and Tab completes it. + +Bopomix does two things: mixed Chinese/English typing without switching, and English prediction with Tab completion. Typing Zhuyin, picking candidates and keeping your own phrases work the way you are used to; the engine and the Chinese dictionary underneath come from [McBopomofo](https://github.com/openvanilla/McBopomofo) (OpenVanilla, MIT). + +**Install** (macOS 13+, Apple Silicon): download the latest `Bopomix-*.dmg` from [Releases](https://github.com/lmanchu/bopomix/releases), open it, double-click **安裝混打注音** (Install Bopomix), then add **Bopomix** once under System Settings ▸ Keyboard ▸ Input Sources ▸ Edit ▸ + ▸ Traditional Chinese. From then on it is in the input menu; if it is missing from the list, log out and back in, since macOS only rescans newly installed input methods at login. Both the installer and the input method are signed and notarized. Coming from McBopomofo? Your settings and user phrases are copied over on first launch and the original is left untouched. + +**What it does**: letters that cannot form a Zhuyin syllable (`th`, the second letter of `acer`) become English on the spot. Words that are valid both ways (`app`, `ell`) stay Chinese, with the English reading one Tab away; once chosen, the same word plus a space is English from then on. After four letters of English you get a completion hint; Tab accepts it, Shift+Tab shows more. Words you type are learned into a local list. Standard (Dachen) layout and the Bopomofo mode only. + +**Privacy**: everything runs locally. The only network access is a daily update check against GitHub, which you can turn off. Learned English words live in `~/Library/Application Support/Bopomix/latin-user.txt`, one per line; nothing Chinese or phonetic is ever recorded. + +**Settings**: four `defaults write io.github.lmanchu.inputmethod.bopomix -bool false` switches — `MixedScriptEnabled`, `LatinCompletionEnabled`, `LatinLearnTypedWords`, `MixedScriptLatinOnSpaceForUserWords`. + +**Limits**: the Chinese/English decision is rule-based, not a language model; completion ranking is coarse beyond the top few; deleting a learned word takes effect after the input method restarts. See [CHANGELOG.md](CHANGELOG.md) for details and [CONTRIBUTING.md](CONTRIBUTING.md) to build from source. + +MIT License. Derived from McBopomofo, Copyright 2011-2026 Mengjuei Hsieh et al.; English word list from SCOWL. See [ACKNOWLEDGEMENTS.md](ACKNOWLEDGEMENTS.md). diff --git a/Source/Data/latin-tech-seed.txt b/Source/Data/latin-tech-seed.txt index db6faa8..dd151e3 100644 --- a/Source/Data/latin-tech-seed.txt +++ b/Source/Data/latin-tech-seed.txt @@ -1,5 +1,5 @@ # Hand-curated tech/product term seed list for the P1 zh/en mixed-typing -# feature (see ~/.claude/plans/zhuyin-ime-personal.md's "P1 設計" section). +# feature (see the design notes' "P1 設計" section). # Loaded on top of latin-words.txt (see tools/lexicon/build_lexicon.py) so # these terms are guaranteed present even though many are not real English # dictionary words and would never appear in a system word list. Rank @@ -24,15 +24,7 @@ github 12 notion 13 zoom 14 xcode 15 -irisgo 16 openrouter 17 -tachikoma 18 -hermes 19 -gloria 20 -apollo 21 -argus 22 -wells 23 -magi 24 computex 25 oem 26 dau 27 diff --git a/Source/Engine/McBopomofoLM.h b/Source/Engine/McBopomofoLM.h index 7968eaf..43413df 100644 --- a/Source/Engine/McBopomofoLM.h +++ b/Source/Engine/McBopomofoLM.h @@ -111,7 +111,7 @@ class McBopomofoLM : public Formosa::Gramambular2::LanguageModel { std::function macroConverter); std::string convertMacro(const std::string& input) const; - // P1 zh/en mixed typing (see ~/.claude/plans/zhuyin-ime-personal.md). + // P1 zh/en mixed typing (see the design notes). // When disabled (the default), getUnigrams()/hasUnigrams() ignore // mixedScriptLM() entirely, so mixedScriptEnabled=false leaves existing // behavior byte-for-byte unchanged. KeyHandler drives mixedScriptLM() diff --git a/Source/Engine/MixedScript/CMakeLists.txt b/Source/Engine/MixedScript/CMakeLists.txt index 779ee23..32e5e1e 100644 --- a/Source/Engine/MixedScript/CMakeLists.txt +++ b/Source/Engine/MixedScript/CMakeLists.txt @@ -4,7 +4,7 @@ project(MixedScript) set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC") -# P1 zh/en mixed-typing (see ~/.claude/plans/zhuyin-ime-personal.md). Mirrors +# P1 zh/en mixed-typing (see the design notes). Mirrors # the Mandarin/ and gramambular2/ subdirectory pattern: a small static lib # plus its own gtest binary, consumed by McBopomofoLMLib (McBopomofoLM # merges LatinPassthroughLM into its unigram dispatch) and by KeyHandler.mm diff --git a/Source/Engine/MixedScript/bopomofo_shape_tracker.h b/Source/Engine/MixedScript/bopomofo_shape_tracker.h index 443b2cd..b86aa2d 100644 --- a/Source/Engine/MixedScript/bopomofo_shape_tracker.h +++ b/Source/Engine/MixedScript/bopomofo_shape_tracker.h @@ -51,7 +51,7 @@ namespace McBopomofo::MixedScript { // (there is only one consonant/medial/vowel slot per syllable, and a tone // key -- which is exempt from this check -- always ends the syllable and // is composed immediately by KeyHandler, which resets this tracker before -// the next key). See zhuyin-ime-personal.md's P1 design section for the +// the next key). See the design notes' P1 design section for the // quantification this mirrors (~74% of English tokens become // "uncomposable" within 2-3 letters) and the explicit "th"/"sl" examples. // diff --git a/Source/Engine/MixedScript/bopomofo_shape_tracker_test.cpp b/Source/Engine/MixedScript/bopomofo_shape_tracker_test.cpp index cd85ee8..941e6c2 100644 --- a/Source/Engine/MixedScript/bopomofo_shape_tracker_test.cpp +++ b/Source/Engine/MixedScript/bopomofo_shape_tracker_test.cpp @@ -67,7 +67,7 @@ TEST(BopomofoShapeTrackerTest, ToneKeyNeverConflicts) { // "th": t -> CH (consonant), h -> C (consonant). Two different consonants // in a row is the required early-reject example from -// zhuyin-ime-personal.md's P1 design section. +// the design notes' P1 design section. TEST(BopomofoShapeTrackerTest, ThRejectsAtSecondLetter) { BopomofoShapeTracker tracker; EXPECT_TRUE(tracker.feed(Standard(), 't')); diff --git a/Source/Engine/MixedScript/latin_lexicon.h b/Source/Engine/MixedScript/latin_lexicon.h index 453a4b2..5bf36cc 100644 --- a/Source/Engine/MixedScript/latin_lexicon.h +++ b/Source/Engine/MixedScript/latin_lexicon.h @@ -25,7 +25,7 @@ // MixedScriptTracker (see mixed_script_tracker.h) to tell whether a run of // ASCII letters typed without leaving Bopomofo mode is more likely an // English word than a Chinese reading. See -// ~/.claude/plans/zhuyin-ime-personal.md's "P1 設計" section for the +// the design notes' "P1 設計" section for the // product rationale (rules A/B/C). #ifndef SRC_ENGINE_MIXEDSCRIPT_LATIN_LEXICON_H_ @@ -141,7 +141,7 @@ class LatinLexicon { // trailing space: the built-in list is far too permissive for that (a // two-letter run like "up"/"el" is both an English word and an extremely // common tone-1 syllable), whereas a word in this store is one the user - // has personally disambiguated before. See zhuyin-ime-personal.md's P1 + // has personally disambiguated before. See the design notes' P1 // section and docs/REVIEW-P1-2026-09-10.md's B1/B2. bool isUserWord(const std::string& text) const; @@ -156,7 +156,7 @@ class LatinLexicon { // sorted lazily on first use, which used to cost ~294 ms on the key // thread the first time anything called this. Not on P1's hot path // (MixedScriptTracker's decision logic never calls it); kept for P3's - // predictive-typing work, see zhuyin-ime-personal.md's F3 scope. + // predictive-typing work, see the design notes' F3 scope. bool isPrefix(const std::string& text) const; // Returns the word's rank (0 = most frequent) or -1 if `text` is not a @@ -196,7 +196,7 @@ class LatinLexicon { // "th" completing normally. int sourceTier(const std::string& text) const; - // P3 predictive typing (see zhuyin-ime-personal.md's F3 scope). Returns + // P3 predictive typing (see the design notes' F3 scope). Returns // up to `n` known words that start with (but are longer than) `prefix`, // best completion first. Ordering: a *confirmed* user word (score >= // kUserWordConfirmedScore) beats every builtin word, ranked among diff --git a/Source/Engine/MixedScript/latin_passthrough_lm.h b/Source/Engine/MixedScript/latin_passthrough_lm.h index 439e58e..2a6d7f0 100644 --- a/Source/Engine/MixedScript/latin_passthrough_lm.h +++ b/Source/Engine/MixedScript/latin_passthrough_lm.h @@ -85,7 +85,7 @@ class LatinPassthroughLM : public Formosa::Gramambular2::LanguageModel { // window -- both funnel through fixNodeWithReading:) to tell whether the // user just explicitly picked a mixedScript Latin candidate, which is // when (and only when) LatinLexicon::rememberWord() should be called -- - // see zhuyin-ime-personal.md's P1 design notes on why the automatic + // see the design notes' P1 design notes on why the automatic // dictionary-plus-space default must NOT itself count as learning. bool hasValue(const std::string& key, const std::string& value) const; diff --git a/Source/Engine/MixedScript/mixed_script_tracker.h b/Source/Engine/MixedScript/mixed_script_tracker.h index 7a8de41..bcc6bf0 100644 --- a/Source/Engine/MixedScript/mixed_script_tracker.h +++ b/Source/Engine/MixedScript/mixed_script_tracker.h @@ -33,7 +33,7 @@ namespace McBopomofo::MixedScript { // The three-way call for one contiguous run of ASCII letters typed without -// leaving Bopomofo mode. See zhuyin-ime-personal.md's P1 design section +// leaving Bopomofo mode. See the design notes' P1 design section // for the product rationale. enum class Verdict { // No English signal (yet). Caller keeps feeding the real @@ -148,7 +148,7 @@ class MixedScriptTracker { // way -- this only touches mixedScript's own bookkeeping. void popLastLatinChar(); - // P3 predictive typing (see zhuyin-ime-personal.md's F3 scope and + // P3 predictive typing (see the design notes' F3 scope and // LatinLexicon::complete()): replaces the pending run's text with a // longer completion the user just accepted (Tab, or a pick from the // completion candidate window -- see KeyHandler's mixed-script hookup). @@ -166,7 +166,7 @@ class MixedScriptTracker { void acceptCompletion(const std::string& word); // P3 "learn from what you actually type" (see - // zhuyin-ime-personal.md's P3 fix #2): true if the run's *current* text + // the design notes' P3 fix #2): true if the run's *current* text // was already explicitly written to the user's lexicon by a completion // accept (acceptCompletion()) and has not changed since. KeyHandler's // _commitMixedScriptLatinRun uses this to avoid double-counting a word diff --git a/Source/Engine/MixedScript/mixed_script_tracker_test.cpp b/Source/Engine/MixedScript/mixed_script_tracker_test.cpp index 6f9714a..f13d5f3 100644 --- a/Source/Engine/MixedScript/mixed_script_tracker_test.cpp +++ b/Source/Engine/MixedScript/mixed_script_tracker_test.cpp @@ -320,7 +320,7 @@ TEST(MixedScriptTrackerTest, PopLastLatinCharAfterAcceptingCompletion) { EXPECT_EQ(tracker.latinRun(), "the"); } -// P3 fix #2 (see zhuyin-ime-personal.md's P3 fix #2 and +// P3 fix #2 (see the design notes' P3 fix #2 and // LatinLexicon::rememberWord()'s use-count doc): acceptCompletion() marks // the run as already remembered so KeyHandler's natural-typing learn hook // does not double-count it when the same run later reaches an ordinary diff --git a/Source/InputMethodController+CandidateControllerDelegate.swift b/Source/InputMethodController+CandidateControllerDelegate.swift index 9bf2678..773fba2 100644 --- a/Source/InputMethodController+CandidateControllerDelegate.swift +++ b/Source/InputMethodController+CandidateControllerDelegate.swift @@ -57,7 +57,7 @@ extension BopomixInputMethodController: CandidateControllerDelegate { let selectedCandidate = state.candidates[Int(index)] // P3 English prediction + Tab completion (see - // ~/.claude/plans/zhuyin-ime-personal.md's F3 scope): a pick + // the design notes' F3 scope): a pick // from the completion candidate window is not a grid // position fixNode(...) could override (see // KeyHandler.h's acceptLatinCompletionWithValue: doc), so it diff --git a/Source/KeyHandler.h b/Source/KeyHandler.h index 7f42f60..bff91ef 100644 --- a/Source/KeyHandler.h +++ b/Source/KeyHandler.h @@ -78,7 +78,7 @@ extern InputMode InputModePlainBopomofo; associatedPhraseValue:(NSString *)phraseValue; // P3 English prediction + Tab completion (see -// ~/.claude/plans/zhuyin-ime-personal.md's F3 scope): accepts `value` as +// the design notes' F3 scope): accepts `value` as // the pending Latin run's completion, the same way Tab does, but from a // pick in the completion candidate window (see // InputMethodController+CandidateControllerDelegate.swift's diff --git a/Source/KeyHandler.mm b/Source/KeyHandler.mm index de566d6..bc85868 100644 --- a/Source/KeyHandler.mm +++ b/Source/KeyHandler.mm @@ -51,7 +51,7 @@ InputMode InputModePlainBopomofo = @"io.github.lmanchu.inputmethod.bopomix.PlainBopomofo"; // P3 English prediction + Tab completion (see -// ~/.claude/plans/zhuyin-ime-personal.md's F3 scope). The synthetic +// the design notes' F3 scope). The synthetic // "reading" every candidate in the completion candidate window carries -- // never a real Bopomofo reading or grid position, so it doubles as this // window's marker: _isLatinCompletionCandidateState: and the Swift @@ -103,7 +103,7 @@ @implementation KeyHandler { NSString *_inputMode; - // P1 zh/en mixed typing (see ~/.claude/plans/zhuyin-ime-personal.md). + // P1 zh/en mixed typing (see the design notes). McBopomofo::MixedScript::MixedScriptTracker *_mixedScriptTracker; // The (reading, value) pairs this composition registered as rule-B @@ -133,7 +133,7 @@ - (void)setInputMode:(NSString *)value newLanguageModel = [LanguageModelManager languageModelPlainBopomofo]; newLanguageModel->setPhraseReplacementEnabled(false); // P1 zh/en mixed typing is scoped to InputModeBopomofo only (see - // zhuyin-ime-personal.md's P1 design section); Plain Bopomofo gets + // the design notes' P1 design section); Plain Bopomofo gets // no English-awareness in this pass. newLanguageModel->setMixedScriptEnabled(false); } else { @@ -565,7 +565,7 @@ - (BOOL)handleInput:(KeyHandlerInput *)input state:(InputState *)inState stateCa // see if it's valid BPMF reading bool isValidKey = _bpmfReadingBuffer->isValidKey((char)charCode); - // MARK: P1 zh/en mixed typing (see ~/.claude/plans/zhuyin-ime-personal.md) + // MARK: P1 zh/en mixed typing (see the design notes) // // Only lowercase ASCII letters are ever fed to the tracker. Shift+letter // already took the CapsLock/Shift-forces-English path above (:391-405) @@ -1162,7 +1162,7 @@ - (BOOL)handleInput:(KeyHandlerInput *)input state:(InputState *)inState stateCa return NO; } -// MARK: P1 zh/en mixed typing (see ~/.claude/plans/zhuyin-ime-personal.md) +// MARK: P1 zh/en mixed typing (see the design notes) // Commits the tracker's pending Latin run (rule A) into the grid as a // single literal-text node and resets the tracker. Callers are @@ -1201,7 +1201,7 @@ - (void)_commitMixedScriptLatinRun:(BOOL)mayLearn return; } - // P3 fix #2 (see ~/.claude/plans/zhuyin-ime-personal.md's P3 fix #2): + // P3 fix #2 (see the design notes' P3 fix #2): // this is the single chokepoint every Rule-A run's boundary commit // passes through, so it is also the right place to learn from what // the user actually typed, not just what they explicitly accepted a @@ -1251,7 +1251,7 @@ - (void)_rememberLatinWord:(const std::string&)word weight:(int)weight } } -// P3 fix #2 (see ~/.claude/plans/zhuyin-ime-personal.md's P3 fix #2): +// P3 fix #2 (see the design notes' P3 fix #2): // "learn from what you actually type", not only from an explicit // completion accept. Called only from _commitMixedScriptLatinRun, i.e. // only for a Rule-A run that reached an ordinary boundary commit -- Esc @@ -1358,7 +1358,7 @@ - (BOOL)_mixedScriptAvailable } // P3 English prediction + Tab completion (see -// ~/.claude/plans/zhuyin-ime-personal.md's F3 scope): a strict narrowing +// the design notes' F3 scope): a strict narrowing // of _mixedScriptAvailable, so turning mixedScript off (or an unsupported // keyboard layout) disables completion the same way it disables // everything else P1 added, with no separate gate to keep in sync. @@ -1386,7 +1386,7 @@ - (BOOL)_isFinishedWord:(const std::string&)word against:(const std::string&)top // completion, or an empty string when there is none or when `word` is // already a finished word. // -// "Finished" (P3 fix #3, see ~/.claude/plans/zhuyin-ime-personal.md) +// "Finished" (P3 fix #3, see the design notes) // means all three of: // // 1. at least kMinFinishedLatinWordLength characters. A one- or @@ -1459,7 +1459,7 @@ - (BOOL)_isMixedScriptAlternateWithReading:(NSString *)reading value:(NSString * } // MARK: P3 English prediction + Tab completion (see -// ~/.claude/plans/zhuyin-ime-personal.md's F3 scope) +// the design notes' F3 scope) // Builds up to Preferences.candidateKeys.length completions for the // pending Latin run as a candidate window, or nil if there are none to @@ -2366,7 +2366,7 @@ - (BOOL)_handleCandidateState:(InputState *)state VTCandidateController *gCurrentCandidateController = [self.delegate candidateControllerForKeyHandler:self]; // MARK: P3 Latin completion candidate window (see - // ~/.claude/plans/zhuyin-ime-personal.md's F3 scope). Not grid-backed + // the design notes' F3 scope). Not grid-backed // -- the pending run it lists completions for has not been committed // to _grid (see handleInput:'s Tab branch) -- so it needs its own, // narrower handling instead of this method's grid-cursor and @@ -3543,7 +3543,7 @@ - (InputStateInputting *)buildInputtingState ? @(_mixedScriptTracker->latinRun().c_str()) : @(_bpmfReadingBuffer->composedString().c_str()); - // P3 English prediction (see ~/.claude/plans/zhuyin-ime-personal.md's + // P3 English prediction (see the design notes' // F3 scope): show the top-1 completion as a tooltip, never touching // the composing buffer itself -- unlike the candidate window, this // must not pop up on every single English letter. complete() already diff --git a/Source/LanguageModelManager+Privates.h b/Source/LanguageModelManager+Privates.h index c4f7921..515cfb1 100644 --- a/Source/LanguageModelManager+Privates.h +++ b/Source/LanguageModelManager+Privates.h @@ -34,7 +34,7 @@ NS_ASSUME_NONNULL_BEGIN @property (class, readonly, nonatomic) McBopomofo::McBopomofoLM *languageModelPlainBopomofo; @property (class, readonly, nonatomic) McBopomofo::UserOverrideModel *userOverrideModel; @property (class, readonly, nonatomic) McBopomofo::VariantAnnotator *variantAnnotator; -// P1 zh/en mixed typing (see ~/.claude/plans/zhuyin-ime-personal.md). +// P1 zh/en mixed typing (see the design notes). // The bundled latin-words.txt / latin-tech-seed.txt plus the user's own // latin-user.txt (same folder as McBopomofo's user phrases -- see // +latinUserWordListPath). The first access kicks off a background load diff --git a/Source/LanguageModelManager.mm b/Source/LanguageModelManager.mm index cf1d003..9677c8f 100644 --- a/Source/LanguageModelManager.mm +++ b/Source/LanguageModelManager.mm @@ -93,7 +93,7 @@ static void LTLoadVariantAnnotatorData() } } -// P1 zh/en mixed typing (see ~/.claude/plans/zhuyin-ime-personal.md). +// P1 zh/en mixed typing (see the design notes). // Loads the two bundled word lists plus the user's own learned-word file // into gLatinLexicon. Safe to call more than once; only the first call // does any work. This never touches gLanguageModelMcBopomofo's @@ -132,7 +132,7 @@ static void LTLoadMixedScriptLexicon() NSString *userPath = [LanguageModelManager latinUserWordListPath]; dispatch_async(dispatch_get_global_queue(QOS_CLASS_UTILITY, 0), ^{ - // P3 predictive typing (see zhuyin-ime-personal.md's F3 scope): + // P3 predictive typing (see the design notes' F3 scope): // tech-seed is loaded *before* the dictionary so its hand-ranked // terms always outrank every dictionary word regardless of the // dictionary's own SCOWL-tier rank -- see diff --git a/Source/Preferences.swift b/Source/Preferences.swift index b315ca2..7849987 100644 --- a/Source/Preferences.swift +++ b/Source/Preferences.swift @@ -40,17 +40,17 @@ private let kChineseConversionEnabledKey = "ChineseConversionEnabled" private let kHalfWidthPunctuationEnabledKey = "HalfWidthPunctuationEnable" private let kEscToCleanInputBufferKey = "EscToCleanInputBuffer" private let kKeepReadingUponCompositionError = "KeepReadingUponCompositionError" -// P1 zh/en mixed typing (see ~/.claude/plans/zhuyin-ime-personal.md). No UI +// P1 zh/en mixed typing (see the design notes). No UI // yet (planned for P4) -- these are UserDefaults-only for now, per // AGENTS.md's Preferences convention. private let kMixedScriptEnabledKey = "MixedScriptEnabled" private let kMixedScriptLatinOnSpaceForUserWordsKey = "MixedScriptLatinOnSpaceForUserWords" -// P3 English prediction + Tab completion (see zhuyin-ime-personal.md's F3 +// P3 English prediction + Tab completion (see the design notes' F3 // scope). Gated on kMixedScriptEnabledKey too (see KeyHandler's // _mixedScriptAvailable) -- this only decides whether completion runs on // top of an already-available mixedScript run. private let kLatinCompletionEnabledKey = "LatinCompletionEnabled" -// P3 fix #2 (see zhuyin-ime-personal.md's P3 fix #2): learn from Latin +// P3 fix #2 (see the design notes' P3 fix #2): learn from Latin // runs actually typed and committed, not only from an explicit Tab/ // candidate-window completion accept. private let kLatinLearnTypedWordsKey = "LatinLearnTypedWords" @@ -358,14 +358,14 @@ class Preferences: NSObject { @UserDefault(key: kKeepReadingUponCompositionError, defaultValue: false) @objc static var keepReadingUponCompositionError: Bool - // MARK: P1 zh/en mixed typing (see zhuyin-ime-personal.md) + // MARK: P1 zh/en mixed typing (see the design notes) - /// Master switch for zh/en mixed typing. Opt-in for now (P1): - /// `defaults write io.github.lmanchu.inputmethod.bopomix - /// MixedScriptEnabled -bool true`. Off, every mixedScript code path - /// short-circuits and the input method behaves exactly like upstream - /// McBopomofo. - @UserDefault(key: kMixedScriptEnabledKey, defaultValue: false) + /// Master switch for zh/en mixed typing. On by default -- it is what + /// Bopomix is for; `defaults write io.github.lmanchu.inputmethod.bopomix + /// MixedScriptEnabled -bool false` turns it off. Off, every mixedScript + /// code path short-circuits and the input method behaves exactly like + /// upstream McBopomofo. + @UserDefault(key: kMixedScriptEnabledKey, defaultValue: true) @objc static var mixedScriptEnabled: Bool /// Whether a run that is a word in the user's *own* Latin lexicon @@ -386,7 +386,7 @@ class Preferences: NSObject { @objc static var mixedScriptLatinOnSpaceForUserWords: Bool // MARK: P3 English prediction + Tab completion (see - // zhuyin-ime-personal.md's F3 scope) + // the design notes' F3 scope) /// Whether a Latin run shows a top-1 completion tooltip and Tab/ /// Shift+Tab accept/cycle it. Defaults to on (unlike diff --git a/docs/REVERIFY-P1-2026-09-10.md b/docs/REVERIFY-P1-2026-09-10.md index 2c72d7a..65fd23b 100644 --- a/docs/REVERIFY-P1-2026-09-10.md +++ b/docs/REVERIFY-P1-2026-09-10.md @@ -16,7 +16,7 @@ 本輪新挖出的問題，其中 1 條 blocking。** - **需修後 merge** — 唯一的 blocking 是一行 gate（`mixedScriptActive` 沒檢查鍵盤配置）。 -- **可裝 dogfood，條件：鍵盤配置＝標準（大千）**。Lman 這台 plist 是 `KeyboardLayout = 0` +- **可裝 dogfood，條件：鍵盤配置＝標準（大千）**。維護者這台 plist 是 `KeyboardLayout = 0` （標準），所以裝了就是安全的。**若使用者是許氏或倚天26，開啟本功能會讓中文輸入 幾乎完全失效**（實測 199/200 句全錯，字元正確率 95.8% → 0.3%）。 diff --git a/docs/REVERIFY-P3-2026-09-12.md b/docs/REVERIFY-P3-2026-09-12.md index d4acbc8..dc7aadc 100644 --- a/docs/REVERIFY-P3-2026-09-12.md +++ b/docs/REVERIFY-P3-2026-09-12.md @@ -3,7 +3,7 @@ 複驗對象：`f3-latin-completion` @ `72b5e0f`（round 3：`223817c` + `72b5e0f`） 上一輪：`docs/REVIEW-P3-2026-09-11.md`（B1–B5 blocking、N1–N17 non-blocking） 複驗方式：fresh context、只讀、自寫重現序列（真 `KeyHandler` XCTest）、6 個 mutation -機器：Mac Studio（Iris），Xcode 26.6，未安裝任何 build，未 `killall McBopomofo` +機器：Mac Studio，Xcode 26.6，未安裝任何 build，未 `killall McBopomofo` --- @@ -58,7 +58,7 @@ P1 時代是 append，所以換資料夾只會合併不會刪。 - plist 沒被改（`PLIST_UNCHANGED`），**但** - `~/Library/Application Support/McBopomofo/`（跑之前不存在）被建立， 裡面 `latin-user.txt` 1.1K，內容是 eval200 corpus 的 130 個詞、score 2： - `irisgo 22 / google 28 / computex 12 / acer 10 / asus 10 / tony 8 / claude 8 / phison 6 …` + ` 22 / google 28 / computex 12 / 10 / 10 / 8 / 8 / 6 …` - 檔案 mtime `10:05:13`，parallel run 的 xcresult 是 `Test-McBopomofo-2026.09.12_10-05-08` ——parallel 跑起來 5 秒就寫下去了 @@ -103,7 +103,7 @@ EnableUserPhrasesInPlainBopomofo, AllowMovingCursorWhenChoosingCandidates **這算不算污染：算，而且有實害。** 你機器上現在的值是： ``` -AddPhraseHookPath = /Users/lman/dev/mixime/build/Build/Products/Debug/McBopomofo.app/Contents/Resources/add-phrase-hook.sh +AddPhraseHookPath = ~/dev/mixime/build/Build/Products/Debug/McBopomofo.app/Contents/Resources/add-phrase-hook.sh ``` 安裝的輸入法在 `~/Library/Input Methods/McBopomofo.app`，這個路徑只可能是測試 host 寫的。 @@ -367,9 +367,9 @@ never completable 155/330 **抽樣 30 個 ranking miss，全部 `isLatinWord(lowercased)` = true**（都真的在詞典、小寫後在）： ``` -Games SOW mode SLA Iris Sunny Tool Meet Tony per Stripe PRO Tony love you -Community comments cli Rita Lucy More Electron Wanda redline code RAG Tony -Abby App Family +Games SOW mode SLA Tool Meet per Stripe PRO love you +Community comments cli More Electron redline code RAG + App Family ``` **但其中 15 個從頭到尾沒有 lock 成 Latin run**（見 P-4）： @@ -453,8 +453,8 @@ KeyHandlerBopomofoTests.swift:844/868/872: (AssociatedPhrases state) 決定性的一條：`latin-user.txt` 的斷言失敗訊息裡，出現的是**別的 worker 的**詞： ``` -2 letters is below the floor: ["apple":4, "nitrosense":4, "tony":10, "irixion":2, - "biilabs":2, "lecun":2, "audrey":2, "cofounder":2, ...] +2 letters is below the floor: ["apple":4, "":4, "":10, "":2, + "":2, "":2, "":2, "cofounder":2, ...] ``` 我的測試讀的是自己 UUID 的 throwaway 資料夾，裡面卻出現 eval corpus 的詞—— @@ -527,7 +527,7 @@ P-3 / P-4 / P-5 可以進 merge 後的 follow-up，不擋安裝。 暫時附在 `McBopomofoTests/LatinCompletionKeyHandlerTests.swift`，**已全部清除** - 執行：4 次 serial full/subset run、1 次 full parallel run、6 次 mutation run、 1 次 C++ `MixedScriptTest`（81/81 過） -- 離線重建 lexicon 排序模型交叉驗算 tier / rank（`scratchpad/lex.py`） +- 離線重建 lexicon 排序模型交叉驗算 tier / rank（暫存目錄裡的一次性 script） - `git status` 乾淨（`git checkout -- . && git clean -fd` 已執行，本報告除外） - plist：`/tmp/rv-before.xml` vs `/tmp/rv-after.xml` **完全相同** - `~/Library/Application Support/McBopomofo/`：parallel run 建出來過，**已移除**， diff --git a/docs/REVIEW-P3-2026-09-11.md b/docs/REVIEW-P3-2026-09-11.md index 54d401a..a3a6ed7 100644 --- a/docs/REVIEW-P3-2026-09-11.md +++ b/docs/REVIEW-P3-2026-09-11.md @@ -510,7 +510,7 @@ th + Tab + space -> buffer 'throughput '，詞庫 throughput 計次不 所以新增的 283 個 2 字母詞是死的。Rule B 只是多加一列候選、不覆寫預設中文。 - 我自己測到一個語料外的純中文 ON≠OFF：`2k7apl3` → ON `的apl`、OFF `的卯`。 這是 **Rule A**（Rule A 從不查詞典），屬於 P1 既有的 ~9% 誤判，不是詞典置換造成的回歸—— - 但這正是 Lman 第一天會踩到的東西。 + 但這正是維護者第一天會踩到的東西。 - **SCOWL 授權**：見 N16。結論是**確實允許再散布這份衍生詞表**，引用原文逐字正確。 ### 4. `complete()` 排序與效能 @@ -596,13 +596,13 @@ th + Tab + space -> buffer 'throughput '，詞庫 throughput 計次不 - **154 個不可補全 token 抽樣人工分類**（抽 20）： ``` - Games SOW mode PRD SLA Phison Iris PCClaw Sunny Tool - Meet Tony per Lman EMBA Stripe PRO MOU Filco Tony + Games SOW mode PRD SLA Tool + Meet per EMBA Stripe PRO MOU ``` - 其中 `Games / mode / Tool / Meet / Tony / per / Stripe / PRO / Iris / Sunny`（10 個） + 其中 `Games / mode / Tool / Meet / / per / Stripe / PRO / / `（10 個） 小寫形都在 SCOWL 裡 → 是**排序失敗**； - `SOW / PRD / SLA / MOU / EMBA / PCClaw / Lman / Phison / Filco`（9 個）是真的縮寫／專有名詞。 + `SOW / PRD / SLA / MOU / EMBA / / / / `（9 個）是真的縮寫／專有名詞。 比例約 50:50，與全量統計（62% 排序失敗）方向一致 —— **報告的三分類不準確**。 ### 9. Mutation（存活率 2/6 = 33%） @@ -621,7 +621,7 @@ th + Tab + space -> buffer 'throughput '，詞庫 throughput 計次不 ### 10. 可裝 dogfood 判定 -**不可裝。** Lman 換裝這個 build 後第一天會撞到的前 5 件事： +**不可裝。** 維護者換裝這個 build 後第一天會撞到的前 5 件事： | # | 會發生什麼 | 來源 | 判定 | |---|---|---|---| diff --git a/tools/eval/BASELINE.md b/tools/eval/BASELINE.md index 5cdbf79..0f63628 100644 --- a/tools/eval/BASELINE.md +++ b/tools/eval/BASELINE.md @@ -7,7 +7,8 @@ unmodified F1/F2 logic -- this is the *baseline*, i.e. what the stock McBopomofo engine does today, before any zh/en mixed-typing or AI re-ranking work lands. -Corpus: `/Users/lman/Dev/mixime-private/eval200.tsv` -- 200 rows (vault=200, synthetic=0). +Corpus: the maintainer's private 200-sentence corpus (not in this repo; build +your own with `tools/eval/build_corpus.py`) -- 200 rows (vault=200, synthetic=0). Language model data: `build/Build/Products/Debug/Bopomix.app/Contents/Resources/data.txt` (sha256 0deae7b7c1dcde1d7a30d139e7068543e0c0e7112e944b63eb52947bca1db7ac). ## Commands @@ -15,7 +16,7 @@ Language model data: `build/Build/Products/Debug/Bopomix.app/Contents/Resources/ ``` cmake -S Source/Engine -B build-engine -DENABLE_TEST=ON cmake --build build-engine -python3 tools/eval/run_eval.py --corpus /Users/lman/Dev/mixime-private/eval200.tsv \ +python3 tools/eval/run_eval.py --corpus /corpus.tsv \ --cli build-engine/tools/eval/bopomix-eval --data build/Build/Products/Debug/Bopomix.app/Contents/Resources ``` @@ -36,7 +37,7 @@ each gold English token still appears literally in the composed output. | latency (avg / p50 / p95 / max) | 2475us / 2517us / 4308us / 6012us | This is expected to be near 0%: the current engine has no English-awareness -at all (see zhuyin-ime-personal.md's F1 scope). Every English letter is +at all (see the design notes' F1 scope). Every English letter is also a valid standard-layout Bopomofo key, so an English word typed without switching modes is either silently dropped (composition fails `hasUnigrams`) or misrecognized as unrelated Chinese character(s) -- it is @@ -67,7 +68,7 @@ Generated: 2026-09-10T17:21:35+08:00 Same corpus and language model as the P0.5 baseline above, run with `--mixed on` (Source/Engine/MixedScript/, see -~/.claude/plans/zhuyin-ime-personal.md's P1 design section) instead of the +the design notes' P1 design section) instead of the baseline's unmodified engine. **These numbers are not acceptance criteria.** `bopomix-eval` reimplements @@ -82,7 +83,7 @@ section as an engine regression check only. ### Commands ``` -python3 tools/eval/run_eval.py --corpus /Users/lman/Dev/mixime-private/eval200.tsv \ +python3 tools/eval/run_eval.py --corpus /corpus.tsv \ --cli build-engine/tools/eval/bopomix-eval --data build/Build/Products/Debug/Bopomix.app/Contents/Resources \ --mixed on --lexicon-dir Source/Data ``` diff --git a/tools/eval/README.md b/tools/eval/README.md index d44d7a7..0395c9e 100644 --- a/tools/eval/README.md +++ b/tools/eval/README.md @@ -1,7 +1,7 @@ # bopomix eval harness (P0.5) Headless tooling to measure the engine's zh/en mixed-typing behavior -without an installed input method, per `~/.claude/plans/zhuyin-ime-personal.md`'s +without an installed input method, per the design notes' "build the harness before the feature" principle. Everything here talks to the *real* C++ engine (`Source/Engine/`) -- nothing is re-implemented in Python; Python only orchestrates the compiled CLI and does filtering/scoring. @@ -32,7 +32,7 @@ bopomix-eval --data --mode {keys|readings|keyseq} [--layout stand [--mixed on|off] [--lexicon-dir ] ``` -`--mixed on` (P1, see `zhuyin-ime-personal.md`'s P1 design section) drives +`--mixed on` (P1, see the design notes' P1 design section) drives `keys` mode through the same `Source/Engine/MixedScript/` decision engine KeyHandler.mm uses for zh/en mixed typing -- rule A (structurally impossible Bopomofo shape) and rule B (dictionary word, which only adds a @@ -59,7 +59,7 @@ once a tone marker lands (or on a space, or at end of line, simulating a trailing Enter), the pending reading is composed via `ReadingGrid:: insertReading` and, at the very end, `ReadingGrid::walk()` picks the best path. It does **not** add any English-detection logic -- that gap is -exactly what F1 (see zhuyin-ime-personal.md) is meant to close, so this +exactly what F1 (see the design notes) is meant to close, so this harness can measure it honestly. Concretely: every ASCII letter is *also* a valid standard-layout Bopomofo key, so typing an English word without switching modes gets swallowed into the reading buffer and either silently @@ -157,7 +157,24 @@ Two other divergences worth knowing about, both deliberate: acceptance measurement lives in `BopomixTests/MixedScriptKeyHandlerTests.swift` (`testEval200ThroughKeyHandler`), which types the same corpus into a - real `KeyHandler`. + real `KeyHandler`. That test and + `LatinCompletionKeyHandlerTests.testEval200LatinCompletion` read the + corpus from the `BOPOMIX_EVAL_CORPUS` environment variable and skip + themselves when it is unset or points at a file that does not exist: + + ``` + BOPOMIX_EVAL_CORPUS=/corpus.tsv \ + tools/eval/check_plist_unchanged.sh \ + xcodebuild -project Bopomix.xcodeproj -scheme Bopomix \ + -configuration Debug -derivedDataPath build \ + CODE_SIGN_IDENTITY="-" CODE_SIGNING_REQUIRED=NO DEVELOPMENT_TEAM="" test + ``` + + `xcodebuild` does not pass its own environment to the test host: only + `TEST_RUNNER_` variables get through, with the prefix stripped. + `check_plist_unchanged.sh` re-exports `BOPOMIX_EVAL_CORPUS` under that + name for you; calling `xcodebuild` directly needs + `TEST_RUNNER_BOPOMIX_EVAL_CORPUS=...` instead. ## `build_corpus.py` @@ -168,8 +185,8 @@ out-of-repo** candidate file (see Privacy below). python3 tools/eval/build_corpus.py \ --cli ./build-engine/tools/eval/bopomix-eval \ --data \ - --candidates ~/Dev/mixime-private/corpus_candidates.txt \ - --output ~/Dev/mixime-private/eval200.tsv + --candidates /corpus_candidates.txt \ + --output /corpus.tsv ``` Pipeline: @@ -210,7 +227,7 @@ list-marker glyph like "O" or "V") can still show up as its own one-letter `en` segment in a small number of rows. Not fixed in this pass; flagged for follow-up rather than silently ignored. -### Corpus format (`eval200.tsv`) +### Corpus format (`corpus.tsv`) Six tab-separated columns, no header: @@ -233,7 +250,7 @@ id sentence segments(JSON) readings keys source format artifact, not keystrokes -- see "Why a space after every syllable" above before feeding this column to anything that models the real key handling. -- `source`: `vault` (derived from Lman's own text, see Privacy) or +- `source`: `vault` (derived from your own text, see Privacy) or `synthetic` (hand-written template, safe to publish). ## `run_eval.py` @@ -242,7 +259,7 @@ Runs the two baseline metrics over a corpus TSV and writes `BASELINE.md`. ``` python3 tools/eval/run_eval.py \ - --corpus ~/Dev/mixime-private/eval200.tsv \ + --corpus /corpus.tsv \ --cli ./build-engine/tools/eval/bopomix-eval \ --data ``` @@ -273,9 +290,10 @@ whether the failure is context-dependent or a genuine rule/dictionary gap. ## Privacy -`~/Dev/mixime-private/` (candidates and the generated `eval200.tsv`) can -contain Lman's own text and must **never** enter this repo. The repo's -`.gitignore` has a blanket `*.private.*` rule as a backstop, and +The candidate file and the generated corpus TSV can contain your own text +and must **never** enter this repo. Keep them outside the working tree, or +under the ignored `eval-corpus/` directory. The repo's `.gitignore` has a +blanket `*.private.*` rule and an `eval-corpus/` rule as backstops, and `tools/eval/fixtures/sample10.tsv` (10 synthetic-only rows, regenerated by `build_corpus.py`) is the only corpus-shaped file meant to be committed here. Do not add a `--output` pointing inside this repo. diff --git a/tools/eval/bopomix_eval.cpp b/tools/eval/bopomix_eval.cpp index 3333681..4509478 100644 --- a/tools/eval/bopomix_eval.cpp +++ b/tools/eval/bopomix_eval.cpp @@ -90,7 +90,7 @@ void PrintUsage(const char* argv0) { << "\n" << "--mixed on drives `keys` mode through the same " "Source/Engine/MixedScript/ decision engine KeyHandler.mm uses " - "(see zhuyin-ime-personal.md's P1 section); it is a no-op for " + "(see the design notes' P1 section); it is a no-op for " "readings/keyseq. --mixed on requires --lexicon-dir " "containing latin-words.txt and latin-tech-seed.txt (see " "tools/lexicon/build_lexicon.py and Source/Data/).\n"; @@ -139,7 +139,7 @@ bool ParseArgs(int argc, char** argv, Args* outArgs, std::string* error) { } if (outArgs->layout != "standard") { // Only the standard (dayi... actually "Da Chien"/Standard) layout is - // supported for now. See zhuyin-ime-personal.md P0.5 scope. + // supported for now. See the design notes' P0.5 scope. *error = "only --layout standard is supported by this eval harness"; return false; } diff --git a/tools/eval/build_corpus.py b/tools/eval/build_corpus.py index e273a30..45a2daa 100644 --- a/tools/eval/build_corpus.py +++ b/tools/eval/build_corpus.py @@ -2,9 +2,9 @@ """Build tools/eval's 200-sentence chat-style zh/en mixed-typing eval corpus. Reads candidate sentences from a private, out-of-repo file (see ---candidates; defaults to ~/Dev/mixime-private/corpus_candidates.txt, which -is mostly AI-generated summaries and Slack/gmail text, NOT sentences Lman -actually typed), filters them down to short, colloquial, zh/en-mixed chat +--candidates; the maintainer's own set was mostly AI-generated summaries and +Slack/gmail text, NOT sentences the maintainer actually typed), filters them +down to short, colloquial, zh/en-mixed chat sentences, and pads the remainder up to --target with synthetic sentences generated from templates (clearly tagged source=synthetic). @@ -20,9 +20,9 @@ 4. Writes id / sentence / segments (JSON) / readings / keys / source to the output TSV. -Privacy: the output TSV can contain Lman's own text (via the vault-derived -candidates) and MUST stay under ~/Dev/mixime-private/ (see repo .gitignore's -`*.private.*` rule and tools/eval/README.md). Only tools/eval/fixtures/ +Privacy: the output TSV can contain your own text (via the vault-derived +candidates) and MUST stay out of this repo (see repo .gitignore's +`*.private.*` and `eval-corpus/` rules and tools/eval/README.md). Only tools/eval/fixtures/ sample10.tsv (synthetic-only, 10 rows) is meant to ever enter this repo. """ @@ -37,6 +37,8 @@ import sys from pathlib import Path +REPO_ROOT = Path(__file__).resolve().parents[2] + # --------------------------------------------------------------------------- # Filtering # --------------------------------------------------------------------------- @@ -127,8 +129,11 @@ def segment(norm: str) -> list[dict]: # Synthetic sentence generation (used to pad up to --target) # --------------------------------------------------------------------------- +# Swap the product/company names here for the proper nouns that actually +# recur in your own text -- the synthetic padding is only useful if it +# looks like the English you really type. EN_TOKENS = [ - "Mac", "Slack", "PR", "API", "IrisGo", "OpenRouter", "GitHub", "Notion", + "Mac", "Slack", "PR", "API", "Figma", "OpenRouter", "GitHub", "Notion", "Zoom", "Xcode", "demo", "bug", "repo", "commit", "deploy", "cache", "token", "prompt", "agent", "model", "build", "merge", "review", "sprint", "CLI", "CI", "UI", "UX", "PM", "OKR", @@ -290,12 +295,14 @@ def main() -> int: parser.add_argument( "--candidates", type=Path, - default=Path("~/Dev/mixime-private/corpus_candidates.txt").expanduser(), + default=REPO_ROOT / "eval-corpus" / "corpus_candidates.txt", + help="private, out-of-VCS candidate sentences (one per line)", ) parser.add_argument( "--output", type=Path, - default=Path("~/Dev/mixime-private/eval200.tsv").expanduser(), + default=REPO_ROOT / "eval-corpus" / "corpus.tsv", + help="where to write the generated corpus TSV (never commit it)", ) parser.add_argument("--cli", type=Path, required=True, help="path to bopomix-eval") parser.add_argument("--data", type=Path, required=True, help="Resources dir with data.txt") @@ -374,7 +381,7 @@ def main() -> int: # tools/eval/fixtures/sample10.tsv is a repo-committed, synthetic-only # sample (see .gitignore's *.private.* rule and the module docstring). - # It is generated independently of whether eval200.tsv itself needed any + # It is generated independently of whether the corpus TSV itself needed any # synthetic padding, so it always has 10 rows. fixture_sentences = generate_synthetic( 10, avoid=seen_norm | {r["sentence"] for r in rows} diff --git a/tools/eval/check_plist_unchanged.sh b/tools/eval/check_plist_unchanged.sh index 6be7ecf..a72340c 100755 --- a/tools/eval/check_plist_unchanged.sh +++ b/tools/eval/check_plist_unchanged.sh @@ -38,6 +38,15 @@ set -u +# xcodebuild does not hand its own environment to the test host process: +# only variables named TEST_RUNNER_ get through, with the prefix +# stripped. So `BOPOMIX_EVAL_CORPUS=... check_plist_unchanged.sh xcodebuild +# ... test` would otherwise reach the eval tests as "unset" and skip them. +# Forward it under the name xcodebuild actually propagates. +if [ -n "${BOPOMIX_EVAL_CORPUS:-}" ]; then + export TEST_RUNNER_BOPOMIX_EVAL_CORPUS="${BOPOMIX_EVAL_CORPUS}" +fi + DOMAIN="io.github.lmanchu.inputmethod.bopomix" DATA_FOLDER="${HOME}/Library/Application Support/Bopomix" WORKDIR="$(mktemp -d -t bopomix-plist-guard)" diff --git a/tools/eval/run_eval.py b/tools/eval/run_eval.py index 54e8750..7d7d6bd 100644 --- a/tools/eval/run_eval.py +++ b/tools/eval/run_eval.py @@ -23,7 +23,7 @@ problem that F1 measures. Usage: - python3 tools/eval/run_eval.py --corpus \\ + python3 tools/eval/run_eval.py --corpus \\ --cli ./build-engine/tools/eval/bopomix-eval --data """ @@ -302,7 +302,7 @@ def stats_line(label: str, values: list[int]) -> str: import datetime if mixed: - # P1 (see zhuyin-ime-personal.md): append to the existing + # P1 (see the design notes): append to the existing # BASELINE.md rather than overwriting it, so the P0.5 baseline # numbers above stay intact for comparison. Run with --mixed off # first (the default) to (re)generate that baseline section. @@ -324,7 +324,7 @@ def stats_line(label: str, values: list[int]) -> str: Same corpus and language model as the P0.5 baseline above, run with `--mixed on` (Source/Engine/MixedScript/, see -~/.claude/plans/zhuyin-ime-personal.md's P1 design section) instead of the +the design notes' P1 design section) instead of the baseline's unmodified engine. **These numbers are not acceptance criteria.** `bopomix-eval` reimplements @@ -424,7 +424,7 @@ def stats_line(label: str, values: list[int]) -> str: | latency (avg / p50 / p95 / max) | {statistics.mean(f1_latencies_us):.0f}us / {percentile(f1_latencies_us, 0.5):.0f}us / {percentile(f1_latencies_us, 0.95):.0f}us / {max(f1_latencies_us)}us | This is expected to be near 0%: the current engine has no English-awareness -at all (see zhuyin-ime-personal.md's F1 scope). Every English letter is +at all (see the design notes' F1 scope). Every English letter is also a valid standard-layout Bopomofo key, so an English word typed without switching modes is either silently dropped (composition fails `hasUnigrams`) or misrecognized as unrelated Chinese character(s) -- it is diff --git a/tools/lexicon/build_lexicon.py b/tools/lexicon/build_lexicon.py index 5ad05ce..15357d8 100644 --- a/tools/lexicon/build_lexicon.py +++ b/tools/lexicon/build_lexicon.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 """Builds Source/Data/latin-words.txt for the zh/en mixed-typing feature -(see ~/.claude/plans/zhuyin-ime-personal.md's "P1 設計"/"P3 設計" sections +(see the design notes' "P1 設計"/"P3 設計" sections and Source/Engine/MixedScript/latin_lexicon.h). Source (2026-09-11 P3 fix-round rework): SCOWL/ESDB itself (the "Spell