Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 6 additions & 4 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,10 @@ codecov_comment.md
DerivedData
build-engine

# P0.5 eval harness: Lman's own text must never enter this repo. Any file
# matching *.private.* (corpus candidates, generated eval sets derived from
# personal vault/Slack text) stays under ~/Dev/mixime-private/ only.
# P0.5 eval harness: nobody's personal text may enter this repo. Corpus
# candidates and the eval TSVs generated from them are derived from private
# notes/chat logs, so keep them outside the repo (point BOPOMIX_EVAL_CORPUS
# at them) or under the ignored eval-corpus/ directory.
# See tools/eval/README.md.
*.private.*
*.private.*
eval-corpus/
4 changes: 2 additions & 2 deletions BopomixTests/KeyHandlerBopomofoTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,10 @@ class KeyHandlerBopomofoTests: XCTestCase {

var handler = KeyHandler()
// P3 fix #5 (XCTest/machine-preference isolation, see
// ~/.claude/plans/zhuyin-ime-personal.md's P3 fix #5): these upstream
// the design notes' P3 fix #5): these upstream
// tests know nothing about mixedScript and never save/restore its
// preferences themselves, so on a machine that already has
// MixedScriptEnabled=1 in its real plist (Lman dogfoods a master
// MixedScriptEnabled=1 in its real plist (the maintainer dogfoods a master
// build) they would otherwise inherit that value and see mixedScript
// rewrite readings it never used to (e.g. testInvalidBpmf's "ni4"
// becomes a Rule-A literal run instead of the upstream
Expand Down
27 changes: 19 additions & 8 deletions BopomixTests/LatinCompletionKeyHandlerTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ import XCTest
@testable import Bopomix

/// KeyHandler-level integration tests for P3 English prediction + Tab
/// completion (see ~/.claude/plans/zhuyin-ime-personal.md's F3 scope).
/// completion (see the design notes' F3 scope).
/// Mirrors MixedScriptKeyHandlerTests' approach (drive the real KeyHandler
/// FSM, not the engine in isolation) for the same reason P1's review found:
/// every interesting interaction here is with candidate-window/Tab/grid
Expand Down Expand Up @@ -353,7 +353,7 @@ class LatinCompletionKeyHandlerTests: XCTestCase {
XCTAssertNil(predictedCompletion, "\(state)")
}

/// P3 fix #3 (see ~/.claude/plans/zhuyin-ime-personal.md's P3 fix #3
/// P3 fix #3 (see the design notes' P3 fix #3
/// and KeyHandler's _offeredCompletionFor:lexicon:): once a pending run is
/// itself already a recognized word ranked at least as well as the
/// best longer completion sharing its prefix, that counts as "the
Expand Down Expand Up @@ -1503,9 +1503,11 @@ class LatinCompletionKeyHandlerTests: XCTestCase {
}

/// How many letters of an English token does the user actually have
/// to type before Tab would complete it, on eval200 (the only
/// Traditional-Mandarin-plus-real-project-vocabulary corpus this repo
/// has) -- measured two ways (P3 fix #4): "no history" evaluates
/// to type before Tab would complete it, on the corpus named by
/// `BOPOMIX_EVAL_CORPUS` (a Traditional-Mandarin-plus-real-project-
/// vocabulary TSV, kept outside this repo -- build one from your own
/// text with tools/eval/build_corpus.py) -- measured two ways
/// (P3 fix #4): "no history" evaluates
/// every token cold, against just the built-in dictionary/tech seed;
/// "with history" replays the corpus in row order, and after
/// evaluating each row's tokens, types and commits every eligible one
Expand All @@ -1514,14 +1516,23 @@ class LatinCompletionKeyHandlerTests: XCTestCase {
/// enough to be confirmed becomes more completable later in the
/// corpus. This is a measurement/report, not a
/// correctness gate (except the pure-Chinese control below): skipped
/// outright when the private corpus is not present, and neither table
/// outright when `BOPOMIX_EVAL_CORPUS` is unset, and neither table
/// asserts thresholds the way testEval200ThroughKeyHandler does --
/// there is no prior baseline to hold either to yet, this run creates
/// one for both.
func testEval200LatinCompletion() throws {
let corpusPath = NSHomeDirectory() + "/Dev/mixime-private/eval200.tsv"
guard let corpusPath = ProcessInfo.processInfo.environment["BOPOMIX_EVAL_CORPUS"],
!corpusPath.isEmpty
else {
throw XCTSkip(
"BOPOMIX_EVAL_CORPUS is not set; nothing to measure. Point it at a "
+ "corpus TSV built from your own text with tools/eval/build_corpus.py.")
}
guard FileManager.default.fileExists(atPath: corpusPath) else {
throw XCTSkip("corpus not present at \(corpusPath); nothing to measure")
throw XCTSkip(
"BOPOMIX_EVAL_CORPUS points at \(corpusPath), which does not exist; "
+ "nothing to measure. Build a corpus TSV from your own text with "
+ "tools/eval/build_corpus.py.")
}
let rows = try loadEvalRows(at: corpusPath)
XCTAssertFalse(rows.isEmpty)
Expand Down
20 changes: 14 additions & 6 deletions BopomixTests/MixedScriptKeyHandlerTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ import XCTest
@testable import Bopomix

/// KeyHandler-level integration tests for P1 zh/en mixed typing (see
/// ~/.claude/plans/zhuyin-ime-personal.md). The engine-level tests under
/// the design notes). The engine-level tests under
/// `Source/Engine/MixedScript/` cover the rules in isolation; everything
/// here drives the real `KeyHandler` FSM the way the input method does,
/// because every blocking defect found in the first review round
Expand Down Expand Up @@ -791,10 +791,18 @@ class MixedScriptKeyHandlerTests: XCTestCase {
/// dropping a whole run). This types the same corpus through the real
/// `KeyHandler` instead and is the number that counts.
func testEval200ThroughKeyHandler() throws {
let corpusPath =
NSHomeDirectory() + "/Dev/mixime-private/eval200.tsv"
guard let corpusPath = ProcessInfo.processInfo.environment["BOPOMIX_EVAL_CORPUS"],
!corpusPath.isEmpty
else {
throw XCTSkip(
"BOPOMIX_EVAL_CORPUS is not set; nothing to measure. Point it at a "
+ "corpus TSV built from your own text with tools/eval/build_corpus.py.")
}
guard FileManager.default.fileExists(atPath: corpusPath) else {
throw XCTSkip("corpus not present at \(corpusPath); nothing to measure")
throw XCTSkip(
"BOPOMIX_EVAL_CORPUS points at \(corpusPath), which does not exist; "
+ "nothing to measure. Build a corpus TSV from your own text with "
+ "tools/eval/build_corpus.py.")
}
let rows = try loadCorpus(at: corpusPath)
XCTAssertFalse(rows.isEmpty)
Expand Down Expand Up @@ -946,7 +954,7 @@ class MixedScriptKeyHandlerTests: XCTestCase {
/// R5: a rule-A run has a single candidate. Tab-cycling over it is not a
/// choice between Chinese and English and must not touch latin-user.txt.
///
/// P3 (see ~/.claude/plans/zhuyin-ime-personal.md's F3 scope) gives Tab
/// P3 (see the design notes' F3 scope) gives Tab
/// a new, deliberate job on a pending rule-A run -- completing it into
/// a longer dictionary word -- but P3 fix #3 also gives it a deliberate
/// *non*-job here: "acer" is tech-seed ranked ahead of every longer
Expand All @@ -973,7 +981,7 @@ class MixedScriptKeyHandlerTests: XCTestCase {
XCTAssertEqual(after, before, "Tab over a rule-A run must not remember the word")
}

/// P3 fix #3 (see ~/.claude/plans/zhuyin-ime-personal.md's P3 fix #3
/// P3 fix #3 (see the design notes' P3 fix #3
/// and KeyHandler's _offeredCompletionFor:lexicon:): Tab must not silently
/// grow an already-finished word into a longer dictionary entry just
/// because one happens to share its prefix. Companion to
Expand Down
4 changes: 1 addition & 3 deletions README.markdown
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,7 @@

## mixime:中英混打(P1,預設關閉)

這個 fork 多了「不切換輸入法直接打英文」的功能(見
`~/.claude/plans/zhuyin-ime-personal.md` 的 F1/P1 節)。P1 期間**預設關閉**,
要 dogfood 請自己打開:
P1 期間**預設關閉**,要 dogfood 請自己打開:

```sh
defaults write io.github.lmanchu.inputmethod.bopomix MixedScriptEnabled -bool true
Expand Down
10 changes: 1 addition & 9 deletions Source/Data/latin-tech-seed.txt
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# Hand-curated tech/product term seed list for the P1 zh/en mixed-typing
# feature (see ~/.claude/plans/zhuyin-ime-personal.md's "P1 設計" section).
# feature (see the design notes' "P1 設計" section).
# Loaded on top of latin-words.txt (see tools/lexicon/build_lexicon.py) so
# these terms are guaranteed present even though many are not real English
# dictionary words and would never appear in a system word list. Rank
Expand All @@ -24,15 +24,7 @@ github 12
notion 13
zoom 14
xcode 15
irisgo 16
openrouter 17
tachikoma 18
hermes 19
gloria 20
apollo 21
argus 22
wells 23
magi 24
computex 25
oem 26
dau 27
Expand Down
2 changes: 1 addition & 1 deletion Source/Engine/McBopomofoLM.h
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ class McBopomofoLM : public Formosa::Gramambular2::LanguageModel {
std::function<std::string(const std::string&)> macroConverter);
std::string convertMacro(const std::string& input) const;

// P1 zh/en mixed typing (see ~/.claude/plans/zhuyin-ime-personal.md).
// P1 zh/en mixed typing (see the design notes).
// When disabled (the default), getUnigrams()/hasUnigrams() ignore
// mixedScriptLM() entirely, so mixedScriptEnabled=false leaves existing
// behavior byte-for-byte unchanged. KeyHandler drives mixedScriptLM()
Expand Down
2 changes: 1 addition & 1 deletion Source/Engine/MixedScript/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ project(MixedScript)
set(CMAKE_CXX_STANDARD 20)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC")

# P1 zh/en mixed-typing (see ~/.claude/plans/zhuyin-ime-personal.md). Mirrors
# P1 zh/en mixed-typing (see the design notes). Mirrors
# the Mandarin/ and gramambular2/ subdirectory pattern: a small static lib
# plus its own gtest binary, consumed by McBopomofoLMLib (McBopomofoLM
# merges LatinPassthroughLM into its unigram dispatch) and by KeyHandler.mm
Expand Down
2 changes: 1 addition & 1 deletion Source/Engine/MixedScript/bopomofo_shape_tracker.h
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ namespace McBopomofo::MixedScript {
// (there is only one consonant/medial/vowel slot per syllable, and a tone
// key -- which is exempt from this check -- always ends the syllable and
// is composed immediately by KeyHandler, which resets this tracker before
// the next key). See zhuyin-ime-personal.md's P1 design section for the
// the next key). See the design notes' P1 design section for the
// quantification this mirrors (~74% of English tokens become
// "uncomposable" within 2-3 letters) and the explicit "th"/"sl" examples.
//
Expand Down
2 changes: 1 addition & 1 deletion Source/Engine/MixedScript/bopomofo_shape_tracker_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ TEST(BopomofoShapeTrackerTest, ToneKeyNeverConflicts) {

// "th": t -> CH (consonant), h -> C (consonant). Two different consonants
// in a row is the required early-reject example from
// zhuyin-ime-personal.md's P1 design section.
// the design notes' P1 design section.
TEST(BopomofoShapeTrackerTest, ThRejectsAtSecondLetter) {
BopomofoShapeTracker tracker;
EXPECT_TRUE(tracker.feed(Standard(), 't'));
Expand Down
8 changes: 4 additions & 4 deletions Source/Engine/MixedScript/latin_lexicon.h
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
// MixedScriptTracker (see mixed_script_tracker.h) to tell whether a run of
// ASCII letters typed without leaving Bopomofo mode is more likely an
// English word than a Chinese reading. See
// ~/.claude/plans/zhuyin-ime-personal.md's "P1 設計" section for the
// the design notes' "P1 設計" section for the
// product rationale (rules A/B/C).

#ifndef SRC_ENGINE_MIXEDSCRIPT_LATIN_LEXICON_H_
Expand Down Expand Up @@ -141,7 +141,7 @@ class LatinLexicon {
// trailing space: the built-in list is far too permissive for that (a
// two-letter run like "up"/"el" is both an English word and an extremely
// common tone-1 syllable), whereas a word in this store is one the user
// has personally disambiguated before. See zhuyin-ime-personal.md's P1
// has personally disambiguated before. See the design notes' P1
// section and docs/REVIEW-P1-2026-09-10.md's B1/B2.
bool isUserWord(const std::string& text) const;

Expand All @@ -156,7 +156,7 @@ class LatinLexicon {
// sorted lazily on first use, which used to cost ~294 ms on the key
// thread the first time anything called this. Not on P1's hot path
// (MixedScriptTracker's decision logic never calls it); kept for P3's
// predictive-typing work, see zhuyin-ime-personal.md's F3 scope.
// predictive-typing work, see the design notes' F3 scope.
bool isPrefix(const std::string& text) const;

// Returns the word's rank (0 = most frequent) or -1 if `text` is not a
Expand Down Expand Up @@ -196,7 +196,7 @@ class LatinLexicon {
// "th" completing normally.
int sourceTier(const std::string& text) const;

// P3 predictive typing (see zhuyin-ime-personal.md's F3 scope). Returns
// P3 predictive typing (see the design notes' F3 scope). Returns
// up to `n` known words that start with (but are longer than) `prefix`,
// best completion first. Ordering: a *confirmed* user word (score >=
// kUserWordConfirmedScore) beats every builtin word, ranked among
Expand Down
2 changes: 1 addition & 1 deletion Source/Engine/MixedScript/latin_passthrough_lm.h
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ class LatinPassthroughLM : public Formosa::Gramambular2::LanguageModel {
// window -- both funnel through fixNodeWithReading:) to tell whether the
// user just explicitly picked a mixedScript Latin candidate, which is
// when (and only when) LatinLexicon::rememberWord() should be called --
// see zhuyin-ime-personal.md's P1 design notes on why the automatic
// see the design notes' P1 design notes on why the automatic
// dictionary-plus-space default must NOT itself count as learning.
bool hasValue(const std::string& key, const std::string& value) const;

Expand Down
6 changes: 3 additions & 3 deletions Source/Engine/MixedScript/mixed_script_tracker.h
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@
namespace McBopomofo::MixedScript {

// The three-way call for one contiguous run of ASCII letters typed without
// leaving Bopomofo mode. See zhuyin-ime-personal.md's P1 design section
// leaving Bopomofo mode. See the design notes' P1 design section
// for the product rationale.
enum class Verdict {
// No English signal (yet). Caller keeps feeding the real
Expand Down Expand Up @@ -148,7 +148,7 @@ class MixedScriptTracker {
// way -- this only touches mixedScript's own bookkeeping.
void popLastLatinChar();

// P3 predictive typing (see zhuyin-ime-personal.md's F3 scope and
// P3 predictive typing (see the design notes' F3 scope and
// LatinLexicon::complete()): replaces the pending run's text with a
// longer completion the user just accepted (Tab, or a pick from the
// completion candidate window -- see KeyHandler's mixed-script hookup).
Expand All @@ -166,7 +166,7 @@ class MixedScriptTracker {
void acceptCompletion(const std::string& word);

// P3 "learn from what you actually type" (see
// zhuyin-ime-personal.md's P3 fix #2): true if the run's *current* text
// the design notes' P3 fix #2): true if the run's *current* text
// was already explicitly written to the user's lexicon by a completion
// accept (acceptCompletion()) and has not changed since. KeyHandler's
// _commitMixedScriptLatinRun uses this to avoid double-counting a word
Expand Down
2 changes: 1 addition & 1 deletion Source/Engine/MixedScript/mixed_script_tracker_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -320,7 +320,7 @@ TEST(MixedScriptTrackerTest, PopLastLatinCharAfterAcceptingCompletion) {
EXPECT_EQ(tracker.latinRun(), "the");
}

// P3 fix #2 (see zhuyin-ime-personal.md's P3 fix #2 and
// P3 fix #2 (see the design notes' P3 fix #2 and
// LatinLexicon::rememberWord()'s use-count doc): acceptCompletion() marks
// the run as already remembered so KeyHandler's natural-typing learn hook
// does not double-count it when the same run later reaches an ordinary
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ extension BopomixInputMethodController: CandidateControllerDelegate {
let selectedCandidate = state.candidates[Int(index)]

// P3 English prediction + Tab completion (see
// ~/.claude/plans/zhuyin-ime-personal.md's F3 scope): a pick
// the design notes' F3 scope): a pick
// from the completion candidate window is not a grid
// position fixNode(...) could override (see
// KeyHandler.h's acceptLatinCompletionWithValue: doc), so it
Expand Down
2 changes: 1 addition & 1 deletion Source/KeyHandler.h
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ extern InputMode InputModePlainBopomofo;
associatedPhraseValue:(NSString *)phraseValue;

// P3 English prediction + Tab completion (see
// ~/.claude/plans/zhuyin-ime-personal.md's F3 scope): accepts `value` as
// the design notes' F3 scope): accepts `value` as
// the pending Latin run's completion, the same way Tab does, but from a
// pick in the completion candidate window (see
// InputMethodController+CandidateControllerDelegate.swift's
Expand Down
Loading
Loading