fix(input): preserve Thai segmentation with text expansion

Scott-Nx · Scott-Nx · commit 7b30e768a32c · 2026-07-17T17:34:40.000+07:00
- defer segmentation only for matching immediate shortcuts
- respect per-shortcut prefixes during prefix detection
- commit completed Thai segments individually
- preserve user history and n-gram context
- add regression coverage for expansion and fallback paths
diff --git a/app/src/main/java/helium314/keyboard/latin/inputlogic/InputLogic.java b/app/src/main/java/helium314/keyboard/latin/inputlogic/InputLogic.java
@@ -67,6 +67,7 @@
 import helium314.keyboard.latin.utils.TextRange;
 import helium314.keyboard.latin.utils.TimestampKt;
 
+import java.text.BreakIterator;
 import java.util.ArrayList;
 import java.util.Locale;
 
@@ -78,7 +79,17 @@
 public final class InputLogic {
     private static final String TAG = InputLogic.class.getSimpleName();
     private static final char INLINE_EMOJI_SEARCH_MARKER = ':';
-    private static final int[] EMPTY_CODE_POINTS = new int[0];
+    // Currently only Thai needs word segmentation. If additional scripts are
+    // added to ScriptUtils.needsWordSegmentation(), the BreakIterator locale
+    // selection must be revisited.
+    private static final Locale THAI_LOCALE = Locale.forLanguageTag("th");
+    private static final ThreadLocal<BreakIterator> THAI_WORD_BREAK_ITERATOR =
+            new ThreadLocal<BreakIterator>() {
+                @Override
+                protected BreakIterator initialValue() {
+                    return BreakIterator.getWordInstance(THAI_LOCALE);
+                }
+            };
 
     // TODO : Remove this member when we can.
     private final LatinIME mLatinIME;
@@ -1453,10 +1464,14 @@ private void handleNonSeparatorEvent(final Event event, final SettingsValues set
             if (mWordComposer.isSingleLetter()) {
                 mWordComposer.setCapitalizedModeAtStartComposingTime(inputTransaction.getShiftState());
             }
-            setComposingTextInternal(getTextWithUnderline(mWordComposer.getTypedWord()), 1);
+            boolean didSetComposingText = false;
+            boolean didExpand = false;
+            boolean shouldDeferSegmentation = false;
             if (helium314.keyboard.latin.utils.TextExpanderUtils.INSTANCE.isEnabled(mLatinIME)
                     && helium314.keyboard.latin.utils.TextExpanderUtils.INSTANCE.isImmediateEnabled(mLatinIME)) {
                 final String typedWord = mWordComposer.getTypedWord();
+                setComposingTextInternal(getTextWithUnderline(typedWord), 1);
+                didSetComposingText = true;
                 final CharSequence textBefore = mConnection.getTextBeforeCursor(50, 0);
                 if (textBefore != null) {
                     final String textStr = textBefore.toString();
@@ -1469,7 +1484,19 @@ private void handleNonSeparatorEvent(final Event event, final SettingsValues set
                         }
                         commitExpandedText(result.getMatchedString(), result.getExpandedText());
                         resetComposingState(true);
+                        didExpand = true;
                     }
+                    shouldDeferSegmentation = !didExpand
+                            && ScriptUtils.needsWordSegmentation(settingsValues.mLocale)
+                            && helium314.keyboard.latin.utils.TextExpanderUtils.INSTANCE
+                                    .isPrefixOfNonRegexShortcut(typedWord, textStr, mLatinIME);
+                }
+            }
+            if (!didExpand && !shouldDeferSegmentation) {
+                final boolean didCommitCompletedWordSegments =
+                        maybeCommitCompletedWordSegments(settingsValues);
+                if (!didSetComposingText || didCommitCompletedWordSegments) {
+                    setComposingTextInternal(getTextWithUnderline(mWordComposer.getTypedWord()), 1);
                 }
             }
         } else {
@@ -1499,6 +1526,60 @@ private boolean isCursorAtStartOrAfterSeparator(SettingsValues settingsValues) {
                 || settingsValues.mSpacingAndPunctuations.isWordSeparator(codePointBeforeCursor);
     }
 
+    private boolean maybeCommitCompletedWordSegments(final SettingsValues settingsValues) {
+        if (!ScriptUtils.needsWordSegmentation(settingsValues.mLocale)
+                || settingsValues.mSpacingAndPunctuations.mCurrentLanguageHasSpaces) {
+            return false;
+        }
+
+        final String typedWord = mWordComposer.getTypedWord();
+        final int length = typedWord.length();
+        if (length <= 1) {
+            return false;
+        }
+
+        final BreakIterator iterator = THAI_WORD_BREAK_ITERATOR.get();
+        iterator.setText(typedWord);
+        int segmentStart = iterator.first();
+        int wordBoundary = iterator.next();
+        boolean didCommitSegment = false;
+        while (wordBoundary != BreakIterator.DONE && wordBoundary < length) {
+            final String completedWordSegment = typedWord.substring(segmentStart, wordBoundary);
+            if (!TextUtils.isEmpty(completedWordSegment)) {
+                commitCompletedWordSegment(settingsValues, completedWordSegment);
+                didCommitSegment = true;
+            }
+            segmentStart = wordBoundary;
+            wordBoundary = iterator.next();
+        }
+        if (!didCommitSegment) {
+            return false;
+        }
+
+        // Scripts that require explicit word segmentation (currently only Thai) can accumulate
+        // multiple word segments in one composing span. Commit completed segments and
+        // leave the latest segment composing so underline and candidate handling stay local.
+        final String remainingWord = typedWord.substring(segmentStart);
+        final int[] codePoints = StringUtils.toCodePointArray(remainingWord);
+        mWordComposer.setComposingWord(codePoints,
+                mLatinIME.getCoordinatesForCurrentKeyboard(codePoints));
+        return true;
+    }
+
+    private void commitCompletedWordSegment(final SettingsValues settingsValues,
+            final String completedWordSegment) {
+        final NgramContext ngramContext = getNgramContextFromNthPreviousWordForSuggestion(
+                settingsValues.mSpacingAndPunctuations, 2);
+        mConnection.commitText(completedWordSegment, 1);
+        performAdditionToUserHistoryDictionary(settingsValues, completedWordSegment, ngramContext);
+        mLastComposedWord = new LastComposedWord(new ArrayList<>(), null, completedWordSegment,
+                completedWordSegment, LastComposedWord.NOT_A_SEPARATOR, ngramContext,
+                WordComposer.CAPS_MODE_OFF);
+        StatsUtils.onWordCommitUserTyped(completedWordSegment, mWordComposer.isBatchMode());
+    }
+
+
+
     /**
      * Handle input of a separator code point.
      * 
@@ -1510,10 +1591,14 @@ private void handleSeparatorEvent(final Event event, final InputTransaction inpu
         final int codePoint = event.getCodePoint();
         final SettingsValues settingsValues = inputTransaction.getSettingsValues();
         final boolean wasComposingWord = mWordComposer.isComposingWord();
+        // Scripts that require explicit word segmentation should still allow an
+        // explicit Space to be inserted while committing composing text.
+        final boolean needsSegmentation = ScriptUtils.needsWordSegmentation(settingsValues.mLocale);
         // We avoid sending spaces in languages without spaces if we were composing.
         final boolean shouldAvoidSendingCode = Constants.CODE_SPACE == codePoint
                 && !settingsValues.mSpacingAndPunctuations.mCurrentLanguageHasSpaces
-                && wasComposingWord;
+                && wasComposingWord
+                && !needsSegmentation;
 
         // wrap / unwrap selected text in codepoint pairs
         if (!wasComposingWord && mConnection.hasSelection()) { // we should never be composing when something is
diff --git a/app/src/main/java/helium314/keyboard/latin/utils/ScriptUtils.kt b/app/src/main/java/helium314/keyboard/latin/utils/ScriptUtils.kt
@@ -185,6 +185,14 @@ object ScriptUtils {
         }
     }
 
+    /**
+     * Returns true if the locale uses a script that requires explicit word segmentation.
+     * Currently returns true for Thai only.
+     */
+    @JvmStatic
+    fun needsWordSegmentation(locale: Locale): Boolean =
+        locale.language == "th"
+
     @JvmStatic
     fun isScriptRtl(script: String): Boolean {
         return when (script) {
diff --git a/app/src/main/java/helium314/keyboard/latin/utils/TextExpanderUtils.kt b/app/src/main/java/helium314/keyboard/latin/utils/TextExpanderUtils.kt
@@ -209,6 +209,22 @@ object TextExpanderUtils {
         return result
     }
 
+    fun isPrefixOfNonRegexShortcut(
+        word: String,
+        textBeforeCursor: String,
+        context: Context,
+    ): Boolean =
+        getShortcuts(context).any { (key, entry) ->
+            if (key.startsWith(REGEX_PREFIX) || key.length < entry.prefix.length) {
+                false
+            } else {
+                val shortcut = key.substring(entry.prefix.length)
+                !shortcut.equals(word, ignoreCase = true) &&
+                    shortcut.startsWith(word, ignoreCase = true) &&
+                    textBeforeCursor.endsWith(entry.prefix + word, ignoreCase = true)
+            }
+        }
+
     fun getExpandedWordForTyped(word: String?, textBeforeCursor: String?, context: Context): ExpandedResult? {
         if (word == null || textBeforeCursor == null || !isEnabled(context)) return null
         val shortcuts = getShortcuts(context)
diff --git a/app/src/test/java/helium314/keyboard/latin/InputLogicTest.kt b/app/src/test/java/helium314/keyboard/latin/InputLogicTest.kt
@@ -16,6 +16,7 @@ import helium314.keyboard.keyboard.KeyboardSwitcher
 import helium314.keyboard.keyboard.MainKeyboardView
 import helium314.keyboard.keyboard.internal.keyboard_parser.floris.KeyCode
 import helium314.keyboard.latin.ShadowFacilitator2.Companion.lastAddedWord
+import helium314.keyboard.latin.ShadowFacilitator2.Companion.lastNgramContext
 import helium314.keyboard.latin.SuggestedWords.SuggestedWordInfo
 import helium314.keyboard.latin.common.Constants
 import helium314.keyboard.latin.common.LocaleUtils.constructLocale
@@ -84,6 +85,15 @@ class InputLogicTest {
         assertEquals("", composingText)
     }
 
+    @Test fun `english space-separated typing keeps composing word`() {
+        reset()
+        chainInput("hello")
+        assertEquals("hello", composingText)
+        input(' ')
+        assertEquals("hello ", text)
+        assertEquals("", composingText)
+    }
+
     @Test fun delete() {
         reset()
         setText("hello there ")
@@ -160,6 +170,138 @@ class InputLogicTest {
         assertEquals("ㅛ.", text)
     }
 
+    @Test fun `space after thai composing word inserts space`() {
+        reset()
+        latinIME.switchToSubtype(SubtypeSettings.getResourceSubtypesForLocale("th".constructLocale()).first())
+        currentScript = ScriptUtils.SCRIPT_THAI
+        chainInput("ภาษาไทย")
+        assertEquals("ไทย", composingText)
+        input(' ')
+        assertEquals("ภาษาไทย ", text)
+        assertEquals("", composingText)
+    }
+
+    @Test fun `thai composing word follows word boundaries`() {
+        reset()
+        latinIME.switchToSubtype(SubtypeSettings.getResourceSubtypesForLocale("th".constructLocale()).first())
+        currentScript = ScriptUtils.SCRIPT_THAI
+        chainInput("ภาษาไทยดี")
+        assertEquals("ภาษาไทยดี", text)
+        assertEquals("ดี", composingText)
+        assertEquals("ไทย", lastAddedWord)
+        assertEquals("ภาษา", lastNgramContext)
+    }
+
+    @Test fun `single thai composing segment remains composing`() {
+        reset()
+        latinIME.switchToSubtype(SubtypeSettings.getResourceSubtypesForLocale("th".constructLocale()).first())
+        currentScript = ScriptUtils.SCRIPT_THAI
+        chainInput("ไทย")
+        assertEquals("ไทย", text)
+        assertEquals("ไทย", composingText)
+    }
+
+    @Test fun `space after segmented thai composing word inserts one space`() {
+        reset()
+        latinIME.switchToSubtype(SubtypeSettings.getResourceSubtypesForLocale("th".constructLocale()).first())
+        currentScript = ScriptUtils.SCRIPT_THAI
+        chainInput("ภาษาไทยดี")
+        input(' ')
+        assertEquals("ภาษาไทยดี ", text)
+        assertEquals("", composingText)
+    }
+
+    @Test fun `immediate text expansion uses full segmented thai word`() {
+        reset()
+        latinIME.switchToSubtype(SubtypeSettings.getResourceSubtypesForLocale("th".constructLocale()).first())
+        currentScript = ScriptUtils.SCRIPT_THAI
+        latinIME.prefs().edit().apply {
+            putBoolean(helium314.keyboard.latin.utils.TextExpanderUtils.PREF_ENABLED, true)
+            putBoolean(helium314.keyboard.latin.utils.TextExpanderUtils.PREF_IMMEDIATE, true)
+        }.commit()
+        val shortcuts = mapOf("ภาษาไทย" to helium314.keyboard.latin.utils.TextExpanderUtils.ShortcutEntry("expanded", ""))
+        helium314.keyboard.latin.utils.TextExpanderUtils.saveShortcuts(latinIME, shortcuts)
+
+        typeNoAssert("ภาษาไทย")
+
+        assertEquals("expanded", text)
+        assertEquals("", composingText)
+        assertEquals("", lastAddedWord)
+    }
+
+    @Test fun `immediate text expansion uses prefixed segmented thai word`() {
+        reset()
+        latinIME.switchToSubtype(SubtypeSettings.getResourceSubtypesForLocale("th".constructLocale()).first())
+        currentScript = ScriptUtils.SCRIPT_THAI
+        latinIME.prefs().edit().apply {
+            putBoolean(helium314.keyboard.latin.utils.TextExpanderUtils.PREF_ENABLED, true)
+            putBoolean(helium314.keyboard.latin.utils.TextExpanderUtils.PREF_IMMEDIATE, true)
+        }.commit()
+        val shortcuts = mapOf(".ภาษาไทย" to helium314.keyboard.latin.utils.TextExpanderUtils.ShortcutEntry("expanded", "."))
+        helium314.keyboard.latin.utils.TextExpanderUtils.saveShortcuts(latinIME, shortcuts)
+
+        typeNoAssert(".ภาษาไทย")
+
+        assertEquals("expanded", text)
+        assertEquals("", composingText)
+        assertEquals("", lastAddedWord)
+    }
+
+    @Test fun `prefixed immediate text expansion does not defer thai without prefix`() {
+        reset()
+        latinIME.switchToSubtype(SubtypeSettings.getResourceSubtypesForLocale("th".constructLocale()).first())
+        currentScript = ScriptUtils.SCRIPT_THAI
+        latinIME.prefs().edit().apply {
+            putBoolean(helium314.keyboard.latin.utils.TextExpanderUtils.PREF_ENABLED, true)
+            putBoolean(helium314.keyboard.latin.utils.TextExpanderUtils.PREF_IMMEDIATE, true)
+        }.commit()
+        val shortcuts = mapOf(".ภาษาไทย" to helium314.keyboard.latin.utils.TextExpanderUtils.ShortcutEntry("expanded", "."))
+        helium314.keyboard.latin.utils.TextExpanderUtils.saveShortcuts(latinIME, shortcuts)
+
+        typeNoAssert("ภาษาไทย")
+
+        assertEquals("ภาษาไทย", text)
+        assertEquals("ไทย", composingText)
+        assertEquals("ภาษา", lastAddedWord)
+    }
+
+    @Test fun `immediate text expansion still segments thai non-shortcut`() {
+        reset()
+        latinIME.switchToSubtype(SubtypeSettings.getResourceSubtypesForLocale("th".constructLocale()).first())
+        currentScript = ScriptUtils.SCRIPT_THAI
+        latinIME.prefs().edit().apply {
+            putBoolean(helium314.keyboard.latin.utils.TextExpanderUtils.PREF_ENABLED, true)
+            putBoolean(helium314.keyboard.latin.utils.TextExpanderUtils.PREF_IMMEDIATE, true)
+        }.commit()
+        val shortcuts = mapOf("อื่น" to helium314.keyboard.latin.utils.TextExpanderUtils.ShortcutEntry("expanded", ""))
+        helium314.keyboard.latin.utils.TextExpanderUtils.saveShortcuts(latinIME, shortcuts)
+
+        chainInput("ภาษาไทยดี")
+
+        assertEquals("ภาษาไทยดี", text)
+        assertEquals("ดี", composingText)
+        assertEquals("ไทย", lastAddedWord)
+    }
+
+    @Test fun `failed immediate expansion commits thai segments separately`() {
+        reset()
+        latinIME.switchToSubtype(SubtypeSettings.getResourceSubtypesForLocale("th".constructLocale()).first())
+        currentScript = ScriptUtils.SCRIPT_THAI
+        latinIME.prefs().edit().apply {
+            putBoolean(helium314.keyboard.latin.utils.TextExpanderUtils.PREF_ENABLED, true)
+            putBoolean(helium314.keyboard.latin.utils.TextExpanderUtils.PREF_IMMEDIATE, true)
+        }.commit()
+        val shortcuts = mapOf("ภาษาไทยดี" to helium314.keyboard.latin.utils.TextExpanderUtils.ShortcutEntry("expanded", ""))
+        helium314.keyboard.latin.utils.TextExpanderUtils.saveShortcuts(latinIME, shortcuts)
+
+        typeNoAssert("ภาษาไทยแดง")
+
+        assertEquals("ภาษาไทยแดง", text)
+        assertEquals("แดง", composingText)
+        assertEquals("ไทย", lastAddedWord)
+        assertEquals("ภาษา", lastNgramContext)
+    }
+
     // see issue 1551 (debug only)
     @Test fun deleteHangul() {
         reset()
@@ -779,6 +921,7 @@ class InputLogicTest {
         batchEdit = 0
         currentInputType = InputType.TYPE_CLASS_TEXT
         lastAddedWord = ""
+        lastNgramContext = ""
 
         // reset settings
         latinIME.prefs().edit { clear() }
@@ -1212,8 +1355,10 @@ class ShadowFacilitator2 {
                          ngramContext: NgramContext, timeStampInSeconds: Long,
                          blockPotentiallyOffensive: Boolean) {
         lastAddedWord = suggestion
+        lastNgramContext = ngramContext.extractPrevWordsContext()
     }
     companion object {
         var lastAddedWord = ""
+        var lastNgramContext = ""
     }
 }
diff --git a/app/src/test/java/helium314/keyboard/latin/ScriptUtilsTest.kt b/app/src/test/java/helium314/keyboard/latin/ScriptUtilsTest.kt
@@ -5,9 +5,12 @@ import helium314.keyboard.latin.common.LocaleUtils.constructLocale
 import helium314.keyboard.latin.utils.ScriptUtils.SCRIPT_CYRILLIC
 import helium314.keyboard.latin.utils.ScriptUtils.SCRIPT_DEVANAGARI
 import helium314.keyboard.latin.utils.ScriptUtils.SCRIPT_LATIN
+import helium314.keyboard.latin.utils.ScriptUtils.needsWordSegmentation
 import helium314.keyboard.latin.utils.ScriptUtils.script
 import kotlin.test.Test
 import kotlin.test.assertEquals
+import kotlin.test.assertFalse
+import kotlin.test.assertTrue
 
 class ScriptUtilsTest {
     @Test fun defaultScript() {
@@ -18,4 +21,18 @@ class ScriptUtilsTest {
         assertEquals(SCRIPT_CYRILLIC, "mk".constructLocale().script())
         assertEquals(SCRIPT_CYRILLIC, "fr-Cyrl".constructLocale().script())
     }
+
+    @Test fun needsWordSegmentationThai() {
+        assertTrue(needsWordSegmentation("th".constructLocale()))
+    }
+
+    @Test fun needsWordSegmentationNonThai() {
+        assertFalse(needsWordSegmentation("en".constructLocale()))
+        assertFalse(needsWordSegmentation("ja".constructLocale()))
+        assertFalse(needsWordSegmentation("zh".constructLocale()))
+        assertFalse(needsWordSegmentation("lo".constructLocale()))
+        assertFalse(needsWordSegmentation("km".constructLocale()))
+        assertFalse(needsWordSegmentation("ko".constructLocale()))
+        assertFalse(needsWordSegmentation("my".constructLocale()))
+    }
 }

Original file line number	Diff line number	Diff line change
`@@ -185,6 +185,14 @@ object ScriptUtils {`
`185`	`185`	`}`
`186`	`186`	`}`
`187`	`187`
	`188`	`+ /**`
	`189`	`+ * Returns true if the locale uses a script that requires explicit word segmentation.`
	`190`	`+ * Currently returns true for Thai only.`
	`191`	`+ */`
	`192`	`+ @JvmStatic`
	`193`	`+ fun needsWordSegmentation(locale: Locale): Boolean =`
	`194`	`+ locale.language == "th"`
	`195`	`+`
`188`	`196`	`@JvmStatic`
`189`	`197`	`fun isScriptRtl(script: String): Boolean {`
`190`	`198`	`return when (script) {`