diff --git a/app/src/main/java/helium314/keyboard/latin/inputlogic/InputLogic.java b/app/src/main/java/helium314/keyboard/latin/inputlogic/InputLogic.java index f037806fd..191ece2bc 100644 --- a/app/src/main/java/helium314/keyboard/latin/inputlogic/InputLogic.java +++ b/app/src/main/java/helium314/keyboard/latin/inputlogic/InputLogic.java @@ -67,6 +67,7 @@ import helium314.keyboard.latin.utils.TextRange; import helium314.keyboard.latin.utils.TimestampKt; +import java.text.BreakIterator; import java.util.ArrayList; import java.util.Locale; @@ -78,7 +79,12 @@ public final class InputLogic { private static final String TAG = InputLogic.class.getSimpleName(); private static final char INLINE_EMOJI_SEARCH_MARKER = ':'; - private static final int[] EMPTY_CODE_POINTS = new int[0]; + // Currently only Thai needs word segmentation. If additional scripts are + // added to ScriptUtils.needsWordSegmentation(), the BreakIterator locale + // selection must be revisited. + private static final Locale THAI_LOCALE = Locale.forLanguageTag("th"); + private static final ThreadLocal THAI_WORD_BREAK_ITERATOR = + ThreadLocal.withInitial(() -> BreakIterator.getWordInstance(THAI_LOCALE)); // TODO : Remove this member when we can. private final LatinIME mLatinIME; @@ -1453,10 +1459,14 @@ private void handleNonSeparatorEvent(final Event event, final SettingsValues set if (mWordComposer.isSingleLetter()) { mWordComposer.setCapitalizedModeAtStartComposingTime(inputTransaction.getShiftState()); } - setComposingTextInternal(getTextWithUnderline(mWordComposer.getTypedWord()), 1); + boolean didSetComposingText = false; + boolean didExpand = false; + boolean shouldDeferSegmentation = false; if (helium314.keyboard.latin.utils.TextExpanderUtils.INSTANCE.isEnabled(mLatinIME) && helium314.keyboard.latin.utils.TextExpanderUtils.INSTANCE.isImmediateEnabled(mLatinIME)) { final String typedWord = mWordComposer.getTypedWord(); + setComposingTextInternal(getTextWithUnderline(typedWord), 1); + didSetComposingText = true; final CharSequence textBefore = mConnection.getTextBeforeCursor(50, 0); if (textBefore != null) { final String textStr = textBefore.toString(); @@ -1469,7 +1479,19 @@ private void handleNonSeparatorEvent(final Event event, final SettingsValues set } commitExpandedText(result.getMatchedString(), result.getExpandedText()); resetComposingState(true); + didExpand = true; } + shouldDeferSegmentation = !didExpand + && ScriptUtils.needsWordSegmentation(settingsValues.mLocale) + && helium314.keyboard.latin.utils.TextExpanderUtils.INSTANCE + .isPrefixOfNonRegexShortcut(typedWord, textStr, mLatinIME); + } + } + if (!didExpand && !shouldDeferSegmentation) { + final boolean didCommitCompletedWordSegments = + maybeCommitCompletedWordSegments(settingsValues); + if (!didSetComposingText || didCommitCompletedWordSegments) { + setComposingTextInternal(getTextWithUnderline(mWordComposer.getTypedWord()), 1); } } } else { @@ -1499,6 +1521,60 @@ private boolean isCursorAtStartOrAfterSeparator(SettingsValues settingsValues) { || settingsValues.mSpacingAndPunctuations.isWordSeparator(codePointBeforeCursor); } + private boolean maybeCommitCompletedWordSegments(final SettingsValues settingsValues) { + if (!ScriptUtils.needsWordSegmentation(settingsValues.mLocale) + || settingsValues.mSpacingAndPunctuations.mCurrentLanguageHasSpaces) { + return false; + } + + final String typedWord = mWordComposer.getTypedWord(); + final int length = typedWord.length(); + if (length <= 1) { + return false; + } + + final BreakIterator iterator = THAI_WORD_BREAK_ITERATOR.get(); + iterator.setText(typedWord); + int segmentStart = iterator.first(); + int wordBoundary = iterator.next(); + boolean didCommitSegment = false; + while (wordBoundary != BreakIterator.DONE && wordBoundary < length) { + final String completedWordSegment = typedWord.substring(segmentStart, wordBoundary); + if (!TextUtils.isEmpty(completedWordSegment)) { + commitCompletedWordSegment(settingsValues, completedWordSegment); + didCommitSegment = true; + } + segmentStart = wordBoundary; + wordBoundary = iterator.next(); + } + if (!didCommitSegment) { + return false; + } + + // Scripts that require explicit word segmentation (currently only Thai) can accumulate + // multiple word segments in one composing span. Commit completed segments and + // leave the latest segment composing so underline and candidate handling stay local. + final String remainingWord = typedWord.substring(segmentStart); + final int[] codePoints = StringUtils.toCodePointArray(remainingWord); + mWordComposer.setComposingWord(codePoints, + mLatinIME.getCoordinatesForCurrentKeyboard(codePoints)); + return true; + } + + private void commitCompletedWordSegment(final SettingsValues settingsValues, + final String completedWordSegment) { + final NgramContext ngramContext = getNgramContextFromNthPreviousWordForSuggestion( + settingsValues.mSpacingAndPunctuations, 2); + mConnection.commitText(completedWordSegment, 1); + performAdditionToUserHistoryDictionary(settingsValues, completedWordSegment, ngramContext); + mLastComposedWord = new LastComposedWord(new ArrayList<>(), null, completedWordSegment, + completedWordSegment, LastComposedWord.NOT_A_SEPARATOR, ngramContext, + WordComposer.CAPS_MODE_OFF); + StatsUtils.onWordCommitUserTyped(completedWordSegment, mWordComposer.isBatchMode()); + } + + + /** * Handle input of a separator code point. * @@ -1510,10 +1586,14 @@ private void handleSeparatorEvent(final Event event, final InputTransaction inpu final int codePoint = event.getCodePoint(); final SettingsValues settingsValues = inputTransaction.getSettingsValues(); final boolean wasComposingWord = mWordComposer.isComposingWord(); + // Scripts that require explicit word segmentation should still allow an + // explicit Space to be inserted while committing composing text. + final boolean needsSegmentation = ScriptUtils.needsWordSegmentation(settingsValues.mLocale); // We avoid sending spaces in languages without spaces if we were composing. final boolean shouldAvoidSendingCode = Constants.CODE_SPACE == codePoint && !settingsValues.mSpacingAndPunctuations.mCurrentLanguageHasSpaces - && wasComposingWord; + && wasComposingWord + && !needsSegmentation; // wrap / unwrap selected text in codepoint pairs if (!wasComposingWord && mConnection.hasSelection()) { // we should never be composing when something is diff --git a/app/src/main/java/helium314/keyboard/latin/utils/ScriptUtils.kt b/app/src/main/java/helium314/keyboard/latin/utils/ScriptUtils.kt index ba0fca550..de91d4f2a 100644 --- a/app/src/main/java/helium314/keyboard/latin/utils/ScriptUtils.kt +++ b/app/src/main/java/helium314/keyboard/latin/utils/ScriptUtils.kt @@ -185,6 +185,14 @@ object ScriptUtils { } } + /** + * Returns true if the locale uses a script that requires explicit word segmentation. + * Currently returns true for Thai only. + */ + @JvmStatic + fun needsWordSegmentation(locale: Locale): Boolean = + locale.language == "th" + @JvmStatic fun isScriptRtl(script: String): Boolean { return when (script) { diff --git a/app/src/main/java/helium314/keyboard/latin/utils/TextExpanderUtils.kt b/app/src/main/java/helium314/keyboard/latin/utils/TextExpanderUtils.kt index 899563a1a..7681d50be 100644 --- a/app/src/main/java/helium314/keyboard/latin/utils/TextExpanderUtils.kt +++ b/app/src/main/java/helium314/keyboard/latin/utils/TextExpanderUtils.kt @@ -209,6 +209,22 @@ object TextExpanderUtils { return result } + fun isPrefixOfNonRegexShortcut( + word: String, + textBeforeCursor: String, + context: Context, + ): Boolean = + getShortcuts(context).any { (key, entry) -> + if (key.startsWith(REGEX_PREFIX) || key.length < entry.prefix.length) { + false + } else { + val shortcut = key.substring(entry.prefix.length) + !shortcut.equals(word, ignoreCase = true) && + shortcut.startsWith(word, ignoreCase = true) && + textBeforeCursor.endsWith(entry.prefix + word, ignoreCase = true) + } + } + fun getExpandedWordForTyped(word: String?, textBeforeCursor: String?, context: Context): ExpandedResult? { if (word == null || textBeforeCursor == null || !isEnabled(context)) return null val shortcuts = getShortcuts(context) diff --git a/app/src/test/java/helium314/keyboard/latin/InputLogicTest.kt b/app/src/test/java/helium314/keyboard/latin/InputLogicTest.kt index be000cba2..8cb6854c6 100644 --- a/app/src/test/java/helium314/keyboard/latin/InputLogicTest.kt +++ b/app/src/test/java/helium314/keyboard/latin/InputLogicTest.kt @@ -15,7 +15,10 @@ import helium314.keyboard.event.Event import helium314.keyboard.keyboard.KeyboardSwitcher import helium314.keyboard.keyboard.MainKeyboardView import helium314.keyboard.keyboard.internal.keyboard_parser.floris.KeyCode +import helium314.keyboard.latin.ShadowFacilitator2.Companion.addedWords import helium314.keyboard.latin.ShadowFacilitator2.Companion.lastAddedWord +import helium314.keyboard.latin.ShadowFacilitator2.Companion.lastNgramContext +import helium314.keyboard.latin.ShadowFacilitator2.Companion.ngramContexts import helium314.keyboard.latin.SuggestedWords.SuggestedWordInfo import helium314.keyboard.latin.common.Constants import helium314.keyboard.latin.common.LocaleUtils.constructLocale @@ -84,6 +87,15 @@ class InputLogicTest { assertEquals("", composingText) } + @Test fun `english space-separated typing keeps composing word`() { + reset() + chainInput("hello") + assertEquals("hello", composingText) + input(' ') + assertEquals("hello ", text) + assertEquals("", composingText) + } + @Test fun delete() { reset() setText("hello there ") @@ -160,6 +172,140 @@ class InputLogicTest { assertEquals("ㅛ.", text) } + @Test fun `space after thai composing word inserts space`() { + reset() + latinIME.switchToSubtype(SubtypeSettings.getResourceSubtypesForLocale("th".constructLocale()).first()) + currentScript = ScriptUtils.SCRIPT_THAI + chainInput("ภาษาไทย") + assertEquals("ไทย", composingText) + input(' ') + assertEquals("ภาษาไทย ", text) + assertEquals("", composingText) + } + + @Test fun `thai composing word follows word boundaries`() { + reset() + latinIME.switchToSubtype(SubtypeSettings.getResourceSubtypesForLocale("th".constructLocale()).first()) + currentScript = ScriptUtils.SCRIPT_THAI + chainInput("ภาษาไทยดี") + assertEquals("ภาษาไทยดี", text) + assertEquals("ดี", composingText) + assertEquals("ไทย", lastAddedWord) + assertEquals("ภาษา", lastNgramContext) + assertEquals(listOf("ภาษา", "ไทย"), addedWords) + assertEquals(listOf("", "ภาษา"), ngramContexts) + } + + @Test fun `single thai composing segment remains composing`() { + reset() + latinIME.switchToSubtype(SubtypeSettings.getResourceSubtypesForLocale("th".constructLocale()).first()) + currentScript = ScriptUtils.SCRIPT_THAI + chainInput("ไทย") + assertEquals("ไทย", text) + assertEquals("ไทย", composingText) + } + + @Test fun `space after segmented thai composing word inserts one space`() { + reset() + latinIME.switchToSubtype(SubtypeSettings.getResourceSubtypesForLocale("th".constructLocale()).first()) + currentScript = ScriptUtils.SCRIPT_THAI + chainInput("ภาษาไทยดี") + input(' ') + assertEquals("ภาษาไทยดี ", text) + assertEquals("", composingText) + } + + @Test fun `immediate text expansion uses full segmented thai word`() { + reset() + latinIME.switchToSubtype(SubtypeSettings.getResourceSubtypesForLocale("th".constructLocale()).first()) + currentScript = ScriptUtils.SCRIPT_THAI + latinIME.prefs().edit().apply { + putBoolean(helium314.keyboard.latin.utils.TextExpanderUtils.PREF_ENABLED, true) + putBoolean(helium314.keyboard.latin.utils.TextExpanderUtils.PREF_IMMEDIATE, true) + }.commit() + val shortcuts = mapOf("ภาษาไทย" to helium314.keyboard.latin.utils.TextExpanderUtils.ShortcutEntry("expanded", "")) + helium314.keyboard.latin.utils.TextExpanderUtils.saveShortcuts(latinIME, shortcuts) + + typeNoAssert("ภาษาไทย") + + assertEquals("expanded", text) + assertEquals("", composingText) + assertEquals("", lastAddedWord) + } + + @Test fun `immediate text expansion uses prefixed segmented thai word`() { + reset() + latinIME.switchToSubtype(SubtypeSettings.getResourceSubtypesForLocale("th".constructLocale()).first()) + currentScript = ScriptUtils.SCRIPT_THAI + latinIME.prefs().edit().apply { + putBoolean(helium314.keyboard.latin.utils.TextExpanderUtils.PREF_ENABLED, true) + putBoolean(helium314.keyboard.latin.utils.TextExpanderUtils.PREF_IMMEDIATE, true) + }.commit() + val shortcuts = mapOf(".ภาษาไทย" to helium314.keyboard.latin.utils.TextExpanderUtils.ShortcutEntry("expanded", ".")) + helium314.keyboard.latin.utils.TextExpanderUtils.saveShortcuts(latinIME, shortcuts) + + typeNoAssert(".ภาษาไทย") + + assertEquals("expanded", text) + assertEquals("", composingText) + assertEquals("", lastAddedWord) + } + + @Test fun `prefixed immediate text expansion does not defer thai without prefix`() { + reset() + latinIME.switchToSubtype(SubtypeSettings.getResourceSubtypesForLocale("th".constructLocale()).first()) + currentScript = ScriptUtils.SCRIPT_THAI + latinIME.prefs().edit().apply { + putBoolean(helium314.keyboard.latin.utils.TextExpanderUtils.PREF_ENABLED, true) + putBoolean(helium314.keyboard.latin.utils.TextExpanderUtils.PREF_IMMEDIATE, true) + }.commit() + val shortcuts = mapOf(".ภาษาไทย" to helium314.keyboard.latin.utils.TextExpanderUtils.ShortcutEntry("expanded", ".")) + helium314.keyboard.latin.utils.TextExpanderUtils.saveShortcuts(latinIME, shortcuts) + + typeNoAssert("ภาษาไทย") + + assertEquals("ภาษาไทย", text) + assertEquals("ไทย", composingText) + assertEquals("ภาษา", lastAddedWord) + } + + @Test fun `immediate text expansion still segments thai non-shortcut`() { + reset() + latinIME.switchToSubtype(SubtypeSettings.getResourceSubtypesForLocale("th".constructLocale()).first()) + currentScript = ScriptUtils.SCRIPT_THAI + latinIME.prefs().edit().apply { + putBoolean(helium314.keyboard.latin.utils.TextExpanderUtils.PREF_ENABLED, true) + putBoolean(helium314.keyboard.latin.utils.TextExpanderUtils.PREF_IMMEDIATE, true) + }.commit() + val shortcuts = mapOf("อื่น" to helium314.keyboard.latin.utils.TextExpanderUtils.ShortcutEntry("expanded", "")) + helium314.keyboard.latin.utils.TextExpanderUtils.saveShortcuts(latinIME, shortcuts) + + chainInput("ภาษาไทยดี") + + assertEquals("ภาษาไทยดี", text) + assertEquals("ดี", composingText) + assertEquals("ไทย", lastAddedWord) + } + + @Test fun `failed immediate expansion commits thai segments separately`() { + reset() + latinIME.switchToSubtype(SubtypeSettings.getResourceSubtypesForLocale("th".constructLocale()).first()) + currentScript = ScriptUtils.SCRIPT_THAI + latinIME.prefs().edit().apply { + putBoolean(helium314.keyboard.latin.utils.TextExpanderUtils.PREF_ENABLED, true) + putBoolean(helium314.keyboard.latin.utils.TextExpanderUtils.PREF_IMMEDIATE, true) + }.commit() + val shortcuts = mapOf("ภาษาไทยดี" to helium314.keyboard.latin.utils.TextExpanderUtils.ShortcutEntry("expanded", "")) + helium314.keyboard.latin.utils.TextExpanderUtils.saveShortcuts(latinIME, shortcuts) + + typeNoAssert("ภาษาไทยแดง") + + assertEquals("ภาษาไทยแดง", text) + assertEquals("แดง", composingText) + assertEquals("ไทย", lastAddedWord) + assertEquals("ภาษา", lastNgramContext) + } + // see issue 1551 (debug only) @Test fun deleteHangul() { reset() @@ -779,11 +925,20 @@ class InputLogicTest { batchEdit = 0 currentInputType = InputType.TYPE_CLASS_TEXT lastAddedWord = "" + lastNgramContext = "" + addedWords.clear() + ngramContexts.clear() // reset settings - latinIME.prefs().edit { clear() } + latinIME.prefs().edit { + clear() + putBoolean(Settings.PREF_AUTO_CORRECTION, true) + } - setText("") // (re)sets selection and composing word + setText("") // initializes the input connection before switching subtype + latinIME.switchToSubtype(SubtypeSettings.getResourceSubtypesForLocale("en_US".constructLocale()) + .first { it.languageTag == "en-US" }) + setText("") // (re)sets selection and composing word for the English subtype } private fun chainInput(text: String) = text.forEach { input(it.code) } @@ -1212,8 +1367,14 @@ class ShadowFacilitator2 { ngramContext: NgramContext, timeStampInSeconds: Long, blockPotentiallyOffensive: Boolean) { lastAddedWord = suggestion + lastNgramContext = ngramContext.extractPrevWordsContext() + addedWords.add(suggestion) + ngramContexts.add(lastNgramContext) } companion object { var lastAddedWord = "" + var lastNgramContext = "" + val addedWords = mutableListOf() + val ngramContexts = mutableListOf() } } diff --git a/app/src/test/java/helium314/keyboard/latin/ScriptUtilsTest.kt b/app/src/test/java/helium314/keyboard/latin/ScriptUtilsTest.kt index 9134a5872..3b98802c0 100644 --- a/app/src/test/java/helium314/keyboard/latin/ScriptUtilsTest.kt +++ b/app/src/test/java/helium314/keyboard/latin/ScriptUtilsTest.kt @@ -5,9 +5,12 @@ import helium314.keyboard.latin.common.LocaleUtils.constructLocale import helium314.keyboard.latin.utils.ScriptUtils.SCRIPT_CYRILLIC import helium314.keyboard.latin.utils.ScriptUtils.SCRIPT_DEVANAGARI import helium314.keyboard.latin.utils.ScriptUtils.SCRIPT_LATIN +import helium314.keyboard.latin.utils.ScriptUtils.needsWordSegmentation import helium314.keyboard.latin.utils.ScriptUtils.script import kotlin.test.Test import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertTrue class ScriptUtilsTest { @Test fun defaultScript() { @@ -18,4 +21,18 @@ class ScriptUtilsTest { assertEquals(SCRIPT_CYRILLIC, "mk".constructLocale().script()) assertEquals(SCRIPT_CYRILLIC, "fr-Cyrl".constructLocale().script()) } + + @Test fun needsWordSegmentationThai() { + assertTrue(needsWordSegmentation("th".constructLocale())) + } + + @Test fun needsWordSegmentationNonThai() { + assertFalse(needsWordSegmentation("en".constructLocale())) + assertFalse(needsWordSegmentation("ja".constructLocale())) + assertFalse(needsWordSegmentation("zh".constructLocale())) + assertFalse(needsWordSegmentation("lo".constructLocale())) + assertFalse(needsWordSegmentation("km".constructLocale())) + assertFalse(needsWordSegmentation("ko".constructLocale())) + assertFalse(needsWordSegmentation("my".constructLocale())) + } }