From ec171206f0d9cb55b0f37f75567e9733ff7777f3 Mon Sep 17 00:00:00 2001 From: Asaf Mahlev Date: Sun, 7 Jun 2026 08:56:03 +0300 Subject: [PATCH] feat(dictionary): graduated trust for non-dictionary learned words (#39) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A learned word in no real dictionary (main/contacts/apps/personal) could out-rank a real dictionary word with better geometry after a single misfire. Now, as the LAST ranking step in getSuggestionResults (after session boost, so it can't be undone), an uncurated USER_HISTORY candidate that still outscores the best real-dictionary candidate is CAPPED just below it — until its user-history frequency crosses a confirmation threshold (~3 repetitions). This guarantees a one-off junk word can't hijack a real word regardless of native score magnitude, while a deliberately repeated new word still learns and keeps full score. When no real candidate exists, new words are left untouched (still offerable). - Capping (not score-scaling) avoids any dependence on native score calibration or sign; applied post-session-boost so the boost can't re-promote junk. - Decision is a pure companion helper (shouldPenalizeUnconfirmedWord), unit-tested; uncurated check reuses isInNonHistoryDictionary; threshold is a tunable constant. - Gated by new pref PREF_GRADUATED_TRUST (default on). The actual ranking effect needs the native scorer, so the threshold and the 'real candidate' set want on-device playtesting. --- .../latin/DictionaryFacilitatorImpl.kt | 53 +++++++++++++++++++ .../keyboard/latin/settings/Defaults.kt | 1 + .../keyboard/latin/settings/Settings.java | 1 + .../latin/settings/SettingsValues.java | 3 ++ .../settings/screens/DictionaryScreen.kt | 35 ++++++++++++ app/src/main/res/values/strings.xml | 4 ++ .../keyboard/latin/DictionaryGroupTest.kt | 13 +++++ 7 files changed, 110 insertions(+) diff --git a/app/src/main/java/helium314/keyboard/latin/DictionaryFacilitatorImpl.kt b/app/src/main/java/helium314/keyboard/latin/DictionaryFacilitatorImpl.kt index bea3a62a9..a6195d678 100644 --- a/app/src/main/java/helium314/keyboard/latin/DictionaryFacilitatorImpl.kt +++ b/app/src/main/java/helium314/keyboard/latin/DictionaryFacilitatorImpl.kt @@ -564,6 +564,11 @@ class DictionaryFacilitatorImpl : DictionaryFacilitator { includeAtLeastTwoWordSuggestions(suggestionResults, suggestionsArray, composedData.mTypedWord) + // Graduated trust (#39): runs LAST (after session boost) so it can't be undone, and caps + // rather than scales so the guarantee holds regardless of native score magnitudes. + if (Settings.getValues().mGraduatedTrust) + applyGraduatedTrust(suggestionResults) + return suggestionResults } @@ -640,6 +645,42 @@ class DictionaryFacilitatorImpl : DictionaryFacilitator { } } + /** + * Graduated trust (#39): cap an uncurated learned word (one in no real dictionary) just below the + * best real-dictionary candidate until it has been confirmed by enough repetitions. This + * guarantees a single misfire can't out-rank a real word with better geometry, while a + * deliberately repeated new word still learns and, once confirmed, keeps its full score. Runs + * after session boost so it can't be undone, and only when a real candidate exists to protect. + */ + private fun applyGraduatedTrust(results: SuggestionResults) { + var maxRealScore = Int.MIN_VALUE + for (info in results) { + when (info.mSourceDict?.mDictType) { + Dictionary.TYPE_MAIN, Dictionary.TYPE_CONTACTS, Dictionary.TYPE_APPS, Dictionary.TYPE_USER -> + if (info.mScore > maxRealScore) maxRealScore = info.mScore + } + } + if (maxRealScore == Int.MIN_VALUE) return // no real word to protect — keep new words offerable + + val toRemove = mutableListOf() + val capped = mutableListOf() + for (info in results) { + if (info.mSourceDict?.mDictType != Dictionary.TYPE_USER_HISTORY) continue + if (info.mScore < maxRealScore) continue // already ranked below a real word + val word = info.mWord + if (word.length <= 1) continue + if (dictionaryGroups.any { it.isInNonHistoryDictionary(word) }) continue // curated, trust it + val freq = dictionaryGroups.maxOf { it.getSubDict(Dictionary.TYPE_USER_HISTORY)?.getFrequency(word) ?: -1 } + if (!shouldPenalizeUnconfirmedWord(true, freq)) continue + toRemove.add(info) + capped.add(SuggestedWordInfo(info.mWord, info.mPrevWordsContext, maxRealScore - 1, + info.mKindAndFlags, info.mSourceDict, info.mIndexOfTouchPointOfSecondWord, + info.mAutoCommitFirstWordConfidence)) + } + for (item in toRemove) results.remove(item) + for (item in capped) results.add(item) + } + // Spell checker is using this, and has its own instance of DictionaryFacilitatorImpl, // meaning that it always has default mConfidence. So we cannot choose to only check preferred // locale, and instead simply return true if word is in any of the available dictionaries @@ -726,6 +767,18 @@ class DictionaryFacilitatorImpl : DictionaryFacilitator { // Multiplier to convert session boost values into score-space (native scores are ~1_000_000) private const val BOOST_SCORE_MULTIPLIER = 1000f + // Graduated trust (#39): a non-dictionary learned word (one not in main/contacts/apps/the + // user's personal dict) that has not yet been confirmed by enough repetitions must not + // out-rank a real dictionary word with better geometry. Until then it is capped just below + // the best real candidate (see applyGraduatedTrust); once repeated past the threshold it + // keeps full score, so deliberate new words still get learned (slowly). History frequency + // rises with use (~111 after 2 uses, ~120 after 3), so 120 ≈ 3 confirmations. Tunable. + private const val GRAD_TRUST_CONFIRM_FREQUENCY = 120 + + /** Graduated-trust decision (#39): penalize an uncurated learned word until it is confirmed. */ + fun shouldPenalizeUnconfirmedWord(isUncurated: Boolean, historyFrequency: Int): Boolean = + isUncurated && historyFrequency < GRAD_TRUST_CONFIRM_FREQUENCY + private fun createSubDict( dictType: String, context: Context, locale: Locale, dictFile: File?, dictNamePrefix: String ): ExpandableBinaryDictionary? { diff --git a/app/src/main/java/helium314/keyboard/latin/settings/Defaults.kt b/app/src/main/java/helium314/keyboard/latin/settings/Defaults.kt index b8d13107f..91354e280 100644 --- a/app/src/main/java/helium314/keyboard/latin/settings/Defaults.kt +++ b/app/src/main/java/helium314/keyboard/latin/settings/Defaults.kt @@ -194,6 +194,7 @@ object Defaults { const val PREF_CLEAR_CLIPBOARD_ICON = "bin" const val PREF_ADD_TO_PERSONAL_DICTIONARY = true const val PREF_FLAG_UNKNOWN_WORDS = true + const val PREF_GRADUATED_TRUST = true @JvmField val PREF_NAVBAR_COLOR = Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q const val PREF_NARROW_KEY_GAPS = true diff --git a/app/src/main/java/helium314/keyboard/latin/settings/Settings.java b/app/src/main/java/helium314/keyboard/latin/settings/Settings.java index d3836063e..57a785c9b 100644 --- a/app/src/main/java/helium314/keyboard/latin/settings/Settings.java +++ b/app/src/main/java/helium314/keyboard/latin/settings/Settings.java @@ -228,6 +228,7 @@ public final class Settings implements SharedPreferences.OnSharedPreferenceChang public static final String PREF_ADD_TO_PERSONAL_DICTIONARY = "add_to_personal_dictionary"; public static final String PREF_FLAG_UNKNOWN_WORDS = "flag_unknown_words"; + public static final String PREF_GRADUATED_TRUST = "graduated_trust"; public static final String PREF_NAVBAR_COLOR = "navbar_color"; public static final String PREF_NARROW_KEY_GAPS = "narrow_key_gaps"; public static final String PREF_NARROW_KEY_GAPS_LEVEL = "narrow_key_gaps_level"; diff --git a/app/src/main/java/helium314/keyboard/latin/settings/SettingsValues.java b/app/src/main/java/helium314/keyboard/latin/settings/SettingsValues.java index 6181ac818..13da16258 100644 --- a/app/src/main/java/helium314/keyboard/latin/settings/SettingsValues.java +++ b/app/src/main/java/helium314/keyboard/latin/settings/SettingsValues.java @@ -154,6 +154,7 @@ public class SettingsValues { public final int mScreenMetrics; public final boolean mAddToPersonalDictionary; public final boolean mFlagUnknownWords; + public final boolean mGraduatedTrust; public final boolean mUseContactsDictionary; public final boolean mUseAppsDictionary; public final boolean mCustomNavBarColor; @@ -471,6 +472,8 @@ public SettingsValues(final Context context, final SharedPreferences prefs, fina Defaults.PREF_ADD_TO_PERSONAL_DICTIONARY); mFlagUnknownWords = prefs.getBoolean(Settings.PREF_FLAG_UNKNOWN_WORDS, Defaults.PREF_FLAG_UNKNOWN_WORDS); + mGraduatedTrust = prefs.getBoolean(Settings.PREF_GRADUATED_TRUST, + Defaults.PREF_GRADUATED_TRUST); mUseContactsDictionary = SettingsValues.readUseContactsEnabled(prefs, context); mUseAppsDictionary = prefs.getBoolean(Settings.PREF_USE_APPS, Defaults.PREF_USE_APPS); mCustomNavBarColor = prefs.getBoolean(Settings.PREF_NAVBAR_COLOR, Defaults.PREF_NAVBAR_COLOR); diff --git a/app/src/main/java/helium314/keyboard/settings/screens/DictionaryScreen.kt b/app/src/main/java/helium314/keyboard/settings/screens/DictionaryScreen.kt index 78f5bee2a..031b6116f 100644 --- a/app/src/main/java/helium314/keyboard/settings/screens/DictionaryScreen.kt +++ b/app/src/main/java/helium314/keyboard/settings/screens/DictionaryScreen.kt @@ -173,6 +173,41 @@ fun DictionaryScreen( ) } androidx.compose.material3.Divider(modifier = Modifier.padding(vertical = 4.dp)) + + // Graduated Trust Setting + var graduatedTrustEnabled by remember { mutableStateOf(ctx.prefs().getBoolean(Settings.PREF_GRADUATED_TRUST, Defaults.PREF_GRADUATED_TRUST)) } + Row( + verticalAlignment = Alignment.CenterVertically, + horizontalArrangement = Arrangement.SpaceBetween, + modifier = Modifier + .padding(vertical = 4.dp, horizontal = 16.dp) + .fillMaxWidth() + .clickable { + val newValue = !graduatedTrustEnabled + graduatedTrustEnabled = newValue + ctx.prefs().edit { putBoolean(Settings.PREF_GRADUATED_TRUST, newValue) } + } + ) { + Column(modifier = Modifier.weight(1f)) { + Text( + stringResource(R.string.graduated_trust), + style = MaterialTheme.typography.titleMedium + ) + Text( + stringResource(R.string.graduated_trust_summary), + style = MaterialTheme.typography.bodyMedium, + color = MaterialTheme.colorScheme.onSurfaceVariant + ) + } + androidx.compose.material3.Switch( + checked = graduatedTrustEnabled, + onCheckedChange = { + graduatedTrustEnabled = it + ctx.prefs().edit { putBoolean(Settings.PREF_GRADUATED_TRUST, it) } + } + ) + } + androidx.compose.material3.Divider(modifier = Modifier.padding(vertical = 4.dp)) // Personal Dictionary Setting val prefs = ctx.prefs() diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index ebcca15d0..088cc1fe7 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -116,6 +116,10 @@ Flag unknown words Mark suggestions that are only learned/typed (not in a dictionary) and offer Add/Block on long-press + + Graduated trust for new words + + A learned word that is not in a dictionary must be repeated a few times before it can outrank a real word — so a single misfire can\'t hijack your typing Add to dictionary diff --git a/app/src/test/java/helium314/keyboard/latin/DictionaryGroupTest.kt b/app/src/test/java/helium314/keyboard/latin/DictionaryGroupTest.kt index 95fb14df8..747e46761 100644 --- a/app/src/test/java/helium314/keyboard/latin/DictionaryGroupTest.kt +++ b/app/src/test/java/helium314/keyboard/latin/DictionaryGroupTest.kt @@ -142,4 +142,17 @@ class DictionaryGroupTest { removeFromBlacklist.invoke(instance, "blockedword") assertEquals(false, isBlacklisted.invoke(instance, "blockedword")) } + + @Test + fun graduatedTrust_penalizesUnconfirmedUncuratedWordsOnly() { + // C4-smart (#39): an uncurated learned word (not in a real dictionary) below the confirmation + // frequency is penalized, so a single misfire can't out-rank a real word... + assertEquals(true, DictionaryFacilitatorImpl.shouldPenalizeUnconfirmedWord(true, 0)) + // ...but once it has been repeated enough it is trusted (deliberate new words still learn)... + assertEquals(false, DictionaryFacilitatorImpl.shouldPenalizeUnconfirmedWord(true, 10000)) + // ...and a real dictionary word is never penalized regardless of how rarely it was learned. + assertEquals(false, DictionaryFacilitatorImpl.shouldPenalizeUnconfirmedWord(false, 0)) + // (the actual capping of the score below the best real candidate happens in + // applyGraduatedTrust, which needs the native scorer and is covered by on-device testing.) + } }