Key learner words by language and lemma, not by meaning wording

apps/android/app/src/main/java/chat/mural/core/Languages.kt

@@ -32,7 +32,7 @@ object Themes { val shared = listOf(
 data class LanguageModule(
     val id: String, val name: String, val nativeName: String, val variety: String, val locale: String,
     val greeting: String, val greetingWord: String, val speechGuidance: String, val writingGuidance: String,
-    val lemmaGuidance: String, val teachingFocus: List<String>, val topicPlaceholder: String,
+    val lemmaGuidance: String, val lemmaPrefixes: List<String> = emptyList(), val teachingFocus: List<String>, val topicPlaceholder: String,
     val lookupUnavailableReply: String, val themeOverrides: Map<String, ConversationTheme> = emptyMap()
 ) {
     val themes get() = Themes.shared.map { themeOverrides[it.id] ?: it }
@@ -56,6 +56,7 @@ object LanguageRegistry {
         lemmaGuidance = "Give nouns with their singular grammatical article and verbs in the infinitive, for example en tur and å gå. Accept valid gender variants.",
         topicPlaceholder = "Design, space, life in Norway…",
         lookupUnavailableReply = "Jeg klarte ikke å sjekke det akkurat nå. Vi kan snakke om temaet generelt, hvis du vil.",
+        lemmaPrefixes = listOf("en", "ei", "et", "å"),
         teachingFocus = listOf("Greetings, introductions and short everyday chunks.", "Simple questions, noun gender and present-tense everyday exchanges.", "Connected stories, past tense, word order and familiar situations.", "Reasons and opinions, subordinate clauses and natural connectors.", "Nuanced discussion, idiomatic phrasing and register.", "Flexible advanced conversation with precise, natural Norwegian."),
         themeOverrides = mapOf("groceries" to ConversationTheme("groceries", "At the market", "Find the good tomatoes", "basket", "Everyday", "Help the learner shop at a Norwegian food market. Practise quantities and questions.", 2),
             "travel" to ConversationTheme("travel", "Next stop", "A ticket to somewhere", "tram", "Everyday", "Plan a train trip in Norway. Discuss routes and tickets without inventing current schedules.", 1),
@@ -76,6 +77,7 @@ object LanguageRegistry {
         lemmaGuidance = "Give nouns with their singular grammatical article and verbs in the infinitive, for example la casa and hablar. Keep reflexive verbs such as llamarse distinct. Preserve accents and ñ.",
         topicPlaceholder = "Food, travel, music, life in Spain…",
         lookupUnavailableReply = "No he podido comprobarlo ahora mismo. Si quieres, podemos hablar del tema en general.",
+        lemmaPrefixes = listOf("el", "la", "los", "las", "un", "una", "unos", "unas"),
         teachingFocus = listOf("Greetings, introductions and short useful chunks such as me llamo and quiero.", "Everyday questions, gender and number agreement, present tense and useful ser/estar contrasts.", "Connected stories, past events, object pronouns and familiar situations.", "Reasons and opinions, contrasts between past tenses and common subjunctive contexts.", "Nuance, hypothetical situations, register and regional variation.", "Flexible advanced discussion with precise, idiomatic Spanish."),
         themeOverrides = mapOf("coffee" to ConversationTheme("coffee", "Un café", "Something warm, please", "cup.and.saucer", "Everyday", "Meet in a neighbourhood café in Spain. Order a drink and chat. Ask about the learner's interests.", 0),
             "groceries" to ConversationTheme("groceries", "En el mercado", "A little of everything", "basket", "Everyday", "Visit a local market in a Spanish-speaking community. Practise quantities, prices and polite questions. Respect regional food vocabulary.", 2),
@@ -96,6 +98,7 @@ object LanguageRegistry {
         lemmaGuidance = "Give countable nouns in the singular and verbs in the base form, for example a journey and go. Keep meaningful phrasal verbs such as look after together. Use a short, plain English definition as the stable sense rather than repeating the word itself.",
         topicPlaceholder = "Travel, films, work, everyday life…",
         lookupUnavailableReply = "I couldn't check that just now. We can talk about the topic more generally, if you like.",
+        lemmaPrefixes = listOf("a", "an", "the"),
         teachingFocus = listOf("Greetings, introductions and useful everyday chunks such as I'd like and my name is.", "Everyday questions, present forms, articles and common countable and uncountable nouns.", "Connected stories, past events, future plans and familiar situations.", "Reasons and opinions, present perfect in context, conditionals and natural linking phrases.", "Nuance, idiomatic expressions, reported speech and appropriate register.", "Flexible advanced discussion with precise language, implication and tact."),
         themeOverrides = mapOf("coffee" to ConversationTheme("coffee", "A coffee?", "Something warm, please", "cup.and.saucer", "Everyday", "Meet in a neighbourhood café. Order a drink and chat in English. Follow the learner's interests and accept regional vocabulary.", 0),
             "travel" to ConversationTheme("travel", "Next stop", "A ticket to somewhere", "tram", "Everyday", "Plan a trip using English. Let the learner choose the destination. Discuss transport and tickets without inventing current schedules.", 1),
@@ -114,6 +117,7 @@ object LanguageRegistry {
         lemmaGuidance = "Give nouns with a singular article that makes gender clear where possible and verbs in the infinitive, for example une maison, un ami and parler. Keep pronominal verbs such as se souvenir distinct. Preserve accents and meaningful elisions.",
         topicPlaceholder = "Food, cinema, travel, life in France…",
         lookupUnavailableReply = "Je n'ai pas pu vérifier ça pour le moment. On peut parler du sujet en général, si tu veux.",
+        lemmaPrefixes = listOf("le", "la", "les", "l'", "l’", "un", "une", "des", "du", "de la"),
         teachingFocus = listOf("Greetings, introductions and useful everyday chunks such as je m'appelle and je voudrais.", "Everyday questions, grammatical gender, present tense and common negation in conversation.", "Connected stories, passé composé and imparfait in context, future plans and familiar situations.", "Reasons and opinions, object pronouns, conditional requests and common subjunctive contexts.", "Nuance, hypothetical situations, register, idiomatic phrasing and regional variation.", "Flexible advanced discussion with precise, natural French and appropriate tone."),
         themeOverrides = mapOf("coffee" to ConversationTheme("coffee", "Un café ?", "Something warm, please", "cup.and.saucer", "Everyday", "Meet in a neighbourhood café in France. Order a drink and chat. Use polite greetings with staff and a friendly register with the learner.", 0),
             "groceries" to ConversationTheme("groceries", "Au marché", "A little of everything", "basket", "Everyday", "Visit a local market in France. Practise quantities, prices and polite requests, then ask what the learner likes to cook.", 2),
@@ -134,6 +138,7 @@ object LanguageRegistry {
         lemmaGuidance = "Give nouns with their singular article and verbs in the infinitive, for example das Haus, die Straße and sprechen. Preserve umlauts and ß. Keep separable verbs such as aufstehen and reflexive verbs such as sich erinnern together as dictionary entries, while quoting the learner's actual word order exactly.",
         topicPlaceholder = "Food, travel, music, life in Germany…",
         lookupUnavailableReply = "Das konnte ich gerade nicht überprüfen. Wenn du möchtest, können wir allgemein über das Thema sprechen.",
+        lemmaPrefixes = listOf("der", "die", "das", "den", "dem", "des", "ein", "eine", "einen", "einem", "einer", "eines"),
         teachingFocus = listOf("Greetings, introductions and useful everyday chunks such as ich heiße and ich möchte.", "Everyday questions, grammatical gender, present tense, verb-second word order and common accusative objects.", "Connected stories, conversational past tenses, dative uses, separable verbs and familiar situations.", "Reasons and opinions, subordinate-clause word order, relative clauses and polite Konjunktiv II requests.", "Nuance, hypothetical situations, passive voice, idiomatic phrasing and regional register.", "Flexible advanced discussion with precise, natural German and appropriate tone."),
         themeOverrides = mapOf("coffee" to ConversationTheme("coffee", "Ein Kaffee?", "Something warm, please", "cup.and.saucer", "Everyday", "Meet in a neighbourhood café in Germany. Order a drink and chat. Use polite greetings with staff and follow the learner's interests.", 0),
             "groceries" to ConversationTheme("groceries", "Auf dem Markt", "A little of everything", "basket", "Everyday", "Shop at a weekly market in Germany. Practise quantities, prices and polite requests, accepting regional names for foods.", 2),
@@ -154,6 +159,7 @@ object LanguageRegistry {
         lemmaGuidance = "Give nouns with their singular article and verbs in the infinitive, for example la casa, lo studente and parlare. Preserve elisions and accents. Keep reflexive verbs such as chiamarsi and pronominal verbs such as farcela distinct.",
         topicPlaceholder = "Food, cinema, travel, life in Italy…",
         lookupUnavailableReply = "Non sono riuscito a verificarlo adesso. Se vuoi, possiamo parlare dell'argomento in generale.",
+        lemmaPrefixes = listOf("il", "lo", "la", "l'", "l’", "i", "gli", "le", "un", "uno", "una", "un'", "un’"),
         teachingFocus = listOf("Greetings, introductions and useful everyday chunks such as mi chiamo and vorrei.", "Everyday questions, gender and number agreement, present tense and common prepositions.", "Connected stories, passato prossimo and imperfetto in context, future plans and familiar situations.", "Reasons and opinions, object pronouns, conditional requests and common congiuntivo contexts.", "Nuance, hypothetical situations, pronoun combinations, idiomatic phrasing and regional register.", "Flexible advanced discussion with precise, natural Italian and appropriate tone."),
         themeOverrides = mapOf("coffee" to ConversationTheme("coffee", "Un caffè?", "Something warm, please", "cup.and.saucer", "Everyday", "Meet at a neighbourhood bar in Italy for a coffee. Order a drink, greet the staff politely and chat about the learner's day.", 0),
             "groceries" to ConversationTheme("groceries", "Al mercato", "A little of everything", "basket", "Everyday", "Visit a local market in Italy. Practise quantities, prices and polite requests, then ask what the learner likes to cook.", 2),
@@ -174,6 +180,7 @@ object LanguageRegistry {
         lemmaGuidance = "Give nouns with their singular article and verbs in the infinitive, for example a casa, o pão and falar. Preserve accents, nasal vowels and ç. Keep reflexive and pronominal verbs such as se lembrar distinct. Use a consistent Brazilian dictionary form without treating regional alternatives as errors.",
         topicPlaceholder = "Food, music, travel, life in Brazil…",
         lookupUnavailableReply = "Não consegui verificar isso agora. Se quiser, podemos conversar sobre o assunto de forma geral.",
+        lemmaPrefixes = listOf("o", "a", "os", "as", "um", "uma", "uns", "umas"),
         teachingFocus = listOf("Greetings, introductions and useful everyday chunks such as meu nome é and eu gostaria de.", "Everyday questions, gender and number agreement, present tense, ser and estar, and você and a gente.", "Connected stories, pretérito perfeito and imperfeito in context, future plans and familiar situations.", "Reasons and opinions, object pronouns, polite requests and common subjunctive contexts.", "Nuance, future subjunctive, personal infinitive, hypothetical situations, idiomatic phrasing and regional register.", "Flexible advanced discussion with precise, natural Brazilian Portuguese and appropriate tone."),
         themeOverrides = mapOf("coffee" to ConversationTheme("coffee", "Um cafezinho?", "Something warm, please", "cup.and.saucer", "Everyday", "Meet at a neighbourhood café or padaria in Brazil. Order a drink and chat about the learner's day, using natural Brazilian vocabulary.", 0),
             "groceries" to ConversationTheme("groceries", "Na feira", "A little of everything", "basket", "Everyday", "Shop at a street market in Brazil. Practise quantities, prices and polite requests, respecting regional food names.", 2),
@@ -194,6 +201,7 @@ object LanguageRegistry {
         lemmaGuidance = "Give vocabulary lemmas in simplified characters only, with no pinyin or English in the lemma; the app supplies pronunciation help separately. Keep the exact observed form and quote, including Traditional Chinese or learner-written pinyin. Use dictionary forms and preserve meaningful chunks such as 洗澡 and 见面. Do not infer tone accuracy, pronunciation or spoken recall from typed pinyin or a transcript alone.",
         topicPlaceholder = "Food, travel, films, everyday life…",
         lookupUnavailableReply = "我现在没法查证这件事。如果你愿意，我们可以先聊聊这个话题的一般情况。",
+        lemmaPrefixes = listOf(),
         teachingFocus = listOf("Greetings, introductions and useful everyday chunks such as 我叫 and 我想要.", "Everyday questions, word order, measure words, numbers and common present-time exchanges.", "Connected stories, completed actions with 了, experiences with 过, and familiar situations.", "Reasons and opinions, comparisons, 把 and 被 constructions, and natural linking phrases.", "Nuance, aspect, conditionals, idiomatic phrasing, register and regional variation.", "Flexible advanced discussion with precise, natural Mandarin and appropriate tone."),
         themeOverrides = mapOf("coffee" to ConversationTheme("coffee", "喝杯咖啡？", "Something warm, please", "cup.and.saucer", "Everyday", "在一家社区咖啡馆见面。用普通话点饮料并聊天，跟着学习者的兴趣展开对话。", 0),
             "groceries" to ConversationTheme("groceries", "去买菜", "Find something good", "basket", "Everyday", "在菜市场或超市买日常食材。练习数量、价格和礼貌的提问，尊重不同地区的食物词汇。", 2),

apps/android/app/src/main/java/chat/mural/core/LearningEngine.kt

@@ -37,7 +37,8 @@ object LearningEngine {
         return proposal.copy(nextGoal=proposal.nextGoal.take(300),capability=proposal.capability.take(160),words=words)
     }
     fun project(sessions:List<SessionRecord>,languageID:String=LanguageRegistry.defaultID,hiddenWords:List<String> = emptyList(),now:Double=nowSeconds()):LearnerState {
-        val hidden=hiddenWords.map { it.canonical() }
+        // Legacy hidden keys carry the meaning as a third segment and predate lemma normalisation; current ids are stored as projected.
+        val hidden=hiddenWords.map { stored -> stored.split('|').let { if(it.size>=3) wordKey(it[0],it[1]) else stored } }
         var level=0; var count=0; var successes=0
         var nextGoal="Start with a greeting and one small question. Adjust from what the learner actually says."
         val caps=mutableMapOf<String,MutableSet<String>>()

apps/android/app/src/main/java/chat/mural/core/Models.kt

@@ -80,12 +80,32 @@ fun String.canonical(): String = java.text.Normalizer.normalize(this, java.text.
 
 fun String.containsCanonical(other: String): Boolean = canonical().contains(other.canonical(), ignoreCase = true)
 
+/** Groups every observation of one dictionary word: articles, case, spacing and the meaning wording do not split it. */
+fun wordKey(language: String, lemma: String): String {
+    // Lowercasing can break NFC, so compose last; whitespace follows Unicode White_Space like Swift's Character.isWhitespace.
+    var text = lemma.lowercase().splitWhere { it.isWhitespace() || it == '\u0085' }.joinToString(" ").canonical()
+    for (prefix in LanguageRegistry.get(language)?.lemmaPrefixes ?: emptyList()) {
+        val marker = if (prefix.endsWith("'") || prefix.endsWith("’")) prefix else "$prefix "
+        if (!text.startsWith(marker)) continue
+        val rest = text.removePrefix(marker).trim()
+        if (rest.isNotEmpty()) { text = rest; break }
+    }
+    return "$language|$text"
+}
+
+private fun String.splitWhere(isSeparator: (Char) -> Boolean): List<String> {
+    val parts = mutableListOf<String>(); val current = StringBuilder()
+    for (c in this) if (isSeparator(c)) { if (current.isNotEmpty()) { parts += current.toString(); current.clear() } } else current.append(c)
+    if (current.isNotEmpty()) parts += current.toString()
+    return parts
+}
+
 @Serializable
 data class WordProposal(
     val lemma: String, val meaning: String, val form: String, val kind: EvidenceKind,
     val confidence: Double, val sourceIDs: List<String>, val quote: String,
     val language: String = LanguageRegistry.defaultID
-) { val key get() = "${language}|${lemma.trim().lowercase().canonical()}|${meaning.lowercase().canonical()}" }
+) { val key get() = wordKey(language, lemma) }
 
 @Serializable
 data class Assessment(