diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/core/presentation/text/HebrewSearch.kt b/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/core/presentation/text/HebrewSearch.kt index 8020849a3..93b62fa2f 100644 --- a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/core/presentation/text/HebrewSearch.kt +++ b/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/core/presentation/text/HebrewSearch.kt @@ -1,8 +1,8 @@ package io.github.kdroidfilter.seforimapp.core.presentation.text /* - * Utilities to perform Hebrew-aware, diacritic-insensitive search. - * Strips nikud (vowel points) and ta'amim (cantillation) and normalizes final letters + * Utilities to perform Hebrew-aware, diacritic- and punctuation-insensitive search. + * Strips nikud (vowel points), ta'amim (cantillation) and punctuation and normalizes final letters * to their base forms for matching, while preserving a mapping back to original indices * so we can highlight the right character ranges. */ @@ -63,21 +63,14 @@ internal fun replaceFinalsWithBase(text: String): String = .replace('\u05E3', '\u05E4') // ף -> פ .replace('\u05E5', '\u05E6') // ץ -> צ +/** + * Normalizes a find query exactly as [findAllMatchesNormalized] normalizes the text it scans: + * nikud and ta'amim dropped, punctuation (maqaf included) treated as a word break, quotes inside + * a word (רש"י, רש״י) dropped, whitespace collapsed and trimmed, final letters folded, lowercased. + */ internal fun normalizeQueryForHebrew(raw: String): String { if (raw.isBlank()) return "" - var s = raw.trim() - // Remove biblical cantillation marks (teamim) U+0591–U+05AF - s = s.replace("[\u0591-\u05AF]".toRegex(), "") - // Remove nikud signs including meteg and qamatz qatan - s = s.replace("[\u05B0\u05B1\u05B2\u05B3\u05B4\u05B5\u05B6\u05B7\u05B8\u05B9\u05BB\u05BC\u05BD\u05C1\u05C2\u05C7]".toRegex(), "") - // Replace maqaf with space and remove gershayim/geresh - s = s.replace('\u05BE', ' ') - s = s.replace("\u05F4", "").replace("\u05F3", "") - // Normalize Hebrew final letters (sofit) to base forms - s = replaceFinalsWithBase(s) - // Lowercase for case-insensitive match (safe for Latin) - s = s.lowercase() - return s + return foldForFind(stripForFindWithMap(raw).first) } internal fun mapToOrigIndex( @@ -144,8 +137,8 @@ internal fun findAllMatchesNormalized( ): List { if (q.length < 2) return emptyList() - val (plain, map) = stripDiacriticsWithMap(text) - val plainSearch = replaceFinalsWithBase(plain).lowercase() + val (plain, map) = stripForFindWithMap(text) + val plainSearch = foldForFind(plain) val out = mutableListOf() var from = 0 @@ -153,9 +146,113 @@ internal fun findAllMatchesNormalized( val idx = plainSearch.indexOf(q, startIndex = from) if (idx == -1) break val startOrig = mapToOrigIndex(map, idx) - val endOrig = mapToOrigIndex(map, idx + q.length - 1) + 1 + var endOrig = mapToOrigIndex(map, idx + q.length - 1) + 1 + // Keep the vowel points/cantillation of the last matched letter inside the range + while (endOrig < text.length && isNikudOrTeamim(text[endOrig])) endOrig++ if (endOrig > startOrig) out += (startOrig until endOrig) from = idx + q.length } return out } + +// Quote marks that write geresh/gershayim (רש"י, ס'): dropped, not a word break, between two letters. +// A String rather than a Set so membership checks do not box every scanned character. +private const val WORD_QUOTES = + "\"'`" + + "´" + // ´ acute accent + "‘’" + // ‘ ’ + "“”" + // “ ” + "′″" + // ′ ″ prime, double prime + "׳״" // ׳ geresh, ״ gershayim + +// Unicode general categories of punctuation (P*) +private fun isUnicodePunctuationType(type: Int): Boolean = + when (type.toByte()) { + Character.CONNECTOR_PUNCTUATION, + Character.DASH_PUNCTUATION, + Character.START_PUNCTUATION, + Character.END_PUNCTUATION, + Character.INITIAL_QUOTE_PUNCTUATION, + Character.FINAL_QUOTE_PUNCTUATION, + Character.OTHER_PUNCTUATION, + -> true + else -> false + } + +/** + * True for the characters find-in-page ignores: every printable ASCII character that is neither a + * letter, a digit nor a space (brackets and symbols such as + = < > | ~ $ included), any Unicode + * punctuation (general category P*: maqaf ־, sof pasuq ׃, paseq ׀, geresh ׳, gershayim ״, curly + * quotes, dashes…), and the quote look-alikes in [WORD_QUOTES]. + */ +fun isSearchPunctuation(c: Char): Boolean = + when { + c.code < 0x80 -> c.code in 0x21..0x7E && !c.isLetterOrDigit() + c in WORD_QUOTES -> true + else -> isUnicodePunctuationType(Character.getType(c)) + } + +/** + * The text find-in-page matches against, plus an index map from stripped index -> original index. + * Nikud and ta'amim are dropped; a quote mark between two letters (רש"י, רש״י, רש''י) is dropped + * too, so the acronym reads as one word; any other run of punctuation and/or whitespace becomes a + * single space, mapped to the run's first character, so "שלום, עולם" and "שלום,עולם" both read + * "שלום עולם". Leading and trailing runs are dropped, so a match never starts or ends on one. + * + * Unlike [stripDiacriticsWithMap] (kept as is for the TOC search), this also drops punctuation. + */ +fun stripForFindWithMap(src: String): Pair { + val out = StringBuilder(src.length) + val map = IntArray(src.length) + // Original index of the first character of the pending word break, -1 when there is none + var breakAt = -1 + for (i in src.indices) { + val ch = src[i] + when { + isNikudOrTeamim(ch) -> Unit + breakAt < 0 && isQuoteInsideWord(src, i, out) -> Unit + ch.isWhitespace() || isSearchPunctuation(ch) -> if (breakAt < 0) breakAt = i + else -> { + if (breakAt >= 0 && out.isNotEmpty()) { + map[out.length] = breakAt + out.append(' ') + } + breakAt = -1 + map[out.length] = i + out.append(ch) + } + } + } + val arr = if (out.length == map.size) map else map.copyOf(out.length) + return out.toString() to arr +} + +// Whether src[i] is a quote mark between the letter last kept in [out] and a letter further on +// (nikud and other quote marks skipped) +private fun isQuoteInsideWord( + src: String, + i: Int, + out: CharSequence, +): Boolean { + if (src[i] !in WORD_QUOTES || out.isEmpty() || !out.last().isLetter()) return false + var j = i + 1 + while (j < src.length && (src[j] in WORD_QUOTES || isNikudOrTeamim(src[j]))) j++ + return j < src.length && src[j].isLetter() +} + +// Folds final letters and case one character for one, so indices stay aligned with the map +private fun foldForFind(s: String): String = + buildString(s.length) { + for (c in s) { + val base = + when (c) { + 'ך' -> 'כ' // ך -> כ + 'ם' -> 'מ' // ם -> מ + 'ן' -> 'נ' // ן -> נ + 'ף' -> 'פ' // ף -> פ + 'ץ' -> 'צ' // ץ -> צ + else -> c + } + append(base.lowercaseChar()) + } + } diff --git a/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/core/presentation/text/HebrewSearchTest.kt b/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/core/presentation/text/HebrewSearchTest.kt index 8df68f823..8e1553c53 100644 --- a/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/core/presentation/text/HebrewSearchTest.kt +++ b/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/core/presentation/text/HebrewSearchTest.kt @@ -2,6 +2,7 @@ package io.github.kdroidfilter.seforimapp.core.presentation.text import kotlin.test.Test import kotlin.test.assertEquals +import kotlin.test.assertFalse import kotlin.test.assertTrue class HebrewSearchTest { @@ -190,4 +191,142 @@ class HebrewSearchTest { val result = findAllMatchesOriginal("שלום עולם", "בוקר") assertTrue(result.isEmpty()) } + + // Punctuation-insensitive find + @Test + fun `isSearchPunctuation covers ASCII, Hebrew and Unicode punctuation`() { + val ascii = ",.;:!?\"'`()[]{}<>-_/|+=*&^%#@~\\\$" + // maqaf, sof pasuq, paseq, geresh, gershayim, curly quotes, dashes, guillemets, ellipsis + val unicode = "\u05BE\u05C3\u05C0\u05F3\u05F4\u201C\u201D\u2018\u2019\u2014\u2013\u00AB\u00BB\u2026" + val punctuation = ascii + unicode + for (c in punctuation) assertTrue(isSearchPunctuation(c), "expected punctuation: U+%04X".format(c.code)) + for (c in "אבa Z09\t\u05B0\u0591") assertFalse(isSearchPunctuation(c), "expected not punctuation: U+%04X".format(c.code)) + } + + @Test + fun `stripForFindWithMap turns punctuation runs into one space and maps back`() { + val text = "שלום, עולם." + val (plain, map) = stripForFindWithMap(text) + assertEquals("שלום עולם", plain) + assertEquals(plain.length, map.size) + // The space stands for the comma, the first character of the run + assertEquals(4, map[4]) + assertEquals(7, map[5]) + assertEquals(10, map.last()) + } + + @Test + fun `stripForFindWithMap drops leading and trailing punctuation`() { + assertEquals("שלום", stripForFindWithMap(" (שלום)! ").first) + assertEquals("", stripForFindWithMap(" ,.; ").first) + } + + @Test + fun `find matches across a comma between words`() { + val text = "שלום, עולם" + val result = findAllMatchesOriginal(text, "שלום עולם") + assertEquals(1, result.size) + assertEquals(text, text.substring(result[0])) + assertEquals(1, findAllMatchesOriginal("שלום,עולם", "שלום עולם").size) + } + + @Test + fun `find ignores punctuation typed in the query only`() { + assertEquals(1, findAllMatchesOriginal("שלום עולם", "שלום, עולם!").size) + assertEquals(1, findAllMatchesOriginal("שלום עולם", "שלום-עולם").size) + assertEquals(1, findAllMatchesOriginal("שלום עולם", "\"שלום\"").size) + assertEquals(1, findAllMatchesOriginal("שלום עולם", "שלום עולם").size) + } + + @Test + fun `find matches acronyms with or without gershayim, both ways`() { + val ascii = "כתב רש\"י כאן" + val hebrew = "כתב רש\u05F4י כאן" + for (text in listOf(ascii, hebrew)) { + val result = findAllMatchesOriginal(text, "רשי") + assertEquals(1, result.size) + assertEquals(text.substring(4, 8), text.substring(result[0])) + } + assertEquals(1, findAllMatchesOriginal("כתב רשי כאן", "רש\"י").size) + assertEquals(1, findAllMatchesOriginal("כתב רשי כאן", "רש\u05F4י").size) + assertEquals(1, findAllMatchesOriginal(ascii, "רש\u05F4י").size) + assertEquals(1, findAllMatchesOriginal(hebrew, "רש\"י").size) + assertEquals(1, findAllMatchesOriginal("כתב רש''י כאן", "רשי").size) + assertEquals(1, findAllMatchesOriginal("ס' שלום", "ס שלום").size) + } + + @Test + fun `quote marks opening or closing a quoted word are word breaks, not dropped`() { + val text = "ואמר \"שלום\" לו" + assertEquals("ואמר שלום לו", stripForFindWithMap(text).first) + assertEquals("שלום", text.substring(findAllMatchesOriginal(text, "שלום")[0])) + assertEquals(1, findAllMatchesOriginal(text, "ואמר שלום לו").size) + assertEquals(1, findAllMatchesOriginal("ואמר “שלום” לו", "ואמר שלום").size) + } + + @Test + fun `find matches a Hebrew geresh abbreviation both ways`() { + assertEquals(1, findAllMatchesOriginal("ס׳ שלום", "ס שלום").size) + assertEquals(1, findAllMatchesOriginal("ס' שלום", "ס׳ שלום").size) + assertEquals(1, findAllMatchesOriginal("ס שלום", "ס׳ שלום").size) + } + + @Test + fun `find matches words joined by a maqaf`() { + val text = "את כל\u05BEהארץ" + assertEquals(1, findAllMatchesOriginal(text, "כל הארץ").size) + // Used to never match: the query maqaf became a space while the text kept its own + assertEquals(1, findAllMatchesOriginal(text, "כל\u05BEהארץ").size) + assertEquals(1, findAllMatchesOriginal("את כל הארץ", "כל\u05BEהארץ").size) + assertEquals("כל\u05BEהארץ", text.substring(findAllMatchesOriginal(text, "כל הארץ")[0])) + } + + @Test + fun `find matches across a sof pasuq and leaves it out of a final match`() { + val text = "ויהי אור\u05C3 וירא" + assertEquals(1, findAllMatchesOriginal(text, "אור וירא").size) + assertEquals("אור", text.substring(findAllMatchesOriginal(text, "אור")[0])) + assertEquals("אור", text.substring(findAllMatchesOriginal(text, "אור\u05C3")[0])) + } + + @Test + fun `find ignores punctuation inside pointed text`() { + val text = "בְּרֵאשִׁית, בָּרָא" + val result = findAllMatchesOriginal(text, "בראשית ברא") + assertEquals(1, result.size) + assertEquals(0, result[0].first) + assertEquals(text.length - 1, result[0].last) + assertEquals(1, findAllMatchesOriginal("רַשִׁ\"י", "רשי").size) + } + + @Test + fun `find offsets point into the original text`() { + val text = "(שלום) ושלום" + val result = findAllMatchesOriginal(text, "שלום") + assertEquals(listOf(1..4, 8..11), result) + } + + @Test + fun `highlight range excludes surrounding punctuation`() { + val text = "אמר: \"שלום, עולם.\"" + val result = findAllMatchesOriginal(text, "שלום עולם.") + assertEquals(1, result.size) + assertEquals("שלום, עולם", text.substring(result[0])) + assertEquals("עולם", text.substring(findAllMatchesOriginal(text, "עולם")[0])) + } + + @Test + fun `highlight range keeps the points of the last matched letter`() { + val text = "\u05E9\u05C1\u05B8\u05DC\u05D5\u05B9\u05DD" + assertEquals(listOf(0..5), findAllMatchesOriginal(text, "שלו")) + } + + @Test + fun `normalizeQueryForHebrew drops punctuation like the text`() { + assertEquals("שלומ עולמ", normalizeQueryForHebrew(" שלום, עולם! ")) + assertEquals("רשי", normalizeQueryForHebrew("רש\"י")) + assertEquals("רשי", normalizeQueryForHebrew("רש\u05F4י")) + assertEquals("כל הארצ", normalizeQueryForHebrew("כל\u05BEהארץ")) + assertEquals("", normalizeQueryForHebrew("!?")) + } } diff --git a/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/usecases/BookFindMatchesTest.kt b/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/usecases/BookFindMatchesTest.kt index 8da662cb5..3589315c4 100644 --- a/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/usecases/BookFindMatchesTest.kt +++ b/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/usecases/BookFindMatchesTest.kt @@ -116,6 +116,17 @@ class BookFindMatchesTest { assertEquals(BookFindMatches.Match(2, 12, ordinal = 1, position = 4), matches.nextAfterLine(3, forward = false)) } + @Test + fun `counts matches whatever the punctuation, in the query or the text`() = + runTest { + // Line 2 reads "ויברא אלהים, ויברא" + assertEquals(1, find("אלהים ויברא").occurrences) + assertEquals(1, find("אלהים; ויברא!").occurrences) + assertEquals(0, find(",.").occurrences) + assertTrue(queryNarrows("אלהים, ויברא", "אלהים")) + assertTrue(queryNarrows("אלהים ויברא", "אלהים,")) + } + @Test fun `matches against the displayed text, not the html`() = runTest {