Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
package io.github.kdroidfilter.seforimapp.core.presentation.text

/*
* Utilities to perform Hebrew-aware, diacritic-insensitive search.
* Strips nikud (vowel points) and ta'amim (cantillation) and normalizes final letters
* Utilities to perform Hebrew-aware, diacritic- and punctuation-insensitive search.
* Strips nikud (vowel points), ta'amim (cantillation) and punctuation and normalizes final letters
* to their base forms for matching, while preserving a mapping back to original indices
* so we can highlight the right character ranges.
*/
Expand Down Expand Up @@ -63,21 +63,14 @@ internal fun replaceFinalsWithBase(text: String): String =
.replace('\u05E3', '\u05E4') // ף -> פ
.replace('\u05E5', '\u05E6') // ץ -> צ

/**
* Normalizes a find query exactly as [findAllMatchesNormalized] normalizes the text it scans:
* nikud and ta'amim dropped, punctuation (maqaf included) treated as a word break, quotes inside
* a word (רש"י, רש״י) dropped, whitespace collapsed and trimmed, final letters folded, lowercased.
*/
internal fun normalizeQueryForHebrew(raw: String): String {
if (raw.isBlank()) return ""
var s = raw.trim()
// Remove biblical cantillation marks (teamim) U+0591–U+05AF
s = s.replace("[\u0591-\u05AF]".toRegex(), "")
// Remove nikud signs including meteg and qamatz qatan
s = s.replace("[\u05B0\u05B1\u05B2\u05B3\u05B4\u05B5\u05B6\u05B7\u05B8\u05B9\u05BB\u05BC\u05BD\u05C1\u05C2\u05C7]".toRegex(), "")
// Replace maqaf with space and remove gershayim/geresh
s = s.replace('\u05BE', ' ')
s = s.replace("\u05F4", "").replace("\u05F3", "")
// Normalize Hebrew final letters (sofit) to base forms
s = replaceFinalsWithBase(s)
// Lowercase for case-insensitive match (safe for Latin)
s = s.lowercase()
return s
return foldForFind(stripForFindWithMap(raw).first)
}

internal fun mapToOrigIndex(
Expand Down Expand Up @@ -144,18 +137,122 @@ internal fun findAllMatchesNormalized(
): List<IntRange> {
if (q.length < 2) return emptyList()

val (plain, map) = stripDiacriticsWithMap(text)
val plainSearch = replaceFinalsWithBase(plain).lowercase()
val (plain, map) = stripForFindWithMap(text)
val plainSearch = foldForFind(plain)

val out = mutableListOf<IntRange>()
var from = 0
while (from <= plainSearch.length - q.length) {
val idx = plainSearch.indexOf(q, startIndex = from)
if (idx == -1) break
val startOrig = mapToOrigIndex(map, idx)
val endOrig = mapToOrigIndex(map, idx + q.length - 1) + 1
var endOrig = mapToOrigIndex(map, idx + q.length - 1) + 1
// Keep the vowel points/cantillation of the last matched letter inside the range
while (endOrig < text.length && isNikudOrTeamim(text[endOrig])) endOrig++
if (endOrig > startOrig) out += (startOrig until endOrig)
from = idx + q.length
}
return out
}

// Quote marks that write geresh/gershayim (רש"י, ס'): dropped, not a word break, between two letters.
// A String rather than a Set<Char> so membership checks do not box every scanned character.
private const val WORD_QUOTES =
"\"'`" +
"´" + // ´ acute accent
"‘’" + // ‘ ’
"“”" + // “ ”
"′″" + // ′ ″ prime, double prime
"׳״" // ׳ geresh, ״ gershayim

// Unicode general categories of punctuation (P*)
private fun isUnicodePunctuationType(type: Int): Boolean =
when (type.toByte()) {
Character.CONNECTOR_PUNCTUATION,
Character.DASH_PUNCTUATION,
Character.START_PUNCTUATION,
Character.END_PUNCTUATION,
Character.INITIAL_QUOTE_PUNCTUATION,
Character.FINAL_QUOTE_PUNCTUATION,
Character.OTHER_PUNCTUATION,
-> true
else -> false
}

/**
* True for the characters find-in-page ignores: every printable ASCII character that is neither a
* letter, a digit nor a space (brackets and symbols such as + = < > | ~ $ included), any Unicode
* punctuation (general category P*: maqaf ־, sof pasuq ׃, paseq ׀, geresh ׳, gershayim ״, curly
* quotes, dashes…), and the quote look-alikes in [WORD_QUOTES].
*/
fun isSearchPunctuation(c: Char): Boolean =
when {
c.code < 0x80 -> c.code in 0x21..0x7E && !c.isLetterOrDigit()
c in WORD_QUOTES -> true
else -> isUnicodePunctuationType(Character.getType(c))
}

/**
* The text find-in-page matches against, plus an index map from stripped index -> original index.
* Nikud and ta'amim are dropped; a quote mark between two letters (רש"י, רש״י, רש''י) is dropped
* too, so the acronym reads as one word; any other run of punctuation and/or whitespace becomes a
* single space, mapped to the run's first character, so "שלום, עולם" and "שלום,עולם" both read
* "שלום עולם". Leading and trailing runs are dropped, so a match never starts or ends on one.
*
* Unlike [stripDiacriticsWithMap] (kept as is for the TOC search), this also drops punctuation.
*/
fun stripForFindWithMap(src: String): Pair<String, IntArray> {
val out = StringBuilder(src.length)
val map = IntArray(src.length)
// Original index of the first character of the pending word break, -1 when there is none
var breakAt = -1
for (i in src.indices) {
val ch = src[i]
when {
isNikudOrTeamim(ch) -> Unit
breakAt < 0 && isQuoteInsideWord(src, i, out) -> Unit
ch.isWhitespace() || isSearchPunctuation(ch) -> if (breakAt < 0) breakAt = i
else -> {
if (breakAt >= 0 && out.isNotEmpty()) {
map[out.length] = breakAt
out.append(' ')
}
breakAt = -1
map[out.length] = i
out.append(ch)
}
}
}
val arr = if (out.length == map.size) map else map.copyOf(out.length)
return out.toString() to arr
}

// Whether src[i] is a quote mark between the letter last kept in [out] and a letter further on
// (nikud and other quote marks skipped)
private fun isQuoteInsideWord(
src: String,
i: Int,
out: CharSequence,
): Boolean {
if (src[i] !in WORD_QUOTES || out.isEmpty() || !out.last().isLetter()) return false
var j = i + 1
while (j < src.length && (src[j] in WORD_QUOTES || isNikudOrTeamim(src[j]))) j++
return j < src.length && src[j].isLetter()
}

// Folds final letters and case one character for one, so indices stay aligned with the map
private fun foldForFind(s: String): String =
buildString(s.length) {
for (c in s) {
val base =
when (c) {
'ך' -> 'כ' // ך -> כ
'ם' -> 'מ' // ם -> מ
'ן' -> 'נ' // ן -> נ
'ף' -> 'פ' // ף -> פ
'ץ' -> 'צ' // ץ -> צ
else -> c
}
append(base.lowercaseChar())
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package io.github.kdroidfilter.seforimapp.core.presentation.text

import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertFalse
import kotlin.test.assertTrue

class HebrewSearchTest {
Expand Down Expand Up @@ -190,4 +191,142 @@ class HebrewSearchTest {
val result = findAllMatchesOriginal("שלום עולם", "בוקר")
assertTrue(result.isEmpty())
}

// Punctuation-insensitive find
@Test
fun `isSearchPunctuation covers ASCII, Hebrew and Unicode punctuation`() {
val ascii = ",.;:!?\"'`()[]{}<>-_/|+=*&^%#@~\\\$"
// maqaf, sof pasuq, paseq, geresh, gershayim, curly quotes, dashes, guillemets, ellipsis
val unicode = "\u05BE\u05C3\u05C0\u05F3\u05F4\u201C\u201D\u2018\u2019\u2014\u2013\u00AB\u00BB\u2026"
val punctuation = ascii + unicode
for (c in punctuation) assertTrue(isSearchPunctuation(c), "expected punctuation: U+%04X".format(c.code))
for (c in "אבa Z09\t\u05B0\u0591") assertFalse(isSearchPunctuation(c), "expected not punctuation: U+%04X".format(c.code))
}

@Test
fun `stripForFindWithMap turns punctuation runs into one space and maps back`() {
val text = "שלום, עולם."
val (plain, map) = stripForFindWithMap(text)
assertEquals("שלום עולם", plain)
assertEquals(plain.length, map.size)
// The space stands for the comma, the first character of the run
assertEquals(4, map[4])
assertEquals(7, map[5])
assertEquals(10, map.last())
}

@Test
fun `stripForFindWithMap drops leading and trailing punctuation`() {
assertEquals("שלום", stripForFindWithMap(" (שלום)! ").first)
assertEquals("", stripForFindWithMap(" ,.; ").first)
}

@Test
fun `find matches across a comma between words`() {
val text = "שלום, עולם"
val result = findAllMatchesOriginal(text, "שלום עולם")
assertEquals(1, result.size)
assertEquals(text, text.substring(result[0]))
assertEquals(1, findAllMatchesOriginal("שלום,עולם", "שלום עולם").size)
}

@Test
fun `find ignores punctuation typed in the query only`() {
assertEquals(1, findAllMatchesOriginal("שלום עולם", "שלום, עולם!").size)
assertEquals(1, findAllMatchesOriginal("שלום עולם", "שלום-עולם").size)
assertEquals(1, findAllMatchesOriginal("שלום עולם", "\"שלום\"").size)
assertEquals(1, findAllMatchesOriginal("שלום עולם", "שלום עולם").size)
}

@Test
fun `find matches acronyms with or without gershayim, both ways`() {
val ascii = "כתב רש\"י כאן"
val hebrew = "כתב רש\u05F4י כאן"
for (text in listOf(ascii, hebrew)) {
val result = findAllMatchesOriginal(text, "רשי")
assertEquals(1, result.size)
assertEquals(text.substring(4, 8), text.substring(result[0]))
}
assertEquals(1, findAllMatchesOriginal("כתב רשי כאן", "רש\"י").size)
assertEquals(1, findAllMatchesOriginal("כתב רשי כאן", "רש\u05F4י").size)
assertEquals(1, findAllMatchesOriginal(ascii, "רש\u05F4י").size)
assertEquals(1, findAllMatchesOriginal(hebrew, "רש\"י").size)
assertEquals(1, findAllMatchesOriginal("כתב רש''י כאן", "רשי").size)
assertEquals(1, findAllMatchesOriginal("ס' שלום", "ס שלום").size)
}

@Test
fun `quote marks opening or closing a quoted word are word breaks, not dropped`() {
val text = "ואמר \"שלום\" לו"
assertEquals("ואמר שלום לו", stripForFindWithMap(text).first)
assertEquals("שלום", text.substring(findAllMatchesOriginal(text, "שלום")[0]))
assertEquals(1, findAllMatchesOriginal(text, "ואמר שלום לו").size)
assertEquals(1, findAllMatchesOriginal("ואמר “שלום” לו", "ואמר שלום").size)
}

@Test
fun `find matches a Hebrew geresh abbreviation both ways`() {
assertEquals(1, findAllMatchesOriginal("ס׳ שלום", "ס שלום").size)
assertEquals(1, findAllMatchesOriginal("ס' שלום", "ס׳ שלום").size)
assertEquals(1, findAllMatchesOriginal("ס שלום", "ס׳ שלום").size)
}

@Test
fun `find matches words joined by a maqaf`() {
val text = "את כל\u05BEהארץ"
assertEquals(1, findAllMatchesOriginal(text, "כל הארץ").size)
// Used to never match: the query maqaf became a space while the text kept its own
assertEquals(1, findAllMatchesOriginal(text, "כל\u05BEהארץ").size)
assertEquals(1, findAllMatchesOriginal("את כל הארץ", "כל\u05BEהארץ").size)
assertEquals("כל\u05BEהארץ", text.substring(findAllMatchesOriginal(text, "כל הארץ")[0]))
}

@Test
fun `find matches across a sof pasuq and leaves it out of a final match`() {
val text = "ויהי אור\u05C3 וירא"
assertEquals(1, findAllMatchesOriginal(text, "אור וירא").size)
assertEquals("אור", text.substring(findAllMatchesOriginal(text, "אור")[0]))
assertEquals("אור", text.substring(findAllMatchesOriginal(text, "אור\u05C3")[0]))
}

@Test
fun `find ignores punctuation inside pointed text`() {
val text = "בְּרֵאשִׁית, בָּרָא"
val result = findAllMatchesOriginal(text, "בראשית ברא")
assertEquals(1, result.size)
assertEquals(0, result[0].first)
assertEquals(text.length - 1, result[0].last)
assertEquals(1, findAllMatchesOriginal("רַשִׁ\"י", "רשי").size)
}

@Test
fun `find offsets point into the original text`() {
val text = "(שלום) ושלום"
val result = findAllMatchesOriginal(text, "שלום")
assertEquals(listOf(1..4, 8..11), result)
}

@Test
fun `highlight range excludes surrounding punctuation`() {
val text = "אמר: \"שלום, עולם.\""
val result = findAllMatchesOriginal(text, "שלום עולם.")
assertEquals(1, result.size)
assertEquals("שלום, עולם", text.substring(result[0]))
assertEquals("עולם", text.substring(findAllMatchesOriginal(text, "עולם")[0]))
}

@Test
fun `highlight range keeps the points of the last matched letter`() {
val text = "\u05E9\u05C1\u05B8\u05DC\u05D5\u05B9\u05DD"
assertEquals(listOf(0..5), findAllMatchesOriginal(text, "שלו"))
}

@Test
fun `normalizeQueryForHebrew drops punctuation like the text`() {
assertEquals("שלומ עולמ", normalizeQueryForHebrew(" שלום, עולם! "))
assertEquals("רשי", normalizeQueryForHebrew("רש\"י"))
assertEquals("רשי", normalizeQueryForHebrew("רש\u05F4י"))
assertEquals("כל הארצ", normalizeQueryForHebrew("כל\u05BEהארץ"))
assertEquals("", normalizeQueryForHebrew("!?"))
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,17 @@ class BookFindMatchesTest {
assertEquals(BookFindMatches.Match(2, 12, ordinal = 1, position = 4), matches.nextAfterLine(3, forward = false))
}

@Test
fun `counts matches whatever the punctuation, in the query or the text`() =
runTest {
// Line 2 reads "ויברא אלהים, ויברא"
assertEquals(1, find("אלהים ויברא").occurrences)
assertEquals(1, find("אלהים; ויברא!").occurrences)
assertEquals(0, find(",.").occurrences)
assertTrue(queryNarrows("אלהים, ויברא", "אלהים"))
assertTrue(queryNarrows("אלהים ויברא", "אלהים,"))
}

@Test
fun `matches against the displayed text, not the html`() =
runTest {
Expand Down
Loading