diff --git a/.gitignore b/.gitignore index aba81386d..c6ffd130c 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,6 @@ Pods/ *.gpg *yarn.lock .claude/ + +# Kotlin SwiftPM import (Sentry Cocoa for the iOS binaries) +.swiftpm-locks/ diff --git a/CLAUDE.md b/CLAUDE.md index 4f0037a22..c01893041 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -117,10 +117,22 @@ through `SiddurFeature`/`installedSiddur`, never by importing `addons/siddur` co The semantic search (own embedding model, dense KNN fused with lexical search by RRF) follows the same pattern: the private `io.github.kdroidfilter:seforim-semantic` package (module `semantic/` of the private `SeforimEmbedding` repo, read with the siddur's token by default, or `semantic.token`/`SEFORIM_SEMANTIC_TOKEN`/`gh`, or `-Psemantic.local=true`), -whose jar carries the int8 model as a resource (never in the DB bundle). `addons/semantic/enabled` holds the -`HybridSearchEngine`; `disabled` makes `installedSemanticSearch` null and the app uses `LuceneSearchEngine` alone. Core -code reaches it only through `SemanticSearchFeature`/`installedSemanticSearch`. With the same token, SeforimLibrary's -generator embeds the corpus into the index's vectors (`:searchindex:embedCorpus`, GPU via CUDA, no Python). +Kotlin Multiplatform since 0.6.0, whose JVM jar carries the int8 model as a resource (never in the DB bundle; mobile +apps download it). `searchkit/addons/semantic/enabled` holds the `HybridSearchEngine`; `disabled` makes +`installedSemanticSearch` null and the app searches lexically alone. Core code reaches it only through +`SemanticSearchFeature`/`installedSemanticSearch`. With the same token, SeforimLibrary's generator embeds the corpus +into the index's vectors (`:searchindex:embedCorpus`, GPU via CUDA, no Python). + +### Search on mobile: `searchkit` +The search without its UI (engines, lookup, use cases, `SearchResultViewModel`/`SearchHomeViewModel`) is the KMP module +`searchkit` (JVM, Android 10+, iOS), shared by the desktop and the mobile apps; see `searchkit/README.md`. Lucene runs +on a real JVM only (ART can't keep up with Lucene's Java versions), through Nucleus JVM Access: SeforimLibrary's +`search-facade` (Kotlin/JVM) is the façade, its common code uses the `LuceneIndex`/`IndexQuery`... NJA generates; the +desktop's own JVM, HotSpot in the app's process on Android (jdk-mobile's OpenJDK 27), a GraalVM native image linked +into the app on iOS (graalvm-mobile); +see `SeforimLibrary/JVM_RUNTIME.md`. The view models reach the +desktop shell only through the ports of `SearchHost.kt` (`SearchTabs`, `SearchSessionStore`, `SearchSettings`, +`SearchHistory`); keep `java.*`, AWT and desktop types out of `searchkit/src/commonMain`. The verse quote links of the Talmud (temporary, until the DB carries them) follow the same pattern: the private `io.github.kdroidfilter:seforim-quotelinks` package (repo `SeforimQuoteLinks`, read with the siddur's token by default, @@ -157,6 +169,7 @@ See `TAB_SYSTEM_README.md` for complete details. Key points: - `pagination`: Pagination support for large datasets - `texteffects`: Text rendering effects and styling - `network`: Network utilities and Ktor client setup +- `searchkit`: the search without its UI, multiplatform (see "Search on mobile") - `backup`: Backup core, Android-BackupAgent/Transport style: `BackupSource` (what: the app's `UserDataBackup`, user DB + portable prefs), `BackupDestination` (where: `LocalFileDestination`, …), `BackupManager` (the only code linking both) - `backup-drive`: Google Drive `BackupDestination` + `GoogleDriveSync` (OAuth PKCE loopback, hidden `appDataFolder`, append-only with the 10 latest kept as IntelliJ settings sync, never restores without asking, on-quit + daily auto backup paused while a restore is pending) diff --git a/SeforimApp/addons/semantic/enabled/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/InstalledSemanticSearch.kt b/SeforimApp/addons/semantic/enabled/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/InstalledSemanticSearch.kt deleted file mode 100644 index a599730fd..000000000 --- a/SeforimApp/addons/semantic/enabled/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/InstalledSemanticSearch.kt +++ /dev/null @@ -1,15 +0,0 @@ -package io.github.kdroidfilter.seforimapp.features.search.semantic - -import io.github.kdroidfilter.seforimlibrary.search.LineHit -import io.github.kdroidfilter.seforimlibrary.search.SearchEngine -import java.nio.file.Path - -/** An official build: the semantic search, its model inside the SeforimEmbedding package. */ -val installedSemanticSearch: SemanticSearchFeature? = - object : SemanticSearchFeature { - override fun hybrid( - lexical: SearchEngine, - indexDir: Path, - resolveLine: suspend (lineId: Long, query: String) -> LineHit?, - ): SearchEngine = HybridSearchEngine(lexical, indexDir, resolveLine) - } diff --git a/SeforimApp/build.gradle.kts b/SeforimApp/build.gradle.kts index 9363c0aaa..584739e00 100644 --- a/SeforimApp/build.gradle.kts +++ b/SeforimApp/build.gradle.kts @@ -43,10 +43,6 @@ sentry { // (settings.gradle.kts); a community build has none and addons/siddur/disabled says so val withSiddur = gradle.extensions.extraProperties["siddurEnabled"] == true -// The semantic search too (open core): its package (code + model) with its token or its sources beside; without -// them, addons/semantic/disabled leaves the search lexical only -val withSemantic = gradle.extensions.extraProperties["semanticEnabled"] == true - // The verse quote links of the Talmud (open core, temporary until the DB carries them): its package, or // addons/quotelinks/disabled and the reader shows the lines as they are val withQuoteLinks = gradle.extensions.extraProperties["quoteLinksEnabled"] == true @@ -145,11 +141,6 @@ kotlin { } if (withSiddur) jvmTest { kotlin.srcDir("addons/siddur/test/kotlin") } - jvmMain { - kotlin.srcDir(if (withSemantic) "addons/semantic/enabled/kotlin" else "addons/semantic/disabled/kotlin") - if (withSemantic) dependencies { implementation(libs.seforim.semantic) } - } - jvmMain { kotlin.srcDir(if (withQuoteLinks) "addons/quotelinks/enabled/kotlin" else "addons/quotelinks/disabled/kotlin") if (withQuoteLinks) dependencies { implementation(libs.seforim.quotelinks) } @@ -199,6 +190,10 @@ kotlin { // SeforimLibrary search module implementation(libs.seforimlibrary.search) + // The search without its UI (engines, lookup, use cases, view models), shared with the mobile apps; the + // semantic search (open core) comes with it in the official builds + implementation(project(":searchkit")) + // SeforimLibrary CLI: lets the desktop binary run headless search commands // (`zayit cli search ...`) by delegating to its runCli() entry point. implementation(libs.seforimlibrary.cli) diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/core/history/HistoryStore.kt b/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/core/history/HistoryStore.kt index 4af070739..98ba82494 100644 --- a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/core/history/HistoryStore.kt +++ b/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/core/history/HistoryStore.kt @@ -4,6 +4,7 @@ import androidx.compose.runtime.Stable import io.github.kdroidfilter.seforim.tabs.SearchScope import io.github.kdroidfilter.seforimapp.db.UserSettingsDb import io.github.kdroidfilter.seforimapp.db.Visit_history +import io.github.kdroidfilter.seforimapp.features.search.SearchHistory import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow @@ -39,7 +40,7 @@ enum class VisitKind { BOOK, SEARCH } */ class HistoryStore( database: UserSettingsDb, -) { +) : SearchHistory { private val queries = database.visitHistoryQueries private val _revision = MutableStateFlow(0L) @@ -79,7 +80,7 @@ class HistoryStore( * Records an executed search with the [scope] it ran in: one history row per (query, scope, extended), titled * with [scopeLabel] (the scope's name) so a scoped search reads apart from the same query elsewhere. */ - suspend fun recordSearchVisit( + override suspend fun recordSearchVisit( query: String, scope: SearchScope, globalExtended: Boolean, diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/core/settings/AppSettings.kt b/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/core/settings/AppSettings.kt index 6239fc1e5..b782d4456 100644 --- a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/core/settings/AppSettings.kt +++ b/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/core/settings/AppSettings.kt @@ -8,6 +8,7 @@ import dev.zacsweers.metro.SingleIn import io.github.kdroidfilter.seforimapp.core.presentation.theme.AccentColor import io.github.kdroidfilter.seforimapp.core.presentation.theme.IntUiThemes import io.github.kdroidfilter.seforimapp.earthwidget.WidgetAntiAliasing +import io.github.kdroidfilter.seforimapp.features.search.SearchSettings import io.github.kdroidfilter.seforimapp.framework.di.AppScope import io.github.kdroidfilter.seforimapp.framework.platform.PlatformInfo import kotlinx.coroutines.channels.BufferOverflow @@ -28,10 +29,10 @@ import java.util.concurrent.ConcurrentHashMap @SingleIn(AppScope::class) class AppSettings( private val settings: Settings, -) { +) : SearchSettings { // StateFlow to observe text size changes private val _textSizeFlow = MutableStateFlow(getTextSize()) - val textSizeFlow: StateFlow = _textSizeFlow.asStateFlow() + override val textSizeFlow: StateFlow = _textSizeFlow.asStateFlow() // StateFlow to observe line height changes private val _lineHeightFlow = MutableStateFlow(getLineHeight()) @@ -119,7 +120,7 @@ class AppSettings( fun findQueryFlow(tabId: String): StateFlow = queryFlowFor(tabId).asStateFlow() - fun setFindQuery( + override fun setFindQuery( tabId: String, q: String, ) { @@ -128,7 +129,7 @@ class AppSettings( fun findBarOpenFlow(tabId: String): StateFlow = findOpenFlowFor(tabId).asStateFlow() - fun openFindBar(tabId: String) { + override fun openFindBar(tabId: String) { findOpenFlowFor(tabId).value = true } @@ -169,7 +170,7 @@ class AppSettings( fun findSmartModeFlow(tabId: String): StateFlow = smartModeFlowFor(tabId).asStateFlow() - fun setFindSmartMode( + override fun setFindSmartMode( tabId: String, enabled: Boolean, ) { @@ -181,7 +182,7 @@ class AppSettings( flow.value = !flow.value } - fun getTextSize(): Float = settings[KEY_TEXT_SIZE, DEFAULT_TEXT_SIZE] + override fun getTextSize(): Float = settings[KEY_TEXT_SIZE, DEFAULT_TEXT_SIZE] fun setTextSize(size: Float) { settings[KEY_TEXT_SIZE] = size @@ -425,13 +426,13 @@ class AppSettings( // User profile accessors // Reactive flows to observe user identity changes across the app private val _userFirstNameFlow = MutableStateFlow(getUserFirstName() ?: "") - val userFirstNameFlow: StateFlow = _userFirstNameFlow.asStateFlow() + override val userFirstNameFlow: StateFlow = _userFirstNameFlow.asStateFlow() private val _userLastNameFlow = MutableStateFlow(getUserLastName() ?: "") - val userLastNameFlow: StateFlow = _userLastNameFlow.asStateFlow() + override val userLastNameFlow: StateFlow = _userLastNameFlow.asStateFlow() private val _userCommunityCodeFlow = MutableStateFlow(getUserCommunityCode()) - val userCommunityCodeFlow: StateFlow = _userCommunityCodeFlow.asStateFlow() + override val userCommunityCodeFlow: StateFlow = _userCommunityCodeFlow.asStateFlow() fun getUserFirstName(): String? { val value: String = settings[KEY_USER_FIRST_NAME, ""] @@ -631,7 +632,7 @@ class AppSettings( } // Text size constants - const val DEFAULT_TEXT_SIZE = 16f + const val DEFAULT_TEXT_SIZE = SearchSettings.DEFAULT_TEXT_SIZE const val MIN_TEXT_SIZE = 14f const val MAX_TEXT_SIZE = 50f const val TEXT_SIZE_INCREMENT = 2f diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/ui/panels/booktoc/BookTocPanel.kt b/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/ui/panels/booktoc/BookTocPanel.kt index 4e9da0b96..8346f787b 100644 --- a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/ui/panels/booktoc/BookTocPanel.kt +++ b/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/ui/panels/booktoc/BookTocPanel.kt @@ -138,11 +138,12 @@ fun SearchBookTocPanel( onHide = { onEvent(BookContentEvent.ToggleToc) }, ) Column(modifier = Modifier.padding(horizontal = 8.dp)) { - if (searchUi.scopeBook != null) { + val scopeBook = searchUi.scopeBook + if (scopeBook != null) { Box(modifier = Modifier.fillMaxHeight()) { val expanded: Set = uiState.toc.expandedEntries - var autoExpanded by remember(searchUi.scopeBook.id, tocCounts, tocTree) { mutableStateOf(false) } - LaunchedEffect(searchUi.scopeBook.id, tocCounts, tocTree) { + var autoExpanded by remember(scopeBook.id, tocCounts, tocTree) { mutableStateOf(false) } + LaunchedEffect(scopeBook.id, tocCounts, tocTree) { if (!autoExpanded) { val idsWithChildren = tocTree?.children?.keys ?: emptySet() val withResults = tocCounts.keys diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/SemanticSearchFeature.kt b/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/SemanticSearchFeature.kt deleted file mode 100644 index 99d7f3630..000000000 --- a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/SemanticSearchFeature.kt +++ /dev/null @@ -1,22 +0,0 @@ -package io.github.kdroidfilter.seforimapp.features.search.semantic - -import io.github.kdroidfilter.seforimlibrary.search.LineHit -import io.github.kdroidfilter.seforimlibrary.search.SearchEngine -import java.nio.file.Path - -/** - * The semantic search (embedding model + dense search), in Zayit's official builds only (open core): its engine is in - * addons/semantic/enabled, built with the SeforimEmbedding package; a community build compiles addons/semantic/disabled - * instead, where [installedSemanticSearch] is null and the search stays lexical. - */ -interface SemanticSearchFeature { - /** - * Fuses [lexical] with dense search over the vectors of the index at [indexDir]; it degrades to [lexical] when the - * model or the vectors are missing. [resolveLine] turns a dense-only hit into a full [LineHit]. - */ - fun hybrid( - lexical: SearchEngine, - indexDir: Path, - resolveLine: suspend (lineId: Long, query: String) -> LineHit?, - ): SearchEngine -} diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/di/modules/AppCoreBindings.kt b/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/di/modules/AppCoreBindings.kt index e8954640d..d66d96175 100644 --- a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/di/modules/AppCoreBindings.kt +++ b/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/di/modules/AppCoreBindings.kt @@ -28,7 +28,11 @@ import io.github.kdroidfilter.seforimapp.core.settings.AppSettings import io.github.kdroidfilter.seforimapp.core.settings.CategoryDisplaySettingsStore import io.github.kdroidfilter.seforimapp.core.shnayimmikra.ShnayimMikraStore import io.github.kdroidfilter.seforimapp.db.UserSettingsDb +import io.github.kdroidfilter.seforimapp.features.search.SearchHistory import io.github.kdroidfilter.seforimapp.features.search.SearchHomeViewModel +import io.github.kdroidfilter.seforimapp.features.search.SearchSessionStore +import io.github.kdroidfilter.seforimapp.features.search.SearchSettings +import io.github.kdroidfilter.seforimapp.features.search.SearchTabs import io.github.kdroidfilter.seforimapp.framework.backup.UserDataBackup import io.github.kdroidfilter.seforimapp.framework.database.CatalogCache import io.github.kdroidfilter.seforimapp.framework.database.DatabasePathProvider @@ -38,8 +42,11 @@ import io.github.kdroidfilter.seforimapp.framework.database.getUserSettingsDatab import io.github.kdroidfilter.seforimapp.framework.desktop.DesktopManager import io.github.kdroidfilter.seforimapp.framework.di.AppScope import io.github.kdroidfilter.seforimapp.framework.search.AcronymFrequencyCache +import io.github.kdroidfilter.seforimapp.framework.search.DesktopSearchSessionStore +import io.github.kdroidfilter.seforimapp.framework.search.DesktopSearchTabs import io.github.kdroidfilter.seforimapp.framework.search.LuceneLookupSearchService import io.github.kdroidfilter.seforimapp.framework.search.RepositorySnippetSourceProvider +import io.github.kdroidfilter.seforimapp.framework.search.openLuceneIndex import io.github.kdroidfilter.seforimapp.framework.search.withSemanticSearch import io.github.kdroidfilter.seforimapp.framework.session.SessionManager import io.github.kdroidfilter.seforimapp.framework.session.TabPersistedStateStore @@ -190,13 +197,28 @@ object AppCoreBindings { val dictionaryPath = indexPath.resolveSibling("lexical.db") val snippetProvider = RepositorySnippetSourceProvider(repository) val lexical = LuceneSearchEngine(indexPath, snippetProvider, dictionaryPath = dictionaryPath) - return withSemanticSearch(lexical, indexPath, repository) + return withSemanticSearch(lexical, lexical.index, repository) } + // What the search screens (searchkit) ask of the desktop around them + @Provides + @SingleIn(AppScope::class) + fun provideSearchTabs(desktopManager: DesktopManager): SearchTabs = DesktopSearchTabs(desktopManager) + + @Provides + @SingleIn(AppScope::class) + fun provideSearchSessionStore(store: TabPersistedStateStore): SearchSessionStore = DesktopSearchSessionStore(store) + + @Provides + fun provideSearchSettings(appSettings: AppSettings): SearchSettings = appSettings + + @Provides + fun provideSearchHistory(historyStore: HistoryStore): SearchHistory = historyStore + @Provides @SingleIn(AppScope::class) fun provideAcronymFrequencyCache(databasePathProvider: DatabasePathProvider): AcronymFrequencyCache = - AcronymFrequencyCache(databasePathProvider) + AcronymFrequencyCache(databasePathProvider::get) @Provides @SingleIn(AppScope::class) @@ -206,7 +228,7 @@ object AppCoreBindings { ): LuceneLookupSearchService { val dbPath = databasePathProvider.get() val indexPath = if (dbPath.endsWith(".db")) "$dbPath.lookup.lucene" else "$dbPath.lookupindex" - return LuceneLookupSearchService(Paths.get(indexPath), acronymCache = acronymCache) + return LuceneLookupSearchService(openLuceneIndex(indexPath, cacheQueries = true), acronymCache = acronymCache) } @Provides @@ -269,7 +291,7 @@ object AppCoreBindings { // created and disposed by DesktopManager. searchHomeViewModelFactory = { observeProfile -> SearchHomeViewModel( - persistedStore = tabPersistedStateStore, + sessionStore = DesktopSearchSessionStore(tabPersistedStateStore), repository = repository, lookup = lookup, appSettings = appSettings, diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/AcronymFrequencyCache.kt b/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/AcronymFrequencyCache.kt deleted file mode 100644 index 91c727012..000000000 --- a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/AcronymFrequencyCache.kt +++ /dev/null @@ -1,155 +0,0 @@ -package io.github.kdroidfilter.seforimapp.framework.search - -import io.github.kdroidfilter.seforimapp.framework.database.DatabasePathProvider -import io.github.kdroidfilter.seforimapp.logger.infoln -import java.sql.Connection -import java.sql.DriverManager - -/** - * Cache for acronym frequency data from seforim.db. - * Provides fast lookup of how many books use each acronym. - * - * Uses the book_acronym table from seforim.db - no separate database needed! - */ -class AcronymFrequencyCache( - private val databasePath: () -> String, -) { - constructor(databasePathProvider: DatabasePathProvider) : this(databasePathProvider::get) - - private val connection: Connection by lazy { - loadSeforimDb() - } - - // Cache: acronym → frequency (number of books using it) - private val frequencyCache = mutableMapOf() - - // Cache: acronym → list of book titles - private val bookTitlesCache = mutableMapOf>() - - // Cache: book title → list of acronyms - private val titleToAcronymsCache = mutableMapOf>() - - init { - // Pre-load common acronyms into cache - preloadCache() - } - - /** - * Get the frequency (number of books) for an acronym. - */ - fun getFrequency(acronym: String): Int = - frequencyCache.getOrPut(acronym) { - queryFrequency(acronym) - } - - /** - * Get all book titles that use this acronym. - */ - fun getBookTitles(acronym: String): List = - bookTitlesCache.getOrPut(acronym) { - queryBookTitles(acronym) - } - - /** - * Check if an acronym is unique (used by only one book). - */ - fun isUnique(acronym: String): Boolean = getFrequency(acronym) == 1 - - /** - * Get all acronyms for a given book title. - */ - fun getAcronymsForBook(bookTitle: String): List = - titleToAcronymsCache.getOrPut(bookTitle) { - queryAcronymsForTitle(bookTitle) - } - - private fun loadSeforimDb(): Connection { - val dbPath = databasePath() - infoln { "Loading acronym data from seforim.db: $dbPath" } - return DriverManager.getConnection("jdbc:sqlite:$dbPath") - } - - private fun queryFrequency(acronym: String): Int { - val sql = """ - SELECT COUNT(DISTINCT bookId) as frequency - FROM book_acronym - WHERE term = ? - """ - - return connection.prepareStatement(sql).use { stmt -> - stmt.setString(1, acronym) - stmt.executeQuery().use { rs -> - if (rs.next()) rs.getInt("frequency") else 0 - } - } - } - - private fun queryBookTitles(acronym: String): List { - val sql = """ - SELECT DISTINCT b.title - FROM book_acronym ba - JOIN book b ON ba.bookId = b.id - WHERE ba.term = ? - """ - - return connection.prepareStatement(sql).use { stmt -> - stmt.setString(1, acronym) - stmt.executeQuery().use { rs -> - buildList { - while (rs.next()) { - add(rs.getString("title")) - } - } - } - } - } - - private fun queryAcronymsForTitle(bookTitle: String): List { - val sql = """ - SELECT DISTINCT ba.term - FROM book b - JOIN book_acronym ba ON b.id = ba.bookId - WHERE b.title = ? - """ - - return connection.prepareStatement(sql).use { stmt -> - stmt.setString(1, bookTitle) - stmt.executeQuery().use { rs -> - buildList { - while (rs.next()) { - add(rs.getString("term")) - } - } - } - } - } - - private fun preloadCache() { - // Pre-load top 100 most common acronyms - val sql = """ - SELECT - term, - COUNT(DISTINCT bookId) as frequency - FROM book_acronym - GROUP BY term - ORDER BY frequency DESC - LIMIT 100 - """ - - connection.prepareStatement(sql).use { stmt -> - stmt.executeQuery().use { rs -> - while (rs.next()) { - val acronym = rs.getString("term") - val frequency = rs.getInt("frequency") - frequencyCache[acronym] = frequency - } - } - } - - infoln { "Pre-loaded ${frequencyCache.size} acronyms into cache from seforim.db" } - } - - fun close() { - connection.close() - } -} diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/DesktopSearchHost.kt b/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/DesktopSearchHost.kt new file mode 100644 index 000000000..a843422ec --- /dev/null +++ b/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/DesktopSearchHost.kt @@ -0,0 +1,76 @@ +package io.github.kdroidfilter.seforimapp.framework.search + +import io.github.kdroidfilter.seforim.tabs.TabsDestination +import io.github.kdroidfilter.seforimapp.features.search.SearchSessionStore +import io.github.kdroidfilter.seforimapp.features.search.SearchTabs +import io.github.kdroidfilter.seforimapp.framework.desktop.DesktopManager +import io.github.kdroidfilter.seforimapp.framework.session.SearchPersistedState +import io.github.kdroidfilter.seforimapp.framework.session.TabPersistedStateStore +import kotlinx.coroutines.flow.Flow + +/** The search's tabs on the desktop: those of the window that holds the searching tab, across virtual desktops. */ +class DesktopSearchTabs( + private val desktopManager: DesktopManager, +) : SearchTabs { + override fun openTab( + fromTabId: String, + destination: TabsDestination, + ) { + desktopManager.tabsViewModelFor(fromTabId)?.openTab(destination) + } + + override fun replace( + tabId: String, + destination: TabsDestination, + ) { + desktopManager.tabsViewModelFor(tabId)?.replaceCurrentTabDestination(destination) + } + + override fun tabExistsFlow(tabId: String): Flow = desktopManager.tabExistsFlow(tabId) + + override fun isTabOpen(tabId: String): Boolean = desktopManager.isTabOpen(tabId) +} + +/** The search's state in the desktop session: the per-tab state the session saves at close. */ +class DesktopSearchSessionStore( + private val store: TabPersistedStateStore, +) : SearchSessionStore { + override fun search(tabId: String): SearchPersistedState? = store.get(tabId)?.search + + override fun updateSearch( + tabId: String, + transform: (SearchPersistedState?) -> SearchPersistedState?, + ) { + store.update(tabId) { current -> current.copy(search = transform(current.search)) } + } + + override fun selectBook( + tabId: String, + bookId: Long, + ) { + store.update(tabId) { current -> current.copy(bookContent = current.bookContent.copy(selectedBookId = bookId)) } + } + + override fun openBookAtLine( + tabId: String, + bookId: Long, + lineId: Long, + ) { + store.update(tabId) { current -> + current.copy( + bookContent = + current.bookContent.copy( + selectedBookId = bookId, + selectedLineIds = setOf(lineId), + primarySelectedLineId = lineId, + isTocEntrySelection = false, + contentAnchorLineId = lineId, + contentAnchorIndex = 0, + contentScrollIndex = 0, + contentScrollOffset = 0, + isTocVisible = true, + ), + ) + } + } +} diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/session/SessionModels.kt b/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/session/SessionModels.kt index 55090dac0..ab0526e1d 100644 --- a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/session/SessionModels.kt +++ b/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/session/SessionModels.kt @@ -3,11 +3,9 @@ package io.github.kdroidfilter.seforimapp.framework.session import io.github.kdroidfilter.seforim.desktop.VirtualDesktop -import io.github.kdroidfilter.seforim.tabs.SearchScope import io.github.kdroidfilter.seforim.tabs.TabType import io.github.kdroidfilter.seforim.tabs.TabsDestination import io.github.kdroidfilter.seforimapp.features.bookcontent.state.SplitDefaults -import io.github.kdroidfilter.seforimapp.features.search.SearchTabCache import kotlinx.serialization.Serializable /** @@ -183,75 +181,3 @@ data class DesktopsState( .filter { id -> desktops.any { it.id == id } } .distinct() } - -@Serializable -data class SearchPersistedState( - val query: String = "", - val globalExtended: Boolean = false, - val datasetScope: String = "global", - // Unused (a search is no longer scoped to a category), kept for the ProtoBuf field numbers - val filterCategoryId: Long = 0L, - val filterBookId: Long = 0L, - val filterTocId: Long = 0L, - // Unused, as filterCategoryId - val fetchCategoryId: Long = 0L, - val fetchBookId: Long = 0L, - val fetchTocId: Long = 0L, - // View filters (multi-select) - val selectedCategoryIds: Set = emptySet(), - val selectedBookIds: Set = emptySet(), - val selectedTocIds: Set = emptySet(), - // Scroll/anchor persistence - val scrollIndex: Int = 0, - val scrollOffset: Int = 0, - val anchorId: Long = -1L, - val anchorIndex: Int = 0, - // Full search snapshot (results + aggregates) for identical restore - val snapshot: SearchTabCache.Snapshot? = null, - val breadcrumbs: Map> = emptyMap(), - // Appended last: the session is ProtoBuf, numbered by declaration order. - // What the bar holds, typed but not searched yet (restored in the bar; [query] is what is searched) - val draftQuery: String = "", - // The author whose books are searched (none with a book or TOC scope) - val fetchAuthorId: Long = 0L, -) { - /** The scope the search runs in: its fetch scope, else its view filter. */ - val scope: SearchScope - get() { - val bookId = fetchBookId.takeIf { it > 0 } ?: filterBookId.takeIf { it > 0 } - val tocId = fetchTocId.takeIf { it > 0 } ?: filterTocId.takeIf { it > 0 } - return when { - tocId != null && bookId != null -> SearchScope.Toc(bookId = bookId, tocId = tocId) - bookId != null -> SearchScope.Book(bookId) - fetchAuthorId > 0 -> SearchScope.Author(fetchAuthorId) - else -> SearchScope.Global - } - } - - /** Runs the search in [scope]: both its fetch scope and its view filter (a legacy category: everywhere). */ - fun withScope(scope: SearchScope): SearchPersistedState { - val bookId = - when (scope) { - is SearchScope.Book -> scope.bookId - is SearchScope.Toc -> scope.bookId - else -> 0L - } - val tocId = (scope as? SearchScope.Toc)?.tocId ?: 0L - return copy( - datasetScope = - when (scope) { - SearchScope.Global, is SearchScope.Category -> "global" - is SearchScope.Book -> "book" - is SearchScope.Toc -> "toc" - is SearchScope.Author -> "author" - }, - filterCategoryId = 0L, - filterBookId = bookId, - filterTocId = tocId, - fetchCategoryId = 0L, - fetchBookId = bookId, - fetchTocId = tocId, - fetchAuthorId = (scope as? SearchScope.Author)?.authorId ?: 0L, - ) - } -} diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/main.kt b/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/main.kt index ff8423fd3..24fd80d73 100644 --- a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/main.kt +++ b/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/main.kt @@ -59,6 +59,7 @@ import io.github.kdroidfilter.seforimapp.framework.di.LocalAppGraph import io.github.kdroidfilter.seforimapp.framework.platform.PlatformInfo import io.github.kdroidfilter.seforimapp.framework.search.AcronymFrequencyCache import io.github.kdroidfilter.seforimapp.framework.search.LuceneLookupSearchService +import io.github.kdroidfilter.seforimapp.framework.search.openLuceneIndex import io.github.kdroidfilter.seforimapp.framework.search.withSemanticSearch import io.github.kdroidfilter.seforimapp.logger.infoln import io.github.kdroidfilter.seforimapp.logger.isDevEnv @@ -154,7 +155,11 @@ private fun appCliHost() = lookup = { dbPath, query, limit -> val indexPath = Paths.get("$dbPath.lookup.lucene") require(Files.exists(indexPath)) { "Lookup index not found: $indexPath" } - val service = LuceneLookupSearchService(indexPath, acronymCache = AcronymFrequencyCache { dbPath.toString() }) + val service = + LuceneLookupSearchService( + openLuceneIndex(indexPath.toString(), cacheQueries = true), + acronymCache = AcronymFrequencyCache { dbPath.toString() }, + ) LookupOutput( query = query, books = diff --git a/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/PostSelectLineTest.kt b/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/PostSelectLineTest.kt index 75df36fd8..a7a03c2ff 100644 --- a/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/PostSelectLineTest.kt +++ b/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/PostSelectLineTest.kt @@ -16,6 +16,7 @@ import io.github.kdroidfilter.seforimapp.features.bookcontent.usecases.NotesUseC import io.github.kdroidfilter.seforimapp.features.bookcontent.usecases.TocUseCase import io.github.kdroidfilter.seforimapp.features.search.SearchHomeViewModel import io.github.kdroidfilter.seforimapp.framework.desktop.DesktopManager +import io.github.kdroidfilter.seforimapp.framework.search.DesktopSearchSessionStore import io.github.kdroidfilter.seforimapp.framework.session.TabPersistedStateStore import io.github.kdroidfilter.seforimapp.framework.session.TabThumbnailStore import io.github.kdroidfilter.seforimapp.testAppSettings @@ -87,7 +88,12 @@ class PostSelectLineTest { ), titleUpdateManager = titleUpdateManager, searchHomeViewModelFactory = { - SearchHomeViewModel(TabPersistedStateStore(), mockk(relaxed = true), mockk(relaxed = true), testAppSettings()) + SearchHomeViewModel( + DesktopSearchSessionStore(TabPersistedStateStore()), + mockk(relaxed = true), + mockk(relaxed = true), + testAppSettings(), + ) }, defaultDesktopName = "D1", ) diff --git a/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchBench.kt b/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchBench.kt index 8ee87c446..bebac94ab 100644 --- a/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchBench.kt +++ b/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchBench.kt @@ -1,10 +1,9 @@ package io.github.kdroidfilter.seforimapp.features.search -import io.github.kdroidfilter.seforimapp.features.search.semantic.installedSemanticSearch import io.github.kdroidfilter.seforimapp.framework.database.PersistentSqliteDriver import io.github.kdroidfilter.seforimapp.framework.search.RepositorySnippetSourceProvider +import io.github.kdroidfilter.seforimapp.framework.search.withSemanticSearch import io.github.kdroidfilter.seforimlibrary.dao.repository.SeforimRepository -import io.github.kdroidfilter.seforimlibrary.search.LineHit import io.github.kdroidfilter.seforimlibrary.search.LuceneSearchEngine import io.github.kdroidfilter.seforimlibrary.search.SearchEngine import kotlinx.coroutines.runBlocking @@ -117,18 +116,5 @@ private fun buildEngine( val lexical = LuceneSearchEngine(indexPath, RepositorySnippetSourceProvider(repository), dictionaryPath = indexPath.resolveSibling("lexical.db")) if (System.getenv("BENCH_LEXICAL_ONLY") != null) return lexical - val semantic = installedSemanticSearch ?: return lexical - return semantic.hybrid(lexical, indexPath) { lineId, query -> - repository.getLine(lineId)?.let { line -> - LineHit( - bookId = line.bookId, - bookTitle = repository.getBook(line.bookId)?.title.orEmpty(), - lineId = lineId, - lineIndex = line.lineIndex, - snippet = lexical.buildSnippet(line.content, query, 5), - score = 0f, - rawText = line.content, - ) - } - } + return withSemanticSearch(lexical, lexical.index, repository) } diff --git a/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/AuthorScopeSearchTest.kt b/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/AuthorScopeSearchTest.kt index 1f15ce65a..fb9e2677d 100644 --- a/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/AuthorScopeSearchTest.kt +++ b/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/AuthorScopeSearchTest.kt @@ -9,6 +9,8 @@ import io.github.kdroidfilter.seforimapp.db.UserSettingsDb import io.github.kdroidfilter.seforimapp.features.bookcontent.state.StateKeys import io.github.kdroidfilter.seforimapp.features.search.SearchResultViewModel import io.github.kdroidfilter.seforimapp.framework.desktop.DesktopManager +import io.github.kdroidfilter.seforimapp.framework.search.DesktopSearchSessionStore +import io.github.kdroidfilter.seforimapp.framework.search.DesktopSearchTabs import io.github.kdroidfilter.seforimapp.framework.session.SearchPersistedState import io.github.kdroidfilter.seforimapp.framework.session.TabPersistedStateStore import io.github.kdroidfilter.seforimapp.testAppSettings @@ -51,11 +53,11 @@ class AuthorScopeSearchTest : SearchIntegrationTestBase() { every { desktopManager.tabExistsFlow(any()) } returns flowOf(true) return SearchResultViewModel( savedStateHandle = SavedStateHandle(mapOf(StateKeys.TAB_ID to tabId)), - persistedStore = store, + sessionStore = DesktopSearchSessionStore(store), repository = requireRepository(), lucene = requireSearchEngine(), titleUpdateManager = TabTitleUpdateManager(), - desktopManager = desktopManager, + tabs = DesktopSearchTabs(desktopManager), historyStore = HistoryStore(UserSettingsDb(JdbcSqliteDriver(JdbcSqliteDriver.IN_MEMORY).also(UserSettingsDb.Schema::create))), appSettings = testAppSettings(), lookup = mockk(relaxed = true), diff --git a/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/ResultsIndexingUseCaseTest.kt b/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/ResultsIndexingUseCaseTest.kt index b553ceda8..a208f7ed5 100644 --- a/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/ResultsIndexingUseCaseTest.kt +++ b/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/ResultsIndexingUseCaseTest.kt @@ -5,6 +5,7 @@ import kotlinx.coroutines.runBlocking import kotlin.test.Test import kotlin.test.assertEquals import kotlin.test.assertNotNull +import kotlin.test.assertSame import kotlin.test.assertTrue /** @@ -46,7 +47,7 @@ class ResultsIndexingUseCaseTest { val index = useCase.ensureIndex(results) assertNotNull(index) - assertEquals(System.identityHashCode(results), index.identity) + assertSame(results, index.identity) // Check book to indices mapping assertEquals(3, index.bookToIndices.size) diff --git a/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/features/settings/SettingsWindowViewModelTest.kt b/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/features/settings/SettingsWindowViewModelTest.kt index 5f3cf4b2d..63d9301ac 100644 --- a/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/features/settings/SettingsWindowViewModelTest.kt +++ b/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/features/settings/SettingsWindowViewModelTest.kt @@ -3,6 +3,7 @@ package io.github.kdroidfilter.seforimapp.features.settings import io.github.kdroidfilter.seforim.tabs.TabTitleUpdateManager import io.github.kdroidfilter.seforimapp.features.search.SearchHomeViewModel import io.github.kdroidfilter.seforimapp.framework.desktop.DesktopManager +import io.github.kdroidfilter.seforimapp.framework.search.DesktopSearchSessionStore import io.github.kdroidfilter.seforimapp.framework.session.TabPersistedStateStore import io.github.kdroidfilter.seforimapp.framework.session.TabThumbnailStore import io.github.kdroidfilter.seforimapp.testAppSettings @@ -27,7 +28,12 @@ class SettingsWindowViewModelTest { ), titleUpdateManager = TabTitleUpdateManager(), searchHomeViewModelFactory = { - SearchHomeViewModel(TabPersistedStateStore(), mockk(relaxed = true), mockk(relaxed = true), testAppSettings()) + SearchHomeViewModel( + DesktopSearchSessionStore(TabPersistedStateStore()), + mockk(relaxed = true), + mockk(relaxed = true), + testAppSettings(), + ) }, defaultDesktopName = "D1", ) diff --git a/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/framework/desktop/DesktopManagerTest.kt b/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/framework/desktop/DesktopManagerTest.kt index 0714fb9f8..8d4fa52ff 100644 --- a/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/framework/desktop/DesktopManagerTest.kt +++ b/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/framework/desktop/DesktopManagerTest.kt @@ -5,6 +5,7 @@ import io.github.kdroidfilter.seforim.tabs.TabTitleUpdateManager import io.github.kdroidfilter.seforim.tabs.TabType import io.github.kdroidfilter.seforim.tabs.TabsDestination import io.github.kdroidfilter.seforimapp.features.search.SearchHomeViewModel +import io.github.kdroidfilter.seforimapp.framework.search.DesktopSearchSessionStore import io.github.kdroidfilter.seforimapp.framework.session.DesktopTabsSnapshot import io.github.kdroidfilter.seforimapp.framework.session.DesktopsState import io.github.kdroidfilter.seforimapp.framework.session.SerializableTabTitle @@ -57,7 +58,12 @@ class DesktopManagerTest { ), titleUpdateManager = TabTitleUpdateManager(), searchHomeViewModelFactory = { - SearchHomeViewModel(TabPersistedStateStore(), mockk(relaxed = true), mockk(relaxed = true), testAppSettings()) + SearchHomeViewModel( + DesktopSearchSessionStore(TabPersistedStateStore()), + mockk(relaxed = true), + mockk(relaxed = true), + testAppSettings(), + ) }, defaultDesktopName = "D1", bootState = bootState, diff --git a/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/framework/search/BookSuggestionsE2ETest.kt b/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/framework/search/BookSuggestionsE2ETest.kt index 2f050b8bd..87cdb54eb 100644 --- a/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/framework/search/BookSuggestionsE2ETest.kt +++ b/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/framework/search/BookSuggestionsE2ETest.kt @@ -5,7 +5,6 @@ import io.github.kdroidfilter.seforimapp.testAppSettings import io.github.vinceglb.filekit.FileKit import org.junit.Assume.assumeTrue import java.io.File -import java.nio.file.Paths import kotlin.test.Test import kotlin.test.assertEquals import kotlin.test.assertTrue @@ -19,7 +18,10 @@ class BookSuggestionsE2ETest { private val lookup by lazy { FileKit.init("io.github.kdroidfilter.seforimapp") val settings = testAppSettings().apply { setDatabasePath(dbPath) } - LuceneLookupSearchService(Paths.get("$dbPath.lookup.lucene"), acronymCache = AcronymFrequencyCache(DatabasePathProvider(settings))) + LuceneLookupSearchService( + openLuceneIndex("$dbPath.lookup.lucene", cacheQueries = true), + acronymCache = AcronymFrequencyCache(DatabasePathProvider(settings)::get), + ) } private fun titles(query: String): List { diff --git a/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/framework/search/TypedAsWordsOfTest.kt b/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/framework/search/TypedAsWordsOfTest.kt index a48eb3f2d..11bd3e287 100644 --- a/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/framework/search/TypedAsWordsOfTest.kt +++ b/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/framework/search/TypedAsWordsOfTest.kt @@ -1,13 +1,12 @@ package io.github.kdroidfilter.seforimapp.framework.search -import java.nio.file.Paths import kotlin.test.Test import kotlin.test.assertFalse import kotlin.test.assertTrue class TypedAsWordsOfTest { // The index is opened lazily: no lookup here - private val lookup = LuceneLookupSearchService(Paths.get("unused")) + private val lookup = LuceneLookupSearchService(openLuceneIndex("unused")) private fun matches( typed: String, diff --git a/SeforimLibrary b/SeforimLibrary index 7ddf045b9..3f563d94e 160000 --- a/SeforimLibrary +++ b/SeforimLibrary @@ -1 +1 @@ -Subproject commit 7ddf045b9a2ea9fe4ad5c6a24e0513c2022da67d +Subproject commit 3f563d94ef84793bef2a5547ce5bbc06dd9426e2 diff --git a/build.gradle.kts b/build.gradle.kts index 1b4dfa8fa..4def02ae8 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -3,6 +3,7 @@ plugins { alias(libs.plugins.compose.compiler).apply(false) alias(libs.plugins.compose).apply(false) alias(libs.plugins.android.application).apply(false) + alias(libs.plugins.android.kmp.library).apply(false) alias(libs.plugins.hotReload).apply(false) alias(libs.plugins.kotlinx.serialization).apply(false) alias(libs.plugins.buildConfig).apply(false) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index fda8c2192..a6cf0f9f2 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -7,6 +7,7 @@ confettikit = "0.9.0" filekitCore = "0.16.0" hebrewNumerals = "0.2.6" jsoup = "1.23.2" +ksoup = "0.2.6" jvmToolchain = "25" nucleus = "3.0.0-dev-202610132318" koalaplotCore = "0.12.1" @@ -38,7 +39,12 @@ sqlDelight = "2.3.2" adaptiveNavigation = "1.2.0" kosherkotlin = "3.0.5" seforimSiddur = "0.2.2" -seforimSemantic = "0.5.0" +seforimSemantic = "0.6.0" +kermit = "2.1.0" +androidx-test-runner = "1.7.0" +androidx-test-junit = "1.3.0" +atomicfu = "0.33.0" +kotlinx-io = "0.9.1" seforimQuoteLinks = "0.1.0" zstdJni = "1.5.7-10" metro = "1.4.5" @@ -51,6 +57,7 @@ mockk = "1.14.11" kotlinx-coroutines-test = "1.11.0" sentry = "6.23.0" sentrySdk = "8.58.0" +sentryKmp = "0.28.0" graalHotspot = "22.0.0.2" intellijPlatformIcons = "262.10968.63" jbrApi = "1.10.1" @@ -70,6 +77,7 @@ filekit-dialogs-compose = { module = "io.github.vinceglb:filekit-dialogs-compose hebrew-numerals = { module = "io.github.kdroidfilter:hebrewnumerals", version.ref = "hebrewNumerals" } jdbc-driver = { module = "app.cash.sqldelight:jdbc-driver", version.ref = "sqlDelight" } jsoup = { module = "org.jsoup:jsoup", version.ref = "jsoup" } +ksoup = { module = "com.fleeksoft.ksoup:ksoup", version.ref = "ksoup" } nucleus-core-runtime = { module = "dev.nucleusframework:nucleus.core-runtime", version.ref = "nucleus" } nucleus-aot-runtime = { module = "dev.nucleusframework:nucleus.aot-runtime", version.ref = "nucleus" } nucleus-darkmode-detector = { module = "dev.nucleusframework:nucleus.darkmode-detector", version.ref = "nucleus" } @@ -140,9 +148,19 @@ adaptive-layout = { module = "org.jetbrains.compose.material3.adaptive:adaptive- adaptive-navigation = { module = "org.jetbrains.compose.material3.adaptive:adaptive-navigation", version.ref = "adaptiveNavigation" } split-pane-desktop = { module = "org.jetbrains.compose.components:components-splitpane-desktop", version.ref = "compose"} sqlite-driver = { module = "app.cash.sqldelight:sqlite-driver", version.ref = "sqlDelight" } +sqldelight-android-driver = { module = "app.cash.sqldelight:android-driver", version.ref = "sqlDelight" } +sqldelight-native-driver = { module = "app.cash.sqldelight:native-driver", version.ref = "sqlDelight" } kosherkotlin = { module = "io.github.kdroidfilter:kosherkotlin", version.ref = "kosherkotlin" } seforim-siddur = { module = "io.github.kdroidfilter:seforim-siddur", version.ref = "seforimSiddur" } seforim-semantic = { module = "io.github.kdroidfilter:seforim-semantic", version.ref = "seforimSemantic" } +kermit = { module = "co.touchlab:kermit", version.ref = "kermit" } +androidx-test-runner = { module = "androidx.test:runner", version.ref = "androidx-test-runner" } +androidx-test-junit = { module = "androidx.test.ext:junit", version.ref = "androidx-test-junit" } +atomicfu = { module = "org.jetbrains.kotlinx:atomicfu", version.ref = "atomicfu" } +kotlinx-io-core = { module = "org.jetbrains.kotlinx:kotlinx-io-core", version.ref = "kotlinx-io" } +androidx-lifecycle-viewmodel-core = { module = "org.jetbrains.androidx.lifecycle:lifecycle-viewmodel", version.ref = "androidx-lifecycle" } +androidx-lifecycle-viewmodel-savedstate = { module = "org.jetbrains.androidx.lifecycle:lifecycle-viewmodel-savedstate", version.ref = "androidx-lifecycle" } +metrox-viewmodel = { module = "dev.zacsweers.metro:metrox-viewmodel", version.ref = "metro" } seforim-quotelinks = { module = "io.github.kdroidfilter:seforim-quotelinks", version.ref = "seforimQuoteLinks" } zstd-jni = { module = "com.github.luben:zstd-jni", version.ref = "zstdJni" } lucene-core = { module = "org.apache.lucene:lucene-core", version.ref = "lucene" } @@ -160,6 +178,7 @@ compose-ui-test = { module = "org.jetbrains.compose.ui:ui-test-junit4", version. # SeforimLibrary composite build seforimlibrary-core = { module = "io.github.kdroidfilter.seforimlibrary:core" } +seforimlibrary-cache = { module = "io.github.kdroidfilter.seforimlibrary:cache" } seforimlibrary-dao = { module = "io.github.kdroidfilter.seforimlibrary:dao" } seforimlibrary-search = { module = "io.github.kdroidfilter.seforimlibrary:search" } seforimlibrary-cli = { module = "io.github.kdroidfilter.seforimlibrary:cli" } @@ -170,6 +189,7 @@ structured-coroutines-annotations = { module = "io.github.santimattius:structure # Sentry sentry-core = { module = "io.sentry:sentry", version.ref = "sentrySdk" } +sentry-kmp = { module = "io.sentry:sentry-kotlin-multiplatform", version.ref = "sentryKmp" } graal-hotspot-library = { module = "org.graalvm.nativeimage:graal-hotspot-library", version.ref = "graalHotspot" } # Testing @@ -186,6 +206,7 @@ multiplatform = { id = "org.jetbrains.kotlin.multiplatform", version.ref = "kotl compose-compiler = { id = "org.jetbrains.kotlin.plugin.compose", version.ref = "kotlin" } compose = { id = "org.jetbrains.compose", version.ref = "compose" } android-application = { id = "com.android.application", version.ref = "agp" } +android-kmp-library = { id = "com.android.kotlin.multiplatform.library", version.ref = "agp" } hotReload = { id = "org.jetbrains.compose.hot-reload", version.ref = "hotReload" } kotlinx-serialization = { id = "org.jetbrains.kotlin.plugin.serialization", version.ref = "kotlin" } buildConfig = { id = "com.github.gmazzo.buildconfig", version.ref = "buildConfig" } @@ -202,3 +223,4 @@ structured-coroutines = { id = "io.github.santimattius.structured-coroutines", v kover = { id = "org.jetbrains.kotlinx.kover", version.ref = "kover" } ksp = { id = "com.google.devtools.ksp", version.ref = "ksp" } sentryJvmGradle = { id = "io.sentry.jvm.gradle", version.ref = "sentry" } +sentryKmpGradle = { id = "io.sentry.kotlin.multiplatform.gradle", version.ref = "sentryKmp" } diff --git a/htmlparser/build.gradle.kts b/htmlparser/build.gradle.kts index 7307ea3ca..01e3d8b9c 100644 --- a/htmlparser/build.gradle.kts +++ b/htmlparser/build.gradle.kts @@ -4,6 +4,7 @@ plugins { alias(libs.plugins.multiplatform) alias(libs.plugins.compose.compiler) alias(libs.plugins.compose) + alias(libs.plugins.android.kmp.library) } kotlin { @@ -14,20 +15,39 @@ kotlin { ) jvm() + android { + namespace = "io.github.kdroidfilter.seforim.htmlparser" + compileSdk = 37 + minSdk = 21 + } + iosArm64() + iosSimulatorArm64() sourceSets { commonMain.dependencies { implementation(libs.compose.runtime) implementation(libs.compose.foundation) - implementation(libs.jsoup) } commonTest.dependencies { implementation(kotlin("test")) } - jvmMain.dependencies { + // The DOM: jsoup on the JVM and Android, its Kotlin port Ksoup on iOS (see HtmlDom.kt) + val androidJvmMain by creating { + dependsOn(commonMain.get()) + dependencies { implementation(libs.jsoup) } + } + jvmMain { dependsOn(androidJvmMain) } + androidMain { dependsOn(androidJvmMain) } + + // The manual source set above turns off the default hierarchy: iOS shares its own one by hand + val iosMain by creating { + dependsOn(commonMain.get()) + dependencies { implementation(libs.ksoup) } } + iosArm64Main { dependsOn(iosMain) } + iosSimulatorArm64Main { dependsOn(iosMain) } } } diff --git a/htmlparser/src/androidJvmMain/kotlin/io/github/kdroidfilter/seforim/htmlparser/HtmlDom.androidJvm.kt b/htmlparser/src/androidJvmMain/kotlin/io/github/kdroidfilter/seforim/htmlparser/HtmlDom.androidJvm.kt new file mode 100644 index 000000000..887a3806f --- /dev/null +++ b/htmlparser/src/androidJvmMain/kotlin/io/github/kdroidfilter/seforim/htmlparser/HtmlDom.androidJvm.kt @@ -0,0 +1,26 @@ +package io.github.kdroidfilter.seforim.htmlparser + +import org.jsoup.Jsoup +import org.jsoup.nodes.Element +import org.jsoup.nodes.Node +import org.jsoup.nodes.TextNode + +internal actual fun parseHtmlBody(html: String): List = + Jsoup + .parse(html) + .body() + .childNodes() + .mapNotNull(::toHtmlNode) + +// Text and elements: comments and the like carry nothing to render +private fun toHtmlNode(node: Node): HtmlNode? = + when (node) { + is TextNode -> HtmlText(node.text()) + is Element -> + HtmlElement( + tag = node.normalName(), + attributes = node.attributes().associate { it.key to it.value }, + children = node.childNodes().mapNotNull(::toHtmlNode), + ) + else -> null + } diff --git a/htmlparser/src/commonMain/kotlin/io/github/kdroidfilter/seforim/htmlparser/HtmlDom.kt b/htmlparser/src/commonMain/kotlin/io/github/kdroidfilter/seforim/htmlparser/HtmlDom.kt new file mode 100644 index 000000000..1b89effe8 --- /dev/null +++ b/htmlparser/src/commonMain/kotlin/io/github/kdroidfilter/seforim/htmlparser/HtmlDom.kt @@ -0,0 +1,28 @@ +package io.github.kdroidfilter.seforim.htmlparser + +/** A node of a parsed HTML body: what [HtmlParser] reads of jsoup's DOM (JVM, Android) or Ksoup's (iOS). */ +internal sealed interface HtmlNode + +internal class HtmlText( + val text: String, +) : HtmlNode + +internal class HtmlElement( + /** The tag's name, lower-case. */ + val tag: String, + private val attributes: Map, + val children: List, +) : HtmlNode { + /** The attribute [name]'s value, empty when absent. */ + fun attr(name: String): String = attributes[name].orEmpty() + + /** Whether the `class` attribute lists [name] (case-insensitive, as jsoup). */ + fun hasClass(name: String): Boolean = attr("class").split(CLASS_SEPARATOR).any { it.equals(name, ignoreCase = true) } + + private companion object { + val CLASS_SEPARATOR = Regex("\\s+") + } +} + +/** The nodes of [html]'s body, parsed as a browser would. */ +internal expect fun parseHtmlBody(html: String): List diff --git a/htmlparser/src/commonMain/kotlin/io/github/kdroidfilter/seforim/htmlparser/HtmlParser.kt b/htmlparser/src/commonMain/kotlin/io/github/kdroidfilter/seforim/htmlparser/HtmlParser.kt index 76cea1477..555037fa9 100644 --- a/htmlparser/src/commonMain/kotlin/io/github/kdroidfilter/seforim/htmlparser/HtmlParser.kt +++ b/htmlparser/src/commonMain/kotlin/io/github/kdroidfilter/seforim/htmlparser/HtmlParser.kt @@ -1,9 +1,5 @@ package io.github.kdroidfilter.seforim.htmlparser -import org.jsoup.Jsoup -import org.jsoup.nodes.Element -import org.jsoup.nodes.Node -import org.jsoup.nodes.TextNode import kotlin.io.encoding.Base64 import kotlin.io.encoding.ExperimentalEncodingApi @@ -36,9 +32,8 @@ private val WHITESPACE_REGEX = Regex("\\s+") class HtmlParser { fun parse(html: String): List { - val doc = Jsoup.parse(html) val out = mutableListOf() - for (child in doc.body().childNodes()) { + for (child in parseHtmlBody(html)) { processNode( node = child, list = out, @@ -62,7 +57,7 @@ class HtmlParser { } private fun processNode( - node: Node, + node: HtmlNode, list: MutableList, isBold: Boolean, isItalic: Boolean, @@ -76,10 +71,10 @@ class HtmlParser { href: String?, ) { when (node) { - is TextNode -> { + is HtmlText -> { appendSegment( list = list, - textRaw = node.text(), + textRaw = node.text, isBold = isBold, isItalic = isItalic, isSmall = isSmall, @@ -92,8 +87,8 @@ class HtmlParser { href = href, ) } - is Element -> { - val tag = node.normalName() + is HtmlElement -> { + val tag = node.tag if (tag == "br") { appendLineBreak(list) @@ -121,10 +116,11 @@ class HtmlParser { val nextFootnoteContent = isFootnoteContent || isFootnoteContentTag val nextHref = if (tag == "a" && node.hasClass("ref")) node.attr("href").ifBlank { null } else href - if (node.childNodeSize() == 1 && node.childNode(0) is TextNode) { + val onlyText = node.children.singleOrNull() as? HtmlText + if (onlyText != null) { appendSegment( list = list, - textRaw = (node.childNode(0) as TextNode).text(), + textRaw = onlyText.text, isBold = nextBold, isItalic = nextItalic, isSmall = nextSmall, @@ -139,7 +135,7 @@ class HtmlParser { return } - for (child in node.childNodes()) { + for (child in node.children) { processNode( node = child, list = list, @@ -273,7 +269,7 @@ class HtmlParser { // Non-data URIs are ignored (no network fetch — images must be pre-embedded in the DB). private fun appendImage( list: MutableList, - node: Element, + node: HtmlElement, ) { val src = node.attr("src").takeIf { it.isNotBlank() } ?: return val decoded = decodeDataUri(src) ?: return diff --git a/htmlparser/src/iosMain/kotlin/io/github/kdroidfilter/seforim/htmlparser/HtmlDom.ios.kt b/htmlparser/src/iosMain/kotlin/io/github/kdroidfilter/seforim/htmlparser/HtmlDom.ios.kt new file mode 100644 index 000000000..371b015f8 --- /dev/null +++ b/htmlparser/src/iosMain/kotlin/io/github/kdroidfilter/seforim/htmlparser/HtmlDom.ios.kt @@ -0,0 +1,27 @@ +package io.github.kdroidfilter.seforim.htmlparser + +import com.fleeksoft.ksoup.Ksoup +import com.fleeksoft.ksoup.nodes.Element +import com.fleeksoft.ksoup.nodes.Node +import com.fleeksoft.ksoup.nodes.TextNode + +// Ksoup is jsoup's Kotlin port: the same DOM as on the JVM and Android +internal actual fun parseHtmlBody(html: String): List = + Ksoup + .parse(html) + .body() + .childNodes() + .mapNotNull(::toHtmlNode) + +// Text and elements: comments and the like carry nothing to render +private fun toHtmlNode(node: Node): HtmlNode? = + when (node) { + is TextNode -> HtmlText(node.text()) + is Element -> + HtmlElement( + tag = node.normalName(), + attributes = node.attributes().associate { it.key to it.value }, + children = node.childNodes().mapNotNull(::toHtmlNode), + ) + else -> null + } diff --git a/logger/build.gradle.kts b/logger/build.gradle.kts index 510fb2852..bbdd028fb 100644 --- a/logger/build.gradle.kts +++ b/logger/build.gradle.kts @@ -1,11 +1,20 @@ plugins { alias(libs.plugins.multiplatform) alias(libs.plugins.kotlinx.serialization) + alias(libs.plugins.android.kmp.library) } +// The app's logs: the console in development, Sentry once the app initializes it (the desktop: main.kt), on every +// platform. On iOS the app links Sentry's Cocoa SDK, which Sentry KMP calls. kotlin { - jvm() + android { + namespace = "io.github.kdroidfilter.seforimapp.logger" + compileSdk = 37 + minSdk = 21 + } + iosArm64() + iosSimulatorArm64() jvmToolchain( libs.versions.jvmToolchain .get() @@ -14,10 +23,8 @@ kotlin { sourceSets { commonMain.dependencies { - } - - jvmMain.dependencies { - implementation(libs.sentry.core) + implementation(libs.kotlinx.datetime) + implementation(libs.sentry.kmp) } } } diff --git a/logger/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/logger/Logger.kt b/logger/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/logger/Logger.kt index 1130a5408..f9796076a 100644 --- a/logger/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/logger/Logger.kt +++ b/logger/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/logger/Logger.kt @@ -1,9 +1,15 @@ package io.github.kdroidfilter.seforimapp.logger -import io.sentry.Sentry -import io.sentry.SentryLevel -import java.text.SimpleDateFormat -import java.util.Date +import io.sentry.kotlin.multiplatform.Sentry +import io.sentry.kotlin.multiplatform.SentryLevel +import kotlinx.datetime.LocalDate +import kotlinx.datetime.LocalDateTime +import kotlinx.datetime.TimeZone +import kotlinx.datetime.format.char +import kotlinx.datetime.toLocalDateTime +import kotlin.concurrent.Volatile +import kotlin.jvm.JvmField +import kotlin.time.Clock var isDevEnv: Boolean = true var loggingLevel: LoggingLevel = LoggingLevel.VERBOSE @@ -40,9 +46,24 @@ class LoggingLevel( @PublishedApi internal const val COLOR_RESET = "\u001b[0m" -@PublishedApi internal val dateFormat = SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS") +// yyyy-MM-dd HH:mm:ss.SSS +@PublishedApi internal val dateFormat = + LocalDateTime.Format { + date(LocalDate.Formats.ISO) + char(' ') + hour() + char(':') + minute() + char(':') + second() + char('.') + secondFraction(3) + } -@PublishedApi internal fun getCurrentTimestamp(): String = dateFormat.format(Date()) +@PublishedApi internal fun getCurrentTimestamp(): String = + dateFormat.format( + Clock.System.now().toLocalDateTime(TimeZone.currentSystemDefault()), + ) @PublishedApi internal fun shouldLogToConsole(minLevel: LoggingLevel): Boolean = isDevEnv && loggingLevel.priority <= minLevel.priority @@ -79,14 +100,13 @@ class LoggingLevel( if (sendToSentry) { runCatching { val sentryLevel = toSentryLevel(minLevel) - Sentry.withScope { scope -> - scope.level = sentryLevel - if (throwable != null) { + if (throwable != null) { + Sentry.captureException(throwable) { scope -> + scope.level = sentryLevel scope.setExtra("logger.message", renderedMessage) - Sentry.captureException(throwable) - } else { - Sentry.captureMessage(renderedMessage, sentryLevel) } + } else { + Sentry.captureMessage(renderedMessage) { scope -> scope.level = sentryLevel } } } } diff --git a/navigation/build.gradle.kts b/navigation/build.gradle.kts index 1eb67f6d7..026c3681a 100644 --- a/navigation/build.gradle.kts +++ b/navigation/build.gradle.kts @@ -2,6 +2,7 @@ import org.jetbrains.compose.reload.gradle.ComposeHotRun plugins { alias(libs.plugins.multiplatform) + alias(libs.plugins.android.kmp.library) alias(libs.plugins.compose.compiler) alias(libs.plugins.compose) alias(libs.plugins.kotlinx.serialization) @@ -15,6 +16,14 @@ kotlin { ) jvm() + // The tabs' destinations are shared with the mobile apps' search (searchkit) + android { + namespace = "io.github.kdroidfilter.seforim.navigation" + compileSdk = 37 + minSdk = 26 + } + iosArm64() + iosSimulatorArm64() sourceSets { commonMain.dependencies { diff --git a/navigation/src/commonMain/kotlin/io/github/kdroidfilter/seforim/tabs/TabsState.kt b/navigation/src/commonMain/kotlin/io/github/kdroidfilter/seforim/tabs/TabsState.kt index 2af6d8d7a..72e466278 100644 --- a/navigation/src/commonMain/kotlin/io/github/kdroidfilter/seforim/tabs/TabsState.kt +++ b/navigation/src/commonMain/kotlin/io/github/kdroidfilter/seforim/tabs/TabsState.kt @@ -1,13 +1,14 @@ package io.github.kdroidfilter.seforim.tabs import androidx.compose.runtime.Immutable -import java.util.UUID +import kotlin.uuid.ExperimentalUuidApi +import kotlin.uuid.Uuid @Immutable data class TabItem( val id: Int, val title: String = "Default Tab", - val destination: TabsDestination = TabsDestination.Home(UUID.randomUUID().toString()), + val destination: TabsDestination = TabsDestination.Home(newTabId()), val tabType: TabType = TabType.SEARCH, // Pinned tabs stay ahead of the others and are not closed by accident. val pinned: Boolean = false, @@ -20,3 +21,7 @@ data class TabsState( val tabs: List, val selectedTabIndex: Int, ) + +/** A fresh tab id. */ +@OptIn(ExperimentalUuidApi::class) +fun newTabId(): String = Uuid.random().toString() diff --git a/searchkit-check/build.gradle.kts b/searchkit-check/build.gradle.kts new file mode 100644 index 000000000..bc0cbbd09 --- /dev/null +++ b/searchkit-check/build.gradle.kts @@ -0,0 +1,31 @@ +// A test harness, no UI: an Android app that holds searchkit as a real app does (the Java runtime NJA brings with +// SeforimLibrary's search: its native libraries and assets) and runs the search on a device or an emulator through its +// instrumented test. +plugins { + alias(libs.plugins.android.application) +} + +android { + namespace = "io.github.kdroidfilter.seforimapp.searchkit.check" + compileSdk = 37 + + defaultConfig { + applicationId = "io.github.kdroidfilter.seforimapp.searchkit.check" + minSdk = 29 + targetSdk = 36 + testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" + } + + sourceSets { + // The dump compared across platforms + getByName("androidTest").kotlin.directories.add("../searchkit/src/commonTest/kotlin") + } +} + +dependencies { + implementation(project(":searchkit")) + androidTestImplementation(libs.androidx.test.runner) + androidTestImplementation(libs.androidx.test.junit) + androidTestImplementation(libs.kotlinx.coroutines.test) + androidTestImplementation(kotlin("test")) +} diff --git a/searchkit-check/src/androidTest/kotlin/io/github/kdroidfilter/seforimapp/searchkit/check/SearchParityTest.kt b/searchkit-check/src/androidTest/kotlin/io/github/kdroidfilter/seforimapp/searchkit/check/SearchParityTest.kt new file mode 100644 index 000000000..6ca4fe792 --- /dev/null +++ b/searchkit-check/src/androidTest/kotlin/io/github/kdroidfilter/seforimapp/searchkit/check/SearchParityTest.kt @@ -0,0 +1,41 @@ +package io.github.kdroidfilter.seforimapp.searchkit.check + +import android.os.SystemClock +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import io.github.kdroidfilter.seforimapp.features.search.semantic.SemanticModels +import io.github.kdroidfilter.seforimapp.framework.search.SearchKitDump +import io.github.kdroidfilter.seforimapp.framework.search.SearchServices +import io.github.kdroidfilter.seforimapp.framework.search.openSeforimRepository +import kotlinx.coroutines.runBlocking +import org.junit.Test +import org.junit.runner.RunWith +import java.io.File + +/** + * The app's search on this device, as an app runs it: Lucene on the JVM NJA runs in the app's process. The data goes + * in the app's files dir (adb root, push, chown to the app): `seforim/` (seforim.db, its indexes, lexical.db, + * `model/`). Writes SearchKitDump as `searchkit.txt` there; skipped without data. + */ +@RunWith(AndroidJUnit4::class) +class SearchParityTest { + @Test + fun dump() { + val context = InstrumentationRegistry.getInstrumentation().targetContext + val dir = File(context.filesDir, "seforim") + val db = File(dir, "seforim.db").takeIf { it.isFile } ?: return + val model = File(dir, "model") + val models = + SemanticModels( + File(model, "seforim-embed-v5-int8.onnx").path, + File(model, "tokenizer.json").path, + File(model, "seforim-embed-v5-static.i8").path, + ).takeIf { model.isDirectory } + SearchServices(db.path, openSeforimRepository(context, db.path), models).use { services -> + val start = SystemClock.elapsedRealtime() + val dump = runBlocking { SearchKitDump.dump(services) } + println("SearchKit dump in ${SystemClock.elapsedRealtime() - start} ms") + File(context.filesDir, "searchkit.txt").writeText(dump) + } + } +} diff --git a/searchkit-check/src/main/AndroidManifest.xml b/searchkit-check/src/main/AndroidManifest.xml new file mode 100644 index 000000000..9917f7cb8 --- /dev/null +++ b/searchkit-check/src/main/AndroidManifest.xml @@ -0,0 +1,4 @@ + + + + diff --git a/searchkit/README.md b/searchkit/README.md new file mode 100644 index 000000000..b38210ee8 --- /dev/null +++ b/searchkit/README.md @@ -0,0 +1,86 @@ +# searchkit + +Zayit's search without its UI, for the desktop app and the mobile ones (Android 10+, iOS): the engines (lexical, fused +with the semantic search in the official builds), the lookup (books, authors, tables of contents), the search use cases +and the view models of the search screens. The desktop app uses it as is; a mobile app only adds its screens. + +Verified on real data: `SearchKitDump` (lookup, hybrid search, snippets, highlights by meaning, smart find) gives the +same text on the JVM, in the iOS simulator and on Android 11 and 15 emulators (`jvmTest`, `iosSimulatorArm64Test`, +`searchkit-check`). + +## What runs where + +| | Desktop (JVM) | Android 10+ | iOS | +|---|---|---|---| +| Lucene (full-text and vector index) | in the app's JVM | on HotSpot in the app's process (NJA) | compiled into a GraalVM native image, linked into the app (NJA) | +| Database (`SeforimRepository`) | the app's pooled JDBC driver | `openSeforimRepository(context, path)` | `openSeforimRepository(path)` | +| Semantic model (official builds) | ONNX Runtime Java, model in the jar | ONNX Runtime Android | ONNX Runtime C API (`onnxruntime-c` 1.20.0) | +| Model files | bundled | downloaded with the data | downloaded with the data | + +Lucene runs on Java only, never on ART: ART can't keep up with the Java versions Lucene requires (21 for Lucene 10, 25 +for 11). SeforimLibrary's search reaches it through [Nucleus JVM Access](https://github.com/NucleusFramework/NucleusJvmAccess): +its façade (`search-facade`) runs on the JVM, or compiled ahead of time on iOS, and its common code calls the proxies +NJA generates. Android's JVM is jdk-mobile's OpenJDK 27; iOS's native image is GraalVM CE's, with graalvm-mobile's +libraries (see [JVM_RUNTIME.md](../SeforimLibrary/JVM_RUNTIME.md)). + +Everything else (query building, dictionary, ranking, snippets, highlights, RRF fusion, tokenizer, use cases, view +models) is common code. + +## Using it from an app + +```kotlin +// The data: seforim.db with seforim.db.lucene, seforim.db.lookup.lucene and lexical.db beside it, downloaded into the +// app's storage (like the desktop's data). The semantic model too, in the official builds. +val repository = openSeforimRepository(context, dbPath) // iOS: openSeforimRepository(dbPath) +val services = SearchServices(dbPath, repository, SemanticModels(model, tokenizer, staticTable)) +services.engine // SearchEngine: openSession, computeFacets, highlights, semanticFind... +services.lookup // LuceneLookupSearchService: suggestBooks, suggestAuthors, searchTocPrefix +``` + +On Android there is nothing to configure: the JVM (`libnja.so`, arm64-v8a and x86_64) and its runtime come with +SeforimLibrary's search library (its native libraries and assets), and start on the first search. `searchkit-check` is +such an app (no UI). + +On iOS, `./gradlew :searchkit:assembleIosJavaRuntimeDevice` assembles `static/`: what the app links, the search's +native image (`nja.o`) and graalvm-mobile's libraries (the options are in JVM_RUNTIME.md). Nothing goes into `App.app`. + +### The view models + +`SearchResultViewModel` (a results tab: search, facets, filters by category/book/TOC, paging, preview, breadcrumbs, +opening a result) and `SearchHomeViewModel` (the search bar: book/author/TOC/reference suggestions, scopes, deep links) +are the desktop's, unchanged. What they need of the app is in `SearchHost.kt`, four small interfaces: + +| Interface | Desktop implementation | What a mobile app provides | +|---|---|---| +| `SearchTabs` | `DesktopSearchTabs` (windows and tabs) | its navigation: open a destination (`TabsDestination`), screen still shown | +| `SearchSessionStore` | `DesktopSearchSessionStore` (the session) | where it keeps a screen's `SearchPersistedState` and opens a book at a line | +| `SearchSettings` | `AppSettings` | text size, find-in-page bar, user's name | +| `SearchHistory` | `HistoryStore` | the search history | + +`SearchResultViewModel` is built by Metro on the desktop (`@AssistedInject`, `SavedStateHandle` with `StateKeys.TAB_ID`); +a mobile app can call its constructor directly. + +## Builds + +- **Community**: `addons/semantic/disabled`, lexical search only. +- **Official**: `addons/semantic/enabled` with the private `io.github.kdroidfilter:seforim-semantic` package (Kotlin + Multiplatform since 0.6.0: publish it before an official CI build), or its sources beside with `-Psemantic.local=true`. + +## Tests on real data + +```bash +./gradlew :searchkit:jvmTest -PseforimTestDb= # build/parity/jvm.txt +./gradlew :searchkit:iosSimulatorArm64Test -PseforimTestDb= \ + -Pseforim.ortXcframework= -Pseforim.semanticModels= # build/parity/ios.txt +``` + +Use a copy of the database (an APFS clone: `cp -c`): `SeforimRepository` sets its PRAGMAs on open. On Android, install +`searchkit-check` and its test APK, push the data and the runtime into its files dir and run its `SearchParityTest` +(see its KDoc). + +## Not here (the UI's) + +- The screens, and the rendering of snippets and lines (`:htmlparser`, Jsoup and Compose `AnnotatedString`: on iOS it + needs Ksoup). +- The reader's find-in-page (`BookFindMatches`): its candidates come from the engine (`findInBookCandidates`), its + matches are confirmed on the displayed text, so it follows `:htmlparser`. diff --git a/SeforimApp/addons/semantic/disabled/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/InstalledSemanticSearch.kt b/searchkit/addons/semantic/disabled/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/InstalledSemanticSearch.kt similarity index 100% rename from SeforimApp/addons/semantic/disabled/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/InstalledSemanticSearch.kt rename to searchkit/addons/semantic/disabled/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/InstalledSemanticSearch.kt diff --git a/searchkit/addons/semantic/enabled/jvm/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/BundledSemanticModels.jvm.kt b/searchkit/addons/semantic/enabled/jvm/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/BundledSemanticModels.jvm.kt new file mode 100644 index 000000000..4aef78165 --- /dev/null +++ b/searchkit/addons/semantic/enabled/jvm/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/BundledSemanticModels.jvm.kt @@ -0,0 +1,8 @@ +package io.github.kdroidfilter.seforimapp.features.search.semantic + +import io.github.kdroidfilter.seforim.semantic.bundledModelFiles + +// The desktop's semantic jar carries the model, extracted on first use +private val bundled = lazy { bundledModelFiles().let { SemanticModels(it.model, it.tokenizer, it.staticTable) } } + +internal actual fun bundledSemanticModels(): Lazy? = bundled diff --git a/SeforimApp/addons/semantic/enabled/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/HybridSearchEngine.kt b/searchkit/addons/semantic/enabled/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/HybridSearchEngine.kt similarity index 88% rename from SeforimApp/addons/semantic/enabled/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/HybridSearchEngine.kt rename to searchkit/addons/semantic/enabled/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/HybridSearchEngine.kt index d9a74ee03..1b6195fe7 100644 --- a/SeforimApp/addons/semantic/enabled/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/HybridSearchEngine.kt +++ b/searchkit/addons/semantic/enabled/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/HybridSearchEngine.kt @@ -1,7 +1,8 @@ package io.github.kdroidfilter.seforimapp.features.search.semantic +import co.touchlab.kermit.Logger +import io.github.kdroidfilter.seforim.semantic.ModelFiles import io.github.kdroidfilter.seforim.semantic.SemanticSearch -import io.github.kdroidfilter.seforimapp.logger.warnln import io.github.kdroidfilter.seforimlibrary.search.LineHit import io.github.kdroidfilter.seforimlibrary.search.SearchEngine import io.github.kdroidfilter.seforimlibrary.search.SearchFacets @@ -12,13 +13,17 @@ import io.github.kdroidfilter.seforimlibrary.search.SnippetSources import io.github.kdroidfilter.seforimlibrary.search.WARMUP_PAGE_SIZE import io.github.kdroidfilter.seforimlibrary.search.WARMUP_QUERIES import io.github.kdroidfilter.seforimlibrary.search.WARMUP_ROUNDS +import io.github.kdroidfilter.seforimlibrary.search.index.LuceneIndex import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.IO import kotlinx.coroutines.async import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import kotlinx.coroutines.withContext -import java.nio.file.Path +import kotlin.concurrent.Volatile + +private val logger = Logger.withTag("HybridSearch") /** * Hybrid search: the lexical engine (BM25 + MagicDictionary) fused with dense semantic search ([SemanticSearch]) by @@ -30,7 +35,9 @@ import java.nio.file.Path */ internal class HybridSearchEngine( private val lexical: SearchEngine, - private val indexDir: Path, + private val index: LuceneIndex, + // Resolved on the first dense search: the desktop's model is extracted from its jar then, not at startup + private val models: Lazy?, private val resolveLine: suspend (lineId: Long, query: String) -> LineHit?, ) : SearchEngine { // The model (heavy ONNX session) loads lazily on the first dense search, off the main thread, so the first query @@ -40,21 +47,20 @@ internal class HybridSearchEngine( @Volatile private var semanticTried = false private val semanticMutex = Mutex() - // Cheap, no-load check: decides whether to take the dense path at all - private val semanticConfigured = SemanticSearch.isAvailable(indexDir) + // Cheap, no-load check: decides whether to take the dense path at all (the vectors are in the index itself) + private val semanticConfigured = models != null private suspend fun ensureSemantic(): SemanticSearch? { if (semanticTried) return semantic semanticMutex.withLock { if (!semanticTried) { semantic = - if (semanticConfigured) { - // The tokenizer library otherwise "calls home" (an EC2 metadata probe, then a tracking ping) - // when it loads: seconds of network timeouts on the first search, and a privacy leak - System.setProperty("OPT_OUT_TRACKING", "true") + if (models != null) { withContext(Dispatchers.IO) { - runCatching { SemanticSearch.open(indexDir) } - .onFailure { warnln(it) { "[HybridSearch] Semantic search unavailable; lexical only" } } + runCatching { + val files = models.value + SemanticSearch.open(LuceneVectorIndex(index), ModelFiles(files.model, files.tokenizer, files.staticTable)) + }.onFailure { logger.w(it) { "Semantic search unavailable; lexical only" } } .getOrNull() } } else { @@ -194,7 +200,7 @@ internal class HybridSearchEngine( try { semantic.search(query, CANDIDATES, baseOnly, bookIds) } catch (e: Exception) { - warnln(e) { "[HybridSearch] Dense KNN failed; lexical only" } + logger.w(e) { "Dense KNN failed; lexical only" } null } } @@ -206,11 +212,12 @@ internal class HybridSearchEngine( val denseHits = dense.await() ?: return@coroutineScope lexHits.map { FusedEntry(it.lineId, it.score, it) } val rrf = HashMap() - lexHits.forEachIndexed { rank, hit -> rrf.merge(hit.lineId, 1.0 / (RRF_K + rank + 1), Double::plus) } - denseHits.forEachIndexed { rank, hit -> rrf.merge(hit.lineId, 1.0 / (RRF_K + rank + 1), Double::plus) } + lexHits.forEachIndexed { rank, hit -> rrf[hit.lineId] = (rrf[hit.lineId] ?: 0.0) + 1.0 / (RRF_K + rank + 1) } + denseHits.forEachIndexed { rank, hit -> rrf[hit.lineId] = (rrf[hit.lineId] ?: 0.0) + 1.0 / (RRF_K + rank + 1) } val lexById = lexHits.associateBy { it.lineId } + // Ties broken by line: the same order on every platform, whatever the map's iteration order rrf.entries - .sortedByDescending { it.value } + .sortedWith(compareByDescending> { it.value }.thenBy { it.key }) .map { (lineId, score) -> FusedEntry(lineId, score.toFloat(), lexById[lineId]) } } diff --git a/searchkit/addons/semantic/enabled/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/InstalledSemanticSearch.kt b/searchkit/addons/semantic/enabled/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/InstalledSemanticSearch.kt new file mode 100644 index 000000000..4a88969bf --- /dev/null +++ b/searchkit/addons/semantic/enabled/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/InstalledSemanticSearch.kt @@ -0,0 +1,21 @@ +package io.github.kdroidfilter.seforimapp.features.search.semantic + +import io.github.kdroidfilter.seforimlibrary.search.LineHit +import io.github.kdroidfilter.seforimlibrary.search.SearchEngine +import io.github.kdroidfilter.seforimlibrary.search.index.LuceneIndex + +/** An official build: the semantic search of the SeforimEmbedding package. */ +val installedSemanticSearch: SemanticSearchFeature? = + object : SemanticSearchFeature { + override val bundledModels: SemanticModels? get() = bundledSemanticModels()?.value + + override fun hybrid( + lexical: SearchEngine, + index: LuceneIndex, + models: SemanticModels?, + resolveLine: suspend (lineId: Long, query: String) -> LineHit?, + ): SearchEngine = HybridSearchEngine(lexical, index, models?.let { lazyOf(it) } ?: bundledSemanticModels(), resolveLine) + } + +/** The model the package carries on this platform (the desktop's jar, extracted on first use), null on mobile. */ +internal expect fun bundledSemanticModels(): Lazy? diff --git a/searchkit/addons/semantic/enabled/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/LuceneVectorIndex.kt b/searchkit/addons/semantic/enabled/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/LuceneVectorIndex.kt new file mode 100644 index 000000000..894a46b6b --- /dev/null +++ b/searchkit/addons/semantic/enabled/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/LuceneVectorIndex.kt @@ -0,0 +1,67 @@ +package io.github.kdroidfilter.seforimapp.features.search.semantic + +import io.github.kdroidfilter.seforim.semantic.DenseHit +import io.github.kdroidfilter.seforim.semantic.VectorIndex +import io.github.kdroidfilter.seforimlibrary.search.index.BoolBuilder +import io.github.kdroidfilter.seforimlibrary.search.index.IndexQuery +import io.github.kdroidfilter.seforimlibrary.search.index.KnnRequest +import io.github.kdroidfilter.seforimlibrary.search.index.LuceneIndex +import io.github.kdroidfilter.seforimlibrary.search.index.Occur +import io.github.kdroidfilter.seforimlibrary.search.index.VectorEncoding + +/** + * The vectors of the app's fused Lucene index for the semantic search: its KNN runs through [index], in process on the + * JVM and Android, in the Java runtime of the iOS app. Not closed with the semantic search: the lexical one shares it. + */ +internal class LuceneVectorIndex( + private val index: LuceneIndex, +) : VectorIndex { + override fun encoding(field: String): VectorIndex.Encoding? = + when (index.vectorEncoding(field)) { + VectorEncoding.BYTE -> VectorIndex.Encoding.BYTE + VectorEncoding.FLOAT32 -> VectorIndex.Encoding.FLOAT32 + null -> null + } + + override fun knn( + field: String, + k: Int, + floatVector: FloatArray?, + byteVector: ByteArray?, + baseBookOnly: Boolean, + bookIds: Collection?, + oversample: Int, + ): List = + index + .knn( + KnnRequest( + field = field, + k = k, + fields = setOf(FIELD_LINE_ID, FIELD_BOOK_ID), + floatVector = floatVector, + byteVector = byteVector, + filter = filter(baseBookOnly, bookIds), + oversample = oversample, + ), + ).map { DenseHit(lineId = it.longs.getValue(FIELD_LINE_ID), bookId = it.longs.getValue(FIELD_BOOK_ID), score = it.score) } + + // Base books and specific books, as a pre-filter: the KNN only considers eligible lines + private fun filter( + baseBookOnly: Boolean, + bookIds: Collection?, + ): IndexQuery? { + if (!baseBookOnly && bookIds.isNullOrEmpty()) return null + val builder = BoolBuilder() + if (baseBookOnly) builder.add(IndexQuery.IntExact(FIELD_IS_BASE_BOOK, 1), Occur.FILTER) + if (!bookIds.isNullOrEmpty()) builder.add(IndexQuery.IntSet(FIELD_BOOK_ID, bookIds.map { it.toInt() }.toIntArray()), Occur.FILTER) + return builder.build() + } + + override fun close() = Unit + + private companion object { + const val FIELD_LINE_ID = "line_id" + const val FIELD_BOOK_ID = "book_id" + const val FIELD_IS_BASE_BOOK = "is_base_book" + } +} diff --git a/searchkit/addons/semantic/enabled/mobile/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/BundledSemanticModels.mobile.kt b/searchkit/addons/semantic/enabled/mobile/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/BundledSemanticModels.mobile.kt new file mode 100644 index 000000000..67d950f24 --- /dev/null +++ b/searchkit/addons/semantic/enabled/mobile/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/BundledSemanticModels.mobile.kt @@ -0,0 +1,4 @@ +package io.github.kdroidfilter.seforimapp.features.search.semantic + +// A mobile app downloads the model with the data (too large for the package): it passes its paths +internal actual fun bundledSemanticModels(): Lazy? = null diff --git a/searchkit/build.gradle.kts b/searchkit/build.gradle.kts new file mode 100644 index 000000000..8f1292836 --- /dev/null +++ b/searchkit/build.gradle.kts @@ -0,0 +1,189 @@ +plugins { + alias(libs.plugins.multiplatform) + alias(libs.plugins.android.kmp.library) + alias(libs.plugins.kotlinx.serialization) + // The view models contribute their factories to the app's graph (the desktop's AppGraph) + alias(libs.plugins.metro) + // The logger (find-in-book) reports to Sentry: its Cocoa SDK, which the iOS binaries link, installed through SwiftPM + alias(libs.plugins.sentryKmpGradle) +} + +// The semantic search is in the official builds only (open core): with its package's token or its sources beside +// (settings.gradle.kts); a community build compiles addons/semantic/disabled and the search stays lexical +val withSemantic = gradle.extensions.extraProperties["semanticEnabled"] == true + +/** + * Zayit's search, without its UI: the engines (lexical, hybrid with the semantic search), the lookup, the use cases and + * the view models of the search screens, for the desktop app and the mobile ones (Android 10+, iOS). Lucene runs in + * process on the JVM, in the Java runtime the mobile apps embed (see SeforimLibrary's JVM_RUNTIME.md). + */ +kotlin { + jvmToolchain( + libs.versions.jvmToolchain + .get() + .toInt(), + ) + + jvm() + android { + namespace = "io.github.kdroidfilter.seforimapp.searchkit" + compileSdk = 37 + // NJA's bridge runs from Android 10 + minSdk = 29 + } + iosArm64() + iosSimulatorArm64() + + sourceSets { + commonMain { + kotlin.srcDir(if (withSemantic) "addons/semantic/enabled/kotlin" else "addons/semantic/disabled/kotlin") + dependencies { + api(libs.seforimlibrary.core) + api(libs.seforimlibrary.dao) + api(libs.seforimlibrary.search) + implementation(libs.kotlinx.coroutines.core) + implementation(libs.kotlinx.serialization.json) + implementation(libs.kotlinx.collections.immutable) + implementation(libs.kotlinx.io.core) + implementation(libs.androidx.lifecycle.viewmodel.core) + implementation(libs.androidx.lifecycle.viewmodel.savedstate) + implementation(libs.metrox.viewmodel) + api(project(":navigation")) + // The reader's find-in-book (BookFindMatches): lines as plain text, its logs + implementation(project(":htmlparser")) + implementation(project(":logger")) + implementation(libs.compose.runtime) + implementation(libs.hebrew.numerals) + implementation(libs.kermit) + implementation(libs.seforimlibrary.cache) + implementation(libs.atomicfu) + if (withSemantic) implementation(libs.seforim.semantic) + } + } + + jvmMain.dependencies { implementation(libs.sqlite.driver) } + androidMain.dependencies { implementation(libs.sqldelight.android.driver) } + iosMain.dependencies { implementation(libs.sqldelight.native.driver) } + + // The model's location differs by platform: the desktop's jar carries it, mobile apps download it + if (withSemantic) { + jvmMain { kotlin.srcDir("addons/semantic/enabled/jvm/kotlin") } + androidMain { kotlin.srcDir("addons/semantic/enabled/mobile/kotlin") } + iosMain { kotlin.srcDir("addons/semantic/enabled/mobile/kotlin") } + } + + commonTest.dependencies { + implementation(kotlin("test")) + implementation(libs.kotlinx.coroutines.test) + } + } +} + +// --- Tests on real data: the same dump on the JVM and in the iOS simulator (SearchKitDump) --- + +// The Java runtime of the mobile apps comes with SeforimLibrary's search, through Nucleus JVM Access (see its +// JVM_RUNTIME.md): Android gets it from search's library (its native libraries and assets); an iOS binary links its +// GraalVM native image (search/build/nja/nativeImage//nja.o, which NJA builds) with graalvm-mobile's static +// libraries (search/build/nja/graalvmMobile//lib, which NJA downloads): nothing to bundle +val seforimLibrary = gradle.includedBuild("SeforimLibrary") +val searchBuild = seforimLibrary.projectDir.resolve("search/build") + +fun searchTask(name: String) = seforimLibrary.task(":search:$name") + +// ONNX Runtime's xcframework, for the semantic search in the iOS tests: -Pseforim.ortXcframework= +val ortXcframework: File? = providers.gradleProperty("seforim.ortXcframework").orNull?.let(::file) +// Substrate VM's and the JDK's natives, from graalvm-mobile's kit +val nativeImageLibs = + listOf( + "jvm", + "libchelper", + "darwin", + "java", + "nio", + "net", + "zip", + "extnet", + "jimage", + "verify", + "management_ext", + "management", + "prefs", + ) + +fun searchStage(runtime: String) = runtime.split("-").joinToString("") { it.replaceFirstChar(Char::uppercase) } + +// What an iOS binary links for the search: its native image (entry points exported: NJA's client finds them at run +// time), the kit's libraries, and ONNX Runtime for the semantic search +fun org.jetbrains.kotlin.gradle.plugin.mpp.NativeBinary.linkJavaRuntime( + runtime: String, + ortSlice: File?, +) { + val lib = searchBuild.resolve("nja/graalvmMobile/$runtime/lib") + linkerOpts(searchBuild.resolve("nja/nativeImage/$runtime/nja.o").path) + linkerOpts(nativeImageLibs.map { lib.resolve("lib$it.a").path }) + linkerOpts("-export_dynamic", "-lz", "-lc++", "-framework", "Foundation", "-framework", "CoreFoundation") + if (ortSlice != null) linkerOpts("-F", ortSlice.path, "-framework", "onnxruntime") + linkTaskProvider.configure { dependsOn(searchTask("njaNativeImage${searchStage(runtime)}")) } +} + +kotlin { + iosSimulatorArm64 { + binaries.withType().configureEach { + linkJavaRuntime("ios-simulator-aarch64", ortXcframework?.resolve("ios-arm64_x86_64-simulator")) + } + } + // A device build of the tests only checks that everything links (no device runs them here) + iosArm64 { + binaries.withType().configureEach { + linkJavaRuntime("ios-device-aarch64", ortXcframework?.resolve("ios-arm64")) + } + } +} + +val testDb = providers.gradleProperty("seforimTestDb") +val parityDir = layout.buildDirectory.dir("parity") + +tasks.named("jvmTest") { + testDb.orNull?.let { systemProperty("seforimTestDb", it) } + val out = parityDir.get().file("jvm.txt").asFile + systemProperty("parityOut", out.path) + testLogging.showStandardStreams = true + doFirst { out.parentFile.mkdirs() } +} + +// For the Xcode project: build/iosJavaRuntime//static: what the app links (see linkJavaRuntime), the native +// image (nja.o) and the kit's libraries; nothing to copy into App.app +listOf("simulator", "device").forEach { kind -> + tasks.register("assembleIosJavaRuntime${kind.replaceFirstChar { it.uppercase() }}") { + group = "build" + description = "Assembles what the iOS app links for the search's native image ($kind)." + val runtime = "ios-$kind-aarch64" + dependsOn(searchTask("njaNativeImage${searchStage(runtime)}")) + from(searchBuild.resolve("nja/nativeImage/$runtime/nja.o")) { into("static") } + from(searchBuild.resolve("nja/graalvmMobile/$runtime/lib")) { + include(nativeImageLibs.map { "lib$it.a" }) + into("static") + } + into(layout.buildDirectory.dir("iosJavaRuntime/$kind")) + filePermissions { user { write = true } } + } +} + +run { + tasks.withType().configureEach { + device.set(providers.gradleProperty("seforim.iosSimulator").orElse("iPhone 17 Pro")) + val models = providers.gradleProperty("seforim.semanticModels") + val db = providers.gradleProperty("seforimTestDb") + val out = + parityDir + .get() + .file("ios.txt") + .asFile.path + doFirst { + // The simulator passes the SIMCTL_CHILD_ variables to the test process, without the prefix + db.orNull?.let { environment("SIMCTL_CHILD_SEFORIM_TEST_DB", it) } + models.orNull?.let { environment("SIMCTL_CHILD_SEFORIM_MODEL_DIR", it) } + environment("SIMCTL_CHILD_SEFORIM_PARITY_OUT", out) + } + } +} diff --git a/searchkit/src/androidMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/Repositories.android.kt b/searchkit/src/androidMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/Repositories.android.kt new file mode 100644 index 000000000..9ee98ea0b --- /dev/null +++ b/searchkit/src/androidMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/Repositories.android.kt @@ -0,0 +1,38 @@ +package io.github.kdroidfilter.seforimapp.framework.search + +import android.content.Context +import app.cash.sqldelight.db.QueryResult +import app.cash.sqldelight.db.SqlDriver +import app.cash.sqldelight.db.SqlSchema +import app.cash.sqldelight.driver.android.AndroidSqliteDriver +import io.github.kdroidfilter.seforimlibrary.dao.repository.SeforimRepository + +/** + * The repository of the database at [databasePath] (an absolute path: the downloaded database, not one the app ships). + * Its schema exists already: the driver creates and migrates nothing. + */ +fun openSeforimRepository( + context: Context, + databasePath: String, +): SeforimRepository = + SeforimRepository( + databasePath, + AndroidSqliteDriver(ExistingSchema, context, databasePath, windowSizeBytes = CURSOR_WINDOW_BYTES), + ) + +// A book's notes reach 2.5 MB: more than the 2 MB a cursor holds by default on Android +private const val CURSOR_WINDOW_BYTES = 8L * 1024 * 1024 + +// The downloaded database is complete: version 1, never created nor migrated here +private object ExistingSchema : SqlSchema> { + override val version: Long = 1 + + override fun create(driver: SqlDriver) = QueryResult.Unit + + override fun migrate( + driver: SqlDriver, + oldVersion: Long, + newVersion: Long, + vararg callbacks: app.cash.sqldelight.db.AfterVersion, + ) = QueryResult.Unit +} diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/core/coroutines/CoroutineUtils.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/core/coroutines/CoroutineUtils.kt similarity index 100% rename from SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/core/coroutines/CoroutineUtils.kt rename to searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/core/coroutines/CoroutineUtils.kt diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/core/deeplink/ZayitDeepLink.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/core/deeplink/ZayitDeepLink.kt similarity index 67% rename from SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/core/deeplink/ZayitDeepLink.kt rename to searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/core/deeplink/ZayitDeepLink.kt index effb2360d..6774c567c 100644 --- a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/core/deeplink/ZayitDeepLink.kt +++ b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/core/deeplink/ZayitDeepLink.kt @@ -1,10 +1,7 @@ package io.github.kdroidfilter.seforimapp.core.deeplink import io.github.kdroidfilter.seforim.tabs.TabsDestination -import java.net.URLDecoder -import java.net.URLEncoder -import java.nio.charset.StandardCharsets -import java.util.UUID +import io.github.kdroidfilter.seforim.tabs.newTabId /** * Public, shareable deep link scheme for Zayit. @@ -34,7 +31,7 @@ fun bookShareLink( } /** Builds a shareable link to a search query. */ -fun searchShareLink(query: String): String = PREFIX + HOST_SEARCH + "/" + URLEncoder.encode(query, StandardCharsets.UTF_8) +fun searchShareLink(query: String): String = PREFIX + HOST_SEARCH + "/" + formEncode(query) /** * Returns a shareable link for this destination, or null when there is nothing worth sharing @@ -62,7 +59,7 @@ fun parseZayitDeepLink(uri: String): TabsDestination? { val path = uri.removePrefix(PREFIX) val segments = path.split('/').filter { it.isNotEmpty() } if (segments.isEmpty()) return null - val newTabId = UUID.randomUUID().toString() + val newTabId = newTabId() return when (segments[0]) { HOST_BOOK -> { val bookId = segments.getOrNull(1)?.toLongOrNull() ?: return null @@ -77,8 +74,42 @@ fun parseZayitDeepLink(uri: String): TabsDestination? { HOST_SEARCH -> { val encoded = path.substringAfter("$HOST_SEARCH/", "") if (encoded.isEmpty()) return null - TabsDestination.Search(searchQuery = URLDecoder.decode(encoded, StandardCharsets.UTF_8), tabId = newTabId) + TabsDestination.Search(searchQuery = formDecode(encoded), tabId = newTabId) } else -> null } } + +// application/x-www-form-urlencoded in UTF-8, as java.net.URLEncoder writes it and URLDecoder reads it (the links +// already shared must keep working): letters, digits and .-*_ as is, a space as +, any other byte as %XX +private fun formEncode(text: String): String = + buildString { + for (byte in text.encodeToByteArray()) { + val c = (byte.toInt() and 0xFF).toChar() + when { + c in 'a'..'z' || c in 'A'..'Z' || c in '0'..'9' || c in ".-*_" -> append(c) + c == ' ' -> append('+') + else -> append('%').append(HEX[(byte.toInt() shr 4) and 0xF]).append(HEX[byte.toInt() and 0xF]) + } + } + } + +private fun formDecode(text: String): String { + val bytes = ArrayList(text.length) + var i = 0 + while (i < text.length) { + val c = text[i] + when { + c == '+' -> bytes += ' '.code.toByte() + c == '%' && i + 2 <= text.lastIndex -> { + bytes += text.substring(i + 1, i + 3).toInt(16).toByte() + i += 2 + } + else -> bytes += c.toString().encodeToByteArray().toList() + } + i++ + } + return bytes.toByteArray().decodeToString() +} + +private const val HEX = "0123456789ABCDEF" diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/core/presentation/text/DiacriticsMode.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/core/presentation/text/DiacriticsMode.kt similarity index 96% rename from SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/core/presentation/text/DiacriticsMode.kt rename to searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/core/presentation/text/DiacriticsMode.kt index c63da053d..0b5236e91 100644 --- a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/core/presentation/text/DiacriticsMode.kt +++ b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/core/presentation/text/DiacriticsMode.kt @@ -35,7 +35,7 @@ private const val METEG = '\u05BD' private fun isTeamim(c: Char): Boolean = c.code in 0x0591..0x05AF /** True for nikud (vowel points, meteg included) and ta'amim (cantillation) characters. */ -internal fun isNikudOrTeamim(c: Char): Boolean = +fun isNikudOrTeamim(c: Char): Boolean = isTeamim(c) || (c.code in 0x05B0..0x05BD) || (c == '\u05C1') || diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/core/presentation/text/HebrewSearch.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/core/presentation/text/HebrewSearch.kt similarity index 96% rename from SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/core/presentation/text/HebrewSearch.kt rename to searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/core/presentation/text/HebrewSearch.kt index 8020849a3..25c6bf311 100644 --- a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/core/presentation/text/HebrewSearch.kt +++ b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/core/presentation/text/HebrewSearch.kt @@ -55,7 +55,7 @@ fun stripNikudTeamimWithMap( return out.toString() to arr } -internal fun replaceFinalsWithBase(text: String): String = +fun replaceFinalsWithBase(text: String): String = text .replace('\u05DA', '\u05DB') // ך -> כ .replace('\u05DD', '\u05DE') // ם -> מ @@ -63,7 +63,7 @@ internal fun replaceFinalsWithBase(text: String): String = .replace('\u05E3', '\u05E4') // ף -> פ .replace('\u05E5', '\u05E6') // ץ -> צ -internal fun normalizeQueryForHebrew(raw: String): String { +fun normalizeQueryForHebrew(raw: String): String { if (raw.isBlank()) return "" var s = raw.trim() // Remove biblical cantillation marks (teamim) U+0591–U+05AF @@ -80,7 +80,7 @@ internal fun normalizeQueryForHebrew(raw: String): String { return s } -internal fun mapToOrigIndex( +fun mapToOrigIndex( mapToOrig: IntArray, plainIndex: Int, ): Int { @@ -132,13 +132,13 @@ fun mapOriginalToStripped( * Find all diacritic-insensitive matches of [query] in [text], returning original * character index ranges [start, end) suitable for highlighting. */ -internal fun findAllMatchesOriginal( +fun findAllMatchesOriginal( text: String, query: String, ): List = findAllMatchesNormalized(text, normalizeQueryForHebrew(query)) /** [findAllMatchesOriginal] for a query already through [normalizeQueryForHebrew]: scanning many lines normalizes it once. */ -internal fun findAllMatchesNormalized( +fun findAllMatchesNormalized( text: String, q: String, ): List { diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/state/StateKeys.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/state/StateKeys.kt similarity index 100% rename from SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/state/StateKeys.kt rename to searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/state/StateKeys.kt diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/usecases/BookFindMatches.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/usecases/BookFindMatches.kt similarity index 96% rename from SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/usecases/BookFindMatches.kt rename to searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/usecases/BookFindMatches.kt index 146e092ac..00309754f 100644 --- a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/usecases/BookFindMatches.kt +++ b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/usecases/BookFindMatches.kt @@ -10,7 +10,7 @@ import io.github.kdroidfilter.seforimlibrary.search.SearchEngine import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.ensureActive import kotlinx.coroutines.withContext -import java.io.IOException +import kotlinx.io.IOException /** * The matches of the find-in-page query in a book, in reading order. Only each matching line's @@ -41,7 +41,7 @@ class BookFindMatches internal constructor( current: Match, forward: Boolean, ): Match? { - val i = lineIndices.binarySearch(current.lineIndex) + val i = lineIndices.asList().binarySearch(current.lineIndex) if (i < 0) return nextAfterLine(current.lineIndex, forward) val ordinal = current.ordinal + if (forward) 1 else -1 return when { @@ -60,7 +60,7 @@ class BookFindMatches internal constructor( forward: Boolean, ): Match? { if (lineIndices.isEmpty()) return null - val pos = lineIndices.binarySearch(fromLineIndex) + val pos = lineIndices.asList().binarySearch(fromLineIndex) return if (forward) { matchAt((if (pos >= 0) pos + 1 else -pos - 1).takeIf { it < lineIndices.size } ?: 0, 0) } else { @@ -71,7 +71,7 @@ class BookFindMatches internal constructor( /** Whether [match] is one of these matches: a match kept from older results may no longer be. */ operator fun contains(match: Match): Boolean { - val i = lineIndices.binarySearch(match.lineIndex) + val i = lineIndices.asList().binarySearch(match.lineIndex) return i >= 0 && lineIds[i] == match.lineId && match.ordinal in 0 until countIn(i) && diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchHomeViewModel.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchHomeViewModel.kt similarity index 94% rename from SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchHomeViewModel.kt rename to searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchHomeViewModel.kt index 7ffff89d3..970c5a057 100644 --- a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchHomeViewModel.kt +++ b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchHomeViewModel.kt @@ -7,7 +7,6 @@ import io.github.kdroidfilter.seforim.tabs.SearchScope import io.github.kdroidfilter.seforim.tabs.TabsDestination import io.github.kdroidfilter.seforimapp.core.coroutines.runSuspendCatching import io.github.kdroidfilter.seforimapp.core.deeplink.parseZayitDeepLink -import io.github.kdroidfilter.seforimapp.core.settings.AppSettings import io.github.kdroidfilter.seforimapp.features.search.domain.AuthorNames import io.github.kdroidfilter.seforimapp.features.search.domain.reference.ReferenceParser import io.github.kdroidfilter.seforimapp.features.search.domain.reference.ReferenceResolver @@ -17,12 +16,13 @@ import io.github.kdroidfilter.seforimapp.framework.search.LuceneLookupSearchServ import io.github.kdroidfilter.seforimapp.framework.search.LuceneLookupSearchService.AuthorHit import io.github.kdroidfilter.seforimapp.framework.search.MIN_BOOK_QUERY_LENGTH import io.github.kdroidfilter.seforimapp.framework.session.SearchPersistedState -import io.github.kdroidfilter.seforimapp.framework.session.TabPersistedStateStore +import io.github.kdroidfilter.seforimlibrary.cache.concurrentCache import io.github.kdroidfilter.seforimlibrary.core.models.Book import io.github.kdroidfilter.seforimlibrary.core.models.TocEntry import io.github.kdroidfilter.seforimlibrary.dao.repository.SeforimRepository import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.FlowPreview +import kotlinx.coroutines.IO import kotlinx.coroutines.async import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.coroutineScope @@ -40,7 +40,6 @@ import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import kotlinx.coroutines.withContext -import java.util.concurrent.ConcurrentHashMap /** * Navigation events emitted by SearchHomeViewModel. @@ -118,10 +117,10 @@ data class SearchHomeUiState( @OptIn(FlowPreview::class) class SearchHomeViewModel( - private val persistedStore: TabPersistedStateStore, + private val sessionStore: SearchSessionStore, private val repository: SeforimRepository, private val lookup: LuceneLookupSearchService, - private val appSettings: AppSettings, + private val appSettings: SearchSettings, // The home page's greeting needs the user's profile; a results tab's bar doesn't observeProfile: Boolean = true, ) : ViewModel() { @@ -135,7 +134,7 @@ class SearchHomeViewModel( private val referenceResolver = ReferenceResolver(RepositoryReferenceSource(repository)) // Author aliases by id, to tell which one a query was typed as; few authors have any - private val aliasesByAuthor = ConcurrentHashMap>() + private val aliasesByAuthor = concurrentCache>() private suspend fun authorAliases(authorId: Long): List = aliasesByAuthor[authorId] ?: runSuspendCatching { repository.getAuthorDetails(authorId)?.aliases.orEmpty() } @@ -155,11 +154,23 @@ class SearchHomeViewModel( private val maxBookPredictive = 120 // tighter ceiling to avoid heavy allocations private val maxTocPredictive = 300 // TOC suggestions stay bounded - // Lightweight, thread-safe LRU caches to avoid repeated DB hits when typing fast - private class LruCache( + // Lightweight LRU caches (guarded by their mutex) to avoid repeated DB hits when typing fast + private class LruCache( private val maxSize: Int, - ) : LinkedHashMap(maxSize, 0.75f, true) { - override fun removeEldestEntry(eldest: MutableMap.MutableEntry?): Boolean = size > maxSize + ) { + // Insertion order: a hit moves its entry to the end, so the first entry is the least recently used + private val map = LinkedHashMap() + + operator fun get(key: K): V? = map.remove(key)?.also { map[key] = it } + + operator fun set( + key: K, + value: V, + ) { + map.remove(key) + map[key] = value + if (map.size > maxSize) map.remove(map.keys.first()) + } } private val categoryPathCache = LruCache>(512) @@ -616,22 +627,20 @@ class SearchHomeViewModel( val selected = _uiState.value val scope = barScope() - persistedStore.update(currentTabId) { current -> - val nextSearch = - (current.search ?: SearchPersistedState()).withScope(scope).copy( - query = query, - globalExtended = selected.globalExtended, - selectedCategoryIds = emptySet(), - selectedBookIds = emptySet(), - selectedTocIds = emptySet(), - scrollIndex = 0, - scrollOffset = 0, - anchorId = -1L, - anchorIndex = 0, - snapshot = null, - breadcrumbs = emptyMap(), - ) - current.copy(search = nextSearch) + sessionStore.updateSearch(currentTabId) { current -> + (current ?: SearchPersistedState()).withScope(scope).copy( + query = query, + globalExtended = selected.globalExtended, + selectedCategoryIds = emptySet(), + selectedBookIds = emptySet(), + selectedTocIds = emptySet(), + scrollIndex = 0, + scrollOffset = 0, + anchorId = -1L, + anchorIndex = 0, + snapshot = null, + breadcrumbs = emptyMap(), + ) } // Emit navigation event - UI layer handles actual navigation @@ -664,9 +673,7 @@ class SearchHomeViewModel( } // Pre-seed minimal state so the BookContent shell can show a loader instead of flashing Home. - persistedStore.update(currentTabId) { current -> - current.copy(bookContent = current.bookContent.copy(selectedBookId = book.id)) - } + sessionStore.selectBook(currentTabId, book.id) // Emit navigation event - UI layer handles actual navigation _navigationEvents.send( @@ -683,9 +690,7 @@ class SearchHomeViewModel( book: Book, currentTabId: String, ) { - persistedStore.update(currentTabId) { current -> - current.copy(bookContent = current.bookContent.copy(selectedBookId = book.id)) - } + sessionStore.selectBook(currentTabId, book.id) _uiState.value = _uiState.value.copy(suggestionsVisible = false) _navigationEvents.send(SearchHomeNavigationEvent.NavigateToBookContent(bookId = book.id, tabId = currentTabId, lineId = null)) } @@ -706,9 +711,7 @@ class SearchHomeViewModel( jump: ResolvedReference, currentTabId: String, ) { - persistedStore.update(currentTabId) { current -> - current.copy(bookContent = current.bookContent.copy(selectedBookId = jump.book.id)) - } + sessionStore.selectBook(currentTabId, jump.book.id) _uiState.value = _uiState.value.copy(suggestionsVisible = false) _navigationEvents.send( SearchHomeNavigationEvent.NavigateToBookContent( diff --git a/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchHost.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchHost.kt new file mode 100644 index 000000000..05d73ae8a --- /dev/null +++ b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchHost.kt @@ -0,0 +1,100 @@ +package io.github.kdroidfilter.seforimapp.features.search + +import io.github.kdroidfilter.seforim.tabs.SearchScope +import io.github.kdroidfilter.seforim.tabs.TabsDestination +import io.github.kdroidfilter.seforimapp.framework.session.SearchPersistedState +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.StateFlow + +/* + * What the search screens ask of the app around them: the desktop's windows, tabs and session, a mobile app's + * navigation and storage. Each app implements these; the view models know nothing else of it. + */ + +/** The tabs (or screens) the search opens and watches. */ +interface SearchTabs { + /** Opens [destination] in a new tab, from the tab [fromTabId]. */ + fun openTab( + fromTabId: String, + destination: TabsDestination, + ) + + /** Shows [destination] in the tab [tabId] instead of what it shows. */ + fun replace( + tabId: String, + destination: TabsDestination, + ) + + /** Whether the tab [tabId] still exists, as it changes. */ + fun tabExistsFlow(tabId: String): Flow + + fun isTabOpen(tabId: String): Boolean +} + +/** What the tabs keep across sessions, as far as the search is concerned. */ +interface SearchSessionStore { + /** The search state of the tab [tabId], null when it has none. */ + fun search(tabId: String): SearchPersistedState? + + /** Replaces the search state of the tab [tabId] by [transform]'s (null: the tab has none any more). */ + fun updateSearch( + tabId: String, + transform: (SearchPersistedState?) -> SearchPersistedState?, + ) + + /** Makes [bookId] the book the tab [tabId] shows. */ + fun selectBook( + tabId: String, + bookId: Long, + ) + + /** + * Prepares the tab [tabId] to open [bookId] at [lineId]: the line selected and anchored, the table of contents + * shown, as a found line opens. + */ + fun openBookAtLine( + tabId: String, + bookId: Long, + lineId: Long, + ) +} + +/** The app settings the search reads and sets. */ +interface SearchSettings { + val textSizeFlow: StateFlow + + fun getTextSize(): Float + + /** Pre-fills the find-in-page bar of the tab [tabId] with [q]. */ + fun setFindQuery( + tabId: String, + q: String, + ) + + fun setFindSmartMode( + tabId: String, + enabled: Boolean, + ) + + fun openFindBar(tabId: String) + + val userFirstNameFlow: StateFlow + val userLastNameFlow: StateFlow + val userCommunityCodeFlow: StateFlow + + companion object { + /** The text size before the user changes it. */ + const val DEFAULT_TEXT_SIZE = 16f + } +} + +/** The history of what was searched. */ +interface SearchHistory { + suspend fun recordSearchVisit( + query: String, + scope: SearchScope, + globalExtended: Boolean, + scopeLabel: String?, + timestamp: Long, + ) +} diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchResultViewModel.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchResultViewModel.kt similarity index 95% rename from SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchResultViewModel.kt rename to searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchResultViewModel.kt index 328ee5c76..bac687572 100644 --- a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchResultViewModel.kt +++ b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchResultViewModel.kt @@ -16,8 +16,6 @@ import dev.zacsweers.metrox.viewmodel.ViewModelAssistedFactory import dev.zacsweers.metrox.viewmodel.ViewModelAssistedFactoryKey import io.github.kdroidfilter.seforim.tabs.* import io.github.kdroidfilter.seforimapp.core.coroutines.runSuspendCatching -import io.github.kdroidfilter.seforimapp.core.history.HistoryStore -import io.github.kdroidfilter.seforimapp.core.settings.AppSettings import io.github.kdroidfilter.seforimapp.features.bookcontent.state.StateKeys import io.github.kdroidfilter.seforimapp.features.search.domain.AuthorNames import io.github.kdroidfilter.seforimapp.features.search.domain.BookDetailsLoader @@ -33,11 +31,9 @@ import io.github.kdroidfilter.seforimapp.features.search.domain.SearchStatePersi import io.github.kdroidfilter.seforimapp.features.search.domain.SearchTocUseCase import io.github.kdroidfilter.seforimapp.features.search.domain.TocLineIndex import io.github.kdroidfilter.seforimapp.features.search.domain.TocTree -import io.github.kdroidfilter.seforimapp.framework.desktop.DesktopManager import io.github.kdroidfilter.seforimapp.framework.di.AppScope import io.github.kdroidfilter.seforimapp.framework.search.LuceneLookupSearchService import io.github.kdroidfilter.seforimapp.framework.session.SearchPersistedState -import io.github.kdroidfilter.seforimapp.framework.session.TabPersistedStateStore import io.github.kdroidfilter.seforimlibrary.core.models.Book import io.github.kdroidfilter.seforimlibrary.core.models.Category import io.github.kdroidfilter.seforimlibrary.core.models.Line @@ -59,8 +55,8 @@ import kotlinx.coroutines.flow.* import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import kotlinx.serialization.json.Json -import java.util.UUID import kotlin.collections.ArrayDeque +import kotlin.time.Clock private const val LAZY_PAGE_SIZE = 25 @@ -87,7 +83,7 @@ data class SearchUiState( val anchorId: Long = -1L, val anchorIndex: Int = 0, val scrollToAnchorTimestamp: Long = 0L, - val textSize: Float = AppSettings.DEFAULT_TEXT_SIZE, + val textSize: Float = SearchSettings.DEFAULT_TEXT_SIZE, val hasMore: Boolean = false, val isLoadingMore: Boolean = false, val progressCurrent: Int = 0, @@ -97,13 +93,13 @@ data class SearchUiState( @AssistedInject class SearchResultViewModel( @Assisted savedStateHandle: SavedStateHandle, - private val persistedStore: TabPersistedStateStore, + private val sessionStore: SearchSessionStore, private val repository: SeforimRepository, private val lucene: SearchEngine, private val titleUpdateManager: TabTitleUpdateManager, - private val desktopManager: DesktopManager, - private val historyStore: HistoryStore, - private val appSettings: AppSettings, + private val tabs: SearchTabs, + private val historyStore: SearchHistory, + private val appSettings: SearchSettings, lookup: LuceneLookupSearchService, ) : ViewModel() { @AssistedFactory @@ -117,13 +113,10 @@ class SearchResultViewModel( internal val tabId: String = savedStateHandle.get(StateKeys.TAB_ID) ?: "" - private fun persistedSearchState(): SearchPersistedState = persistedStore.get(tabId)?.search ?: SearchPersistedState() + private fun persistedSearchState(): SearchPersistedState = sessionStore.search(tabId) ?: SearchPersistedState() private fun updatePersistedSearch(transform: (SearchPersistedState) -> SearchPersistedState) { - persistedStore.update(tabId) { current -> - val next = transform(current.search ?: SearchPersistedState()) - current.copy(search = next) - } + sessionStore.updateSearch(tabId) { current -> transform(current ?: SearchPersistedState()) } } private fun seedNavigationScope(savedStateHandle: SavedStateHandle) { @@ -440,13 +433,11 @@ class SearchResultViewModel( .drop(1) // ignore initial state on first subscription .flatMapLatest { // Show overlay until visible results emission changes either size or identity - val initial = Pair(visibleResultsFlow.value.size, System.identityHashCode(visibleResultsFlow.value)) + val initial = visibleResultsFlow.value flow { emit(true) visibleResultsFlow - .map { Pair(it.size, System.identityHashCode(it)) } - .distinctUntilChanged() - .filter { it != initial } + .filter { it !== initial } .first() emit(false) } @@ -501,14 +492,12 @@ class SearchResultViewModel( bookId: Long, lineId: Long, ) { - desktopManager.tabsViewModelFor(tabId)?.openTab( - TabsDestination.BookContent(bookId = bookId, tabId = UUID.randomUUID().toString(), lineId = lineId), - ) + tabs.openTab(tabId, TabsDestination.BookContent(bookId = bookId, tabId = newTabId(), lineId = lineId)) } /** Opens an author's page from the panel, in a new tab. */ fun openAuthor(authorId: Long) { - desktopManager.tabsViewModelFor(tabId)?.openTab(TabsDestination.Author(tabId = UUID.randomUUID().toString(), authorId = authorId)) + tabs.openTab(tabId, TabsDestination.Author(tabId = newTabId(), authorId = authorId)) } private val _searchTree = MutableStateFlow>(persistentListOf()) @@ -787,7 +776,7 @@ class SearchResultViewModel( progressCurrent = cached.results.size, progressTotal = cached.totalHits.takeIf { it > 0 } ?: cached.results.size.toLong(), // trigger scroll restoration once items are present - scrollToAnchorTimestamp = System.currentTimeMillis(), + scrollToAnchorTimestamp = Clock.System.now().toEpochMilliseconds(), ) // Re-open a Lucene session if there are more results to load if (cached.hasMore && initialQuery.isNotBlank()) { @@ -881,7 +870,7 @@ class SearchResultViewModel( viewModelScope.launch { // Observe all windows' tabs and cancel search if this tab gets closed everywhere - desktopManager.tabExistsFlow(tabId).collect { exists -> + tabs.tabExistsFlow(tabId).collect { exists -> if (!exists) { // Tab was closed; stop work. cancelSearch() @@ -933,7 +922,7 @@ class SearchResultViewModel( val persisted = persistedSearchState() val scope = persisted.scope viewModelScope.launch { - historyStore.recordSearchVisit(q, scope, persisted.globalExtended, scopeLabel(scope), System.currentTimeMillis()) + historyStore.recordSearchVisit(q, scope, persisted.globalExtended, scopeLabel(scope), Clock.System.now().toEpochMilliseconds()) } // New search: clear any previous streaming job and reset scroll/anchor state currentJob?.cancel() @@ -956,7 +945,7 @@ class SearchResultViewModel( scrollOffset = 0, anchorId = -1L, anchorIndex = 0, - scrollToAnchorTimestamp = System.currentTimeMillis(), + scrollToAnchorTimestamp = Clock.System.now().toEpochMilliseconds(), ) currentJob = viewModelScope.launch(Dispatchers.Default) { @@ -1123,12 +1112,10 @@ class SearchResultViewModel( // but never block indefinitely (important when final results are empty and identical // to the pre-stream empty list reference). runSuspendCatching { - val initial = Pair(visibleResultsFlow.value.size, System.identityHashCode(visibleResultsFlow.value)) + val initial = visibleResultsFlow.value withTimeoutOrNull(300) { visibleResultsFlow - .map { Pair(it.size, System.identityHashCode(it)) } - .distinctUntilChanged() - .filter { it != initial } + .filter { it !== initial } .first() } } @@ -1282,7 +1269,7 @@ class SearchResultViewModel( // without re-searching. val stillExists = runCatching { - desktopManager.isTabOpen(tabId) + tabs.isTabOpen(tabId) }.getOrDefault(false) if (stillExists) { val snap = buildSnapshot(uiState.value.results) @@ -1357,7 +1344,7 @@ class SearchResultViewModel( scopeTocId = null, scrollIndex = 0, scrollOffset = 0, - scrollToAnchorTimestamp = System.currentTimeMillis(), + scrollToAnchorTimestamp = Clock.System.now().toEpochMilliseconds(), ) if (book != null && currentTocBookId != book.id) { val tree = runSuspendCatching { buildTocTreeForBook(book.id) }.getOrNull() @@ -1398,7 +1385,7 @@ class SearchResultViewModel( scopeTocId = tocId, scrollIndex = 0, scrollOffset = 0, - scrollToAnchorTimestamp = System.currentTimeMillis(), + scrollToAnchorTimestamp = Clock.System.now().toEpochMilliseconds(), ) if (scopeBook != null && currentTocBookId != scopeBook.id) { val tree = runSuspendCatching { buildTocTreeForBook(scopeBook.id) }.getOrNull() @@ -1846,7 +1833,7 @@ class SearchResultViewModel( progressCurrent = results.size, scrollIndex = 0, scrollOffset = 0, - scrollToAnchorTimestamp = System.currentTimeMillis(), + scrollToAnchorTimestamp = Clock.System.now().toEpochMilliseconds(), ) refineSnippets(session, firstPage.hits) } finally { @@ -1904,41 +1891,20 @@ class SearchResultViewModel( fun openResult(result: SearchResult) { viewModelScope.launch { - val newTabId = UUID.randomUUID().toString() + val bookTabId = newTabId() // Seed persisted state so the new tab can restore scroll/anchor deterministically. - persistedStore.update(newTabId) { current -> - current.copy( - bookContent = - current.bookContent.copy( - selectedBookId = result.bookId, - selectedLineIds = setOf(result.lineId), - primarySelectedLineId = result.lineId, - isTocEntrySelection = false, - contentAnchorLineId = result.lineId, - contentAnchorIndex = 0, - contentScrollIndex = 0, - contentScrollOffset = 0, - isTocVisible = true, - ), - search = null, - ) - } + sessionStore.openBookAtLine(bookTabId, result.bookId, result.lineId) + sessionStore.updateSearch(bookTabId) { null } // Pre-configure find-in-page with the search query and smart mode enabled val searchQuery = _uiState.value.query if (searchQuery.length >= 2) { - appSettings.setFindQuery(newTabId, searchQuery) - appSettings.setFindSmartMode(newTabId, true) - appSettings.openFindBar(newTabId) + appSettings.setFindQuery(bookTabId, searchQuery) + appSettings.setFindSmartMode(bookTabId, true) + appSettings.openFindBar(bookTabId) } - desktopManager.tabsViewModelFor(tabId)?.openTab( - TabsDestination.BookContent( - bookId = result.bookId, - tabId = newTabId, - lineId = result.lineId, - ), - ) + tabs.openTab(tabId, TabsDestination.BookContent(bookId = result.bookId, tabId = bookTabId, lineId = result.lineId)) } } @@ -1956,22 +1922,7 @@ class SearchResultViewModel( return } viewModelScope.launch { - persistedStore.update(tabId) { current -> - current.copy( - bookContent = - current.bookContent.copy( - selectedBookId = result.bookId, - selectedLineIds = setOf(result.lineId), - primarySelectedLineId = result.lineId, - isTocEntrySelection = false, - contentAnchorLineId = result.lineId, - contentAnchorIndex = 0, - contentScrollIndex = 0, - contentScrollOffset = 0, - isTocVisible = true, - ), - ) - } + sessionStore.openBookAtLine(tabId, result.bookId, result.lineId) // Pre-configure find-in-page with the search query and smart mode enabled val searchQuery = _uiState.value.query @@ -1982,7 +1933,8 @@ class SearchResultViewModel( } // Swap current tab destination to BookContent while preserving tabId - desktopManager.tabsViewModelFor(tabId)?.replaceCurrentTabDestination( + tabs.replace( + tabId, TabsDestination.BookContent( bookId = result.bookId, tabId = tabId, @@ -2070,4 +2022,4 @@ private const val CONTEXT_MAX_LINES = 30 private const val CONTEXT_CHARS = 700 // An HTML tag, to read a line or snippet as plain text -internal val HTML_TAG = Regex("<[^>]+>") +val HTML_TAG = Regex("<[^>]+>") diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchTabCache.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchTabCache.kt similarity index 100% rename from SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchTabCache.kt rename to searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchTabCache.kt diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/AuthorNames.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/AuthorNames.kt similarity index 100% rename from SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/AuthorNames.kt rename to searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/AuthorNames.kt diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/BookDetailsLoader.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/BookDetailsLoader.kt similarity index 91% rename from SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/BookDetailsLoader.kt rename to searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/BookDetailsLoader.kt index 454b0cf4f..2e8fab501 100644 --- a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/BookDetailsLoader.kt +++ b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/BookDetailsLoader.kt @@ -1,10 +1,10 @@ package io.github.kdroidfilter.seforimapp.features.search.domain +import co.touchlab.kermit.Logger import io.github.kdroidfilter.seforimapp.core.coroutines.runSuspendCatching -import io.github.kdroidfilter.seforimapp.logger.warnln +import io.github.kdroidfilter.seforimlibrary.cache.concurrentCache import io.github.kdroidfilter.seforimlibrary.core.models.Book import io.github.kdroidfilter.seforimlibrary.dao.repository.SeforimRepository -import java.util.concurrent.ConcurrentHashMap /** * A book's details as the book-details pane shows them (categories, authors with their years, main @@ -13,12 +13,12 @@ import java.util.concurrent.ConcurrentHashMap class BookDetailsLoader( private val repository: SeforimRepository, ) { - private val cache = ConcurrentHashMap() + private val cache = concurrentCache() /** A book with its categories, authors and main parts, once per book; null (logged) when it fails. */ suspend fun load(bookId: Long): SearchEntity.BookEntity? = cache[bookId] ?: runSuspendCatching { describe(bookId) } - .onFailure { warnln(it) { "Book details of $bookId failed to load" } } + .onFailure { Logger.withTag("BookDetailsLoader").w(it) { "Book details of $bookId failed to load" } } .getOrNull() ?.also { cache[bookId] = it } diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/BuildSearchTreeUseCase.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/BuildSearchTreeUseCase.kt similarity index 100% rename from SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/BuildSearchTreeUseCase.kt rename to searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/BuildSearchTreeUseCase.kt diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/CategoryNavigationUseCase.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/CategoryNavigationUseCase.kt similarity index 100% rename from SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/CategoryNavigationUseCase.kt rename to searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/CategoryNavigationUseCase.kt diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/ExecuteSearchUseCase.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/ExecuteSearchUseCase.kt similarity index 100% rename from SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/ExecuteSearchUseCase.kt rename to searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/ExecuteSearchUseCase.kt diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/GetBreadcrumbPiecesUseCase.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/GetBreadcrumbPiecesUseCase.kt similarity index 100% rename from SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/GetBreadcrumbPiecesUseCase.kt rename to searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/GetBreadcrumbPiecesUseCase.kt diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/ResultsIndexingUseCase.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/ResultsIndexingUseCase.kt similarity index 94% rename from SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/ResultsIndexingUseCase.kt rename to searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/ResultsIndexingUseCase.kt index fe0138438..2abd4412e 100644 --- a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/ResultsIndexingUseCase.kt +++ b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/ResultsIndexingUseCase.kt @@ -10,12 +10,12 @@ import kotlinx.coroutines.sync.withLock /** * Index structure for fast filtering of search results. * - * @property identity Identity hash code of the results list this index was built from + * @property identity the results list this index was built from, compared by identity (===) * @property bookToIndices Map from book ID to sorted array of result indices * @property lineIdToIndex Map from line ID to result index */ data class ResultsIndex( - val identity: Int, + val identity: Any, val bookToIndices: Map, val lineIdToIndex: Map, ) @@ -31,16 +31,16 @@ data class ResultsIndex( * - Parallel processing for large result sets * * The use case is stateful - it maintains a cached index that is invalidated when - * the results list changes (detected by identity hash code). + * the results list changes (detected by identity). */ class ResultsIndexingUseCase { private val indexMutex = Mutex() - @Volatile + @kotlin.concurrent.Volatile private var cachedIndex: ResultsIndex? = null // TOC indices cache - invalidated when results change - private var tocIndicesIdentity: Int = -1 + private var tocIndicesIdentity: Any? = null private val tocIndicesCache: MutableMap = mutableMapOf() /** @@ -51,14 +51,14 @@ class ResultsIndexingUseCase { * @return The built index */ suspend fun ensureIndex(results: List): ResultsIndex { - val identity = System.identityHashCode(results) + val identity: Any = results val current = cachedIndex - if (current != null && current.identity == identity) return current + if (current != null && current.identity === identity) return current indexMutex.withLock { // Double-check after acquiring lock val cur2 = cachedIndex - if (cur2 != null && cur2.identity == identity) return cur2 + if (cur2 != null && cur2.identity === identity) return cur2 val lineMap = HashMap(results.size * 2) @@ -121,7 +121,7 @@ class ResultsIndexingUseCase { index: ResultsIndex, ): IntArray { // Invalidate cache if results changed - if (tocIndicesIdentity != index.identity) { + if (tocIndicesIdentity !== index.identity) { tocIndicesIdentity = index.identity tocIndicesCache.clear() } diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/SearchEntityFinder.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/SearchEntityFinder.kt similarity index 100% rename from SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/SearchEntityFinder.kt rename to searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/SearchEntityFinder.kt diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/SearchKey.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/SearchKey.kt similarity index 92% rename from SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/SearchKey.kt rename to searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/SearchKey.kt index 3e33058d9..7f090792d 100644 --- a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/SearchKey.kt +++ b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/SearchKey.kt @@ -5,7 +5,7 @@ private val QUOTES = Regex("[\"'״׳]") private val SPACES = Regex("\\s+") /** A name as typed, to compare: without nikud, quotes and extra spaces (`שו"ע` = `שוע`). */ -internal fun String.searchKey(): String = +fun String.searchKey(): String = // The maqaf joins words (שולחן־ערוך): a space, before the nikud range (which holds it) goes replace('\u05BE', ' ') .replace(NIKUD, "") diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/SearchStatePersistenceUseCase.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/SearchStatePersistenceUseCase.kt similarity index 100% rename from SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/SearchStatePersistenceUseCase.kt rename to searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/SearchStatePersistenceUseCase.kt diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/SearchTocUseCase.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/SearchTocUseCase.kt similarity index 98% rename from SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/SearchTocUseCase.kt rename to searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/SearchTocUseCase.kt index ae75f8ca4..a33652792 100644 --- a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/SearchTocUseCase.kt +++ b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/SearchTocUseCase.kt @@ -7,7 +7,6 @@ import kotlinx.collections.immutable.ImmutableList import kotlinx.collections.immutable.toImmutableList import kotlinx.coroutines.currentCoroutineContext import kotlinx.coroutines.ensureActive -import java.util.Arrays /** * Use case for TOC-related operations in search. @@ -228,10 +227,10 @@ data class TocLineIndex( val out = LongArray(total) var pos = 0 for (a in arrays) { - System.arraycopy(a, 0, out, pos, a.size) + a.copyInto(out, pos) pos += a.size } - Arrays.parallelSort(out) + out.sort() var write = 0 var i = 0 while (i < out.size) { diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/reference/ReferenceParser.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/reference/ReferenceParser.kt similarity index 100% rename from SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/reference/ReferenceParser.kt rename to searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/reference/ReferenceParser.kt diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/reference/ReferenceResolver.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/reference/ReferenceResolver.kt similarity index 100% rename from SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/reference/ReferenceResolver.kt rename to searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/reference/ReferenceResolver.kt diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/reference/RepositoryReferenceSource.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/reference/RepositoryReferenceSource.kt similarity index 81% rename from SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/reference/RepositoryReferenceSource.kt rename to searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/reference/RepositoryReferenceSource.kt index 6c29052cd..8792fac86 100644 --- a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/reference/RepositoryReferenceSource.kt +++ b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/reference/RepositoryReferenceSource.kt @@ -3,6 +3,8 @@ package io.github.kdroidfilter.seforimapp.features.search.domain.reference import io.github.kdroidfilter.seforimlibrary.core.models.Book import io.github.kdroidfilter.seforimlibrary.core.models.Line import io.github.kdroidfilter.seforimlibrary.dao.repository.SeforimRepository +import kotlinx.atomicfu.locks.SynchronizedObject +import kotlinx.atomicfu.locks.synchronized /** * [ReferenceSource] over the seforim database. The TOCs of the last books looked up are kept, @@ -43,18 +45,22 @@ class RepositoryReferenceSource( private class LruCache( private val maxSize: Int, ) { - private val map = - object : LinkedHashMap(maxSize, 0.75f, true) { - override fun removeEldestEntry(eldest: MutableMap.MutableEntry?): Boolean = size > maxSize - } + private val lock = SynchronizedObject() + + // Insertion order: a hit moves its entry to the end, so the first entry is the least recently used + private val map = LinkedHashMap() suspend fun getOrPut( key: K, load: suspend () -> V, ): V { - synchronized(map) { map[key] }?.let { return it } + synchronized(lock) { map.remove(key)?.also { map[key] = it } }?.let { return it } val value = load() - synchronized(map) { map[key] = value } + synchronized(lock) { + map.remove(key) + map[key] = value + if (map.size > maxSize) map.remove(map.keys.first()) + } return value } } diff --git a/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/SemanticSearchFeature.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/SemanticSearchFeature.kt new file mode 100644 index 000000000..3a16aceda --- /dev/null +++ b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/SemanticSearchFeature.kt @@ -0,0 +1,37 @@ +package io.github.kdroidfilter.seforimapp.features.search.semantic + +import io.github.kdroidfilter.seforimlibrary.search.LineHit +import io.github.kdroidfilter.seforimlibrary.search.SearchEngine +import io.github.kdroidfilter.seforimlibrary.search.index.LuceneIndex + +/** + * The files of the semantic search's model: the ONNX model, its tokenizer and its static distillation. The desktop's + * semantic package carries them; a mobile app downloads them with the data and passes their paths. + */ +data class SemanticModels( + val model: String, + val tokenizer: String, + val staticTable: String, +) + +/** + * The semantic search (embedding model + dense search), in Zayit's official builds only (open core): its engine is in + * addons/semantic/enabled, built with the SeforimEmbedding package; a community build compiles addons/semantic/disabled + * instead, where [installedSemanticSearch] is null and the search stays lexical. + */ +interface SemanticSearchFeature { + /** The model the package carries (the desktop's), null where the app provides it (mobile). */ + val bundledModels: SemanticModels? + + /** + * Fuses [lexical] with dense search over the vectors of [index] (the fused Lucene index), with the model of + * [models] ([bundledModels] when null); it degrades to [lexical] when the model or the vectors are missing. + * [resolveLine] turns a dense-only hit into a full [LineHit]. + */ + fun hybrid( + lexical: SearchEngine, + index: LuceneIndex, + models: SemanticModels? = null, + resolveLine: suspend (lineId: Long, query: String) -> LineHit?, + ): SearchEngine +} diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/di/AppScope.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/framework/di/AppScope.kt similarity index 100% rename from SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/di/AppScope.kt rename to searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/framework/di/AppScope.kt diff --git a/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/AcronymFrequencyCache.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/AcronymFrequencyCache.kt new file mode 100644 index 000000000..a3a8a93b8 --- /dev/null +++ b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/AcronymFrequencyCache.kt @@ -0,0 +1,117 @@ +package io.github.kdroidfilter.seforimapp.framework.search + +import co.touchlab.kermit.Logger +import io.github.kdroidfilter.seforimlibrary.cache.concurrentCache +import io.github.kdroidfilter.seforimlibrary.search.SqliteReader +import io.github.kdroidfilter.seforimlibrary.search.openReadOnlySqlite + +/** + * Cache for acronym frequency data from seforim.db. + * Provides fast lookup of how many books use each acronym. + * + * Uses the book_acronym table from seforim.db - no separate database needed! + */ +class AcronymFrequencyCache( + private val databasePath: () -> String, +) { + private val logger = Logger.withTag("AcronymFrequencyCache") + + private val db: SqliteReader by lazy { + val dbPath = databasePath() + logger.i { "Loading acronym data from seforim.db: $dbPath" } + openReadOnlySqlite(dbPath) + } + + // Cache: acronym → frequency (number of books using it) + private val frequencyCache = concurrentCache() + + // Cache: acronym → list of book titles + private val bookTitlesCache = concurrentCache>() + + // Cache: book title → list of acronyms + private val titleToAcronymsCache = concurrentCache>() + + init { + // Pre-load common acronyms into cache + preloadCache() + } + + /** + * Get the frequency (number of books) for an acronym. + */ + fun getFrequency(acronym: String): Int = frequencyCache[acronym] ?: queryFrequency(acronym).also { frequencyCache[acronym] = it } + + /** + * Get all book titles that use this acronym. + */ + fun getBookTitles(acronym: String): List = + bookTitlesCache[acronym] ?: queryBookTitles(acronym).also { bookTitlesCache[acronym] = it } + + /** + * Check if an acronym is unique (used by only one book). + */ + fun isUnique(acronym: String): Boolean = getFrequency(acronym) == 1 + + /** + * Get all acronyms for a given book title. + */ + fun getAcronymsForBook(bookTitle: String): List = + titleToAcronymsCache[bookTitle] ?: queryAcronymsForTitle(bookTitle).also { titleToAcronymsCache[bookTitle] = it } + + private fun queryFrequency(acronym: String): Int = + db + .query("SELECT COUNT(DISTINCT bookId) as frequency FROM book_acronym WHERE term = ?", listOf(acronym)) + .firstOrNull() + ?.get("frequency") + ?.toIntOrNull() ?: 0 + + private fun queryBookTitles(acronym: String): List = + db + .query( + """ + SELECT DISTINCT b.title + FROM book_acronym ba + JOIN book b ON ba.bookId = b.id + WHERE ba.term = ? + """.trimIndent(), + listOf(acronym), + ).mapNotNull { it["title"] } + + private fun queryAcronymsForTitle(bookTitle: String): List = + db + .query( + """ + SELECT DISTINCT ba.term + FROM book b + JOIN book_acronym ba ON b.id = ba.bookId + WHERE b.title = ? + """.trimIndent(), + listOf(bookTitle), + ).mapNotNull { it["term"] } + + private fun preloadCache() { + // Pre-load top 100 most common acronyms + val rows = + db.query( + """ + SELECT + term, + COUNT(DISTINCT bookId) as frequency + FROM book_acronym + GROUP BY term + ORDER BY frequency DESC + LIMIT 100 + """.trimIndent(), + ) + for (row in rows) { + val acronym = row["term"] ?: continue + frequencyCache[acronym] = row["frequency"]?.toIntOrNull() ?: continue + } + + logger.i { "Pre-loaded ${rows.size} acronyms into cache from seforim.db" } + } + + fun close() { + db.close() + } +} diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/LuceneLookupSearchService.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/LuceneLookupSearchService.kt similarity index 77% rename from SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/LuceneLookupSearchService.kt rename to searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/LuceneLookupSearchService.kt index d6d91283c..e9d00391e 100644 --- a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/LuceneLookupSearchService.kt +++ b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/LuceneLookupSearchService.kt @@ -2,65 +2,30 @@ package io.github.kdroidfilter.seforimapp.framework.search import io.github.kdroidfilter.seforimapp.features.search.domain.AuthorNames import io.github.kdroidfilter.seforimlibrary.core.models.Book -import org.apache.lucene.analysis.Analyzer -import org.apache.lucene.analysis.standard.StandardAnalyzer -import org.apache.lucene.index.IndexReader -import org.apache.lucene.index.Term -import org.apache.lucene.search.BooleanClause -import org.apache.lucene.search.BooleanQuery -import org.apache.lucene.search.IndexSearcher -import org.apache.lucene.search.LRUQueryCache -import org.apache.lucene.search.PrefixQuery -import org.apache.lucene.search.SearcherFactory -import org.apache.lucene.search.SearcherManager -import org.apache.lucene.search.Sort -import org.apache.lucene.search.SortField -import org.apache.lucene.search.TermQuery -import org.apache.lucene.search.UsageTrackingQueryCachingPolicy -import org.apache.lucene.store.FSDirectory -import org.apache.lucene.store.NIOFSDirectory -import java.nio.file.Path - +import io.github.kdroidfilter.seforimlibrary.search.index.BoolBuilder +import io.github.kdroidfilter.seforimlibrary.search.index.IndexDoc +import io.github.kdroidfilter.seforimlibrary.search.index.IndexQuery +import io.github.kdroidfilter.seforimlibrary.search.index.LuceneIndex +import io.github.kdroidfilter.seforimlibrary.search.index.Occur +import io.github.kdroidfilter.seforimlibrary.search.index.SortField +import io.github.kdroidfilter.seforimlibrary.search.index.SortType + +/** + * Book, author and table-of-contents lookup over the lookup index (seforim.db.lookup.lucene), with acronym-aware + * ranking of the books: [index] is that index (opened with its query cache: [openLuceneIndex]), read on a JVM like the + * others. + */ class LuceneLookupSearchService( - indexDir: Path, - private val analyzer: Analyzer = StandardAnalyzer(), + private val index: LuceneIndex, private val acronymCache: AcronymFrequencyCache? = null, ) { - // Open Lucene directory lazily to avoid any I/O at app startup. - // GraalVM native image does not support MMapDirectory (Panama foreign downcalls), use NIOFSDirectory instead. - private val dir by lazy { - if (System.getProperty("org.graalvm.nativeimage.imagecode") != null) { - NIOFSDirectory(indexDir) - } else { - FSDirectory.open(indexDir) - } - } - - // Cache searchers; refresh explicitly when index changes - private val searcherManager by lazy { - SearcherManager( - dir, - object : SearcherFactory() { - override fun newSearcher( - reader: IndexReader, - previousReader: IndexReader?, - ): IndexSearcher { - val s = IndexSearcher(reader) - s.setQueryCache(LRUQueryCache(1024, 64L * 1024 * 1024)) // up to 64MiB cache - s.setQueryCachingPolicy(UsageTrackingQueryCachingPolicy()) - return s - } - }, - ) - } - - private val authorSort = Sort(SortField("book_count", SortField.Type.LONG, true)) + private val authorSort = listOf(SortField("book_count", SortType.LONG, reverse = true)) private val bookSort = - Sort( - SortField("is_base_book", SortField.Type.INT, true), - SortField("order_index", SortField.Type.INT), - SortField("book_id", SortField.Type.LONG), + listOf( + SortField("is_base_book", SortType.INT, reverse = true), + SortField("order_index", SortType.INT), + SortField("book_id", SortType.LONG), ) data class TocHit( @@ -113,19 +78,23 @@ class LuceneLookupSearchService( } fun close() { - kotlin.runCatching { searcherManager.close() } - kotlin.runCatching { dir.close() } + runCatching { index.close() } } - private inline fun withSearcher(block: (IndexSearcher) -> T): T { - // Index is static during app run; refresh is cheap if no changes - searcherManager.maybeRefresh() - val searcher = searcherManager.acquire() - return try { - block(searcher) - } finally { - searcherManager.release(searcher) - } + // Documents of [type] whose words start with each of [tokens], sorted by [sort] + private fun prefixSearch( + type: String, + tokens: List, + limit: Int, + sort: List, + fields: Set, + ): List { + val query = + BoolBuilder() + .add(IndexQuery.Term("type", type), Occur.FILTER) + .apply { tokens.forEach { t -> add(IndexQuery.Prefix("q", t), Occur.MUST) } } + .build() + return index.search(query, limit, fields, sort).docs } fun searchBooksPrefix( @@ -136,22 +105,14 @@ class LuceneLookupSearchService( if (q.isBlank()) return emptyList() val tokens = q.split("\\s+".toRegex()).map { it.trim() }.filter { it.isNotEmpty() } if (tokens.isEmpty()) return emptyList() - return withSearcher { searcher -> - val b = BooleanQuery.Builder() - b.add(TermQuery(Term("type", "book")), BooleanClause.Occur.FILTER) - tokens.forEach { t -> b.add(PrefixQuery(Term("q", t)), BooleanClause.Occur.MUST) } - val top = searcher.search(b.build(), limit, bookSort) - val stored = searcher.storedFields() - top.scoreDocs.map { sd -> - val doc = stored.document(sd.doc) - BookHit( - id = doc.getField("book_id").numericValue().toLong(), - categoryId = doc.getField("category_id").numericValue().toLong(), - title = doc.getField("book_title").stringValue(), - isBaseBook = doc.getField("is_base_book")?.numericValue()?.toInt() == 1, - orderIndex = doc.getField("order_index")?.numericValue()?.toInt() ?: Int.MAX_VALUE, - ) - } + return prefixSearch("book", tokens, limit, bookSort, BOOK_FIELDS).map { doc -> + BookHit( + id = doc.longs.getValue("book_id"), + categoryId = doc.longs.getValue("category_id"), + title = doc.strings.getValue("book_title"), + isBaseBook = doc.longs["is_base_book"]?.toInt() == 1, + orderIndex = doc.longs["order_index"]?.toInt() ?: Int.MAX_VALUE, + ) } } @@ -190,20 +151,12 @@ class LuceneLookupSearchService( ): List { val tokens = normalized.split("\\s+".toRegex()).filter { it.isNotEmpty() } if (tokens.isEmpty()) return emptyList() - return withSearcher { searcher -> - val query = BooleanQuery.Builder() - query.add(TermQuery(Term("type", "author")), BooleanClause.Occur.FILTER) - tokens.forEach { t -> query.add(PrefixQuery(Term("q", t)), BooleanClause.Occur.MUST) } - val top = searcher.search(query.build(), limit, authorSort) - val stored = searcher.storedFields() - top.scoreDocs.map { sd -> - val doc = stored.document(sd.doc) - AuthorHit( - id = doc.getField("author_id").numericValue().toLong(), - name = doc.getField("author_name").stringValue(), - bookCount = doc.getField("book_count").numericValue().toInt(), - ) - } + return prefixSearch("author", tokens, limit, authorSort, AUTHOR_FIELDS).map { doc -> + AuthorHit( + id = doc.longs.getValue("author_id"), + name = doc.strings.getValue("author_name"), + bookCount = doc.longs.getValue("book_count").toInt(), + ) } } @@ -215,23 +168,15 @@ class LuceneLookupSearchService( if (q.isBlank()) return emptyList() val tokens = q.split("\\s+".toRegex()).map { it.trim() }.filter { it.isNotEmpty() } if (tokens.isEmpty()) return emptyList() - return withSearcher { searcher -> - val b = BooleanQuery.Builder() - b.add(TermQuery(Term("type", "toc")), BooleanClause.Occur.FILTER) - tokens.forEach { t -> b.add(PrefixQuery(Term("q", t)), BooleanClause.Occur.MUST) } - val top = searcher.search(b.build(), limit, bookSort) - val stored = searcher.storedFields() - top.scoreDocs.map { sd -> - val doc = stored.document(sd.doc) - TocHit( - tocId = doc.getField("toc_id").numericValue().toLong(), - bookId = doc.getField("book_id").numericValue().toLong(), - bookTitle = doc.getField("book_title").stringValue(), - text = doc.getField("toc_text").stringValue(), - level = doc.getField("toc_level").numericValue().toInt(), - score = sd.score, - ) - } + return prefixSearch("toc", tokens, limit, bookSort, TOC_FIELDS).map { doc -> + TocHit( + tocId = doc.longs.getValue("toc_id"), + bookId = doc.longs.getValue("book_id"), + bookTitle = doc.strings.getValue("book_title"), + text = doc.strings.getValue("toc_text"), + level = doc.longs.getValue("toc_level").toInt(), + score = doc.score, + ) } } @@ -595,3 +540,8 @@ private fun sanitizeHebrewForAcronym(input: String): String { s = s.replace("\\s+".toRegex(), " ").trim() return s } + +// The stored fields each kind of lookup document is read from +private val BOOK_FIELDS = setOf("book_id", "category_id", "book_title", "is_base_book", "order_index") +private val AUTHOR_FIELDS = setOf("author_id", "author_name", "book_count") +private val TOC_FIELDS = setOf("toc_id", "book_id", "book_title", "toc_text", "toc_level") diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/RepositorySnippetSourceProvider.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/RepositorySnippetSourceProvider.kt similarity index 97% rename from SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/RepositorySnippetSourceProvider.kt rename to searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/RepositorySnippetSourceProvider.kt index 12e5795d9..b95cc382e 100644 --- a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/RepositorySnippetSourceProvider.kt +++ b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/RepositorySnippetSourceProvider.kt @@ -6,6 +6,7 @@ import io.github.kdroidfilter.seforimlibrary.search.SnippetProvider import io.github.kdroidfilter.seforimlibrary.search.SnippetSource import io.github.kdroidfilter.seforimlibrary.search.SnippetSources import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.IO import kotlinx.coroutines.withContext /** [SnippetProvider] over the app's database: the lines come from [repository]. */ diff --git a/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/SearchServices.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/SearchServices.kt new file mode 100644 index 000000000..3528eab4f --- /dev/null +++ b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/SearchServices.kt @@ -0,0 +1,67 @@ +package io.github.kdroidfilter.seforimapp.framework.search + +import io.github.kdroidfilter.seforimapp.features.search.semantic.SemanticModels +import io.github.kdroidfilter.seforimlibrary.dao.repository.SeforimRepository +import io.github.kdroidfilter.seforimlibrary.search.HebrewTextUtils +import io.github.kdroidfilter.seforimlibrary.search.LexicalSearchEngine +import io.github.kdroidfilter.seforimlibrary.search.MagicDictionaryIndex +import io.github.kdroidfilter.seforimlibrary.search.SearchEngine +import io.github.kdroidfilter.seforimlibrary.search.index.LuceneIndex + +/** + * The Lucene index at [indexDir], read on a JVM through Nucleus JVM Access (SeforimLibrary's search-facade): the + * desktop's own, the one the mobile apps embed (in the app's process on Android and iOS). [cacheQueries]: cache any + * query seen often (the small lookup index). + */ +fun openLuceneIndex( + indexDir: String, + cacheQueries: Boolean = false, +): LuceneIndex = LuceneIndex(indexDir, cacheQueries) + +/** + * The search of the database at [databasePath] and of the indexes beside it, as the desktop and the mobile apps set it + * up: the search engine (lexical, fused with the semantic search in the official builds) and the lookup (books, + * authors, tables of contents). Everything opens lazily, on first use; [semanticModels] is the semantic model of a + * mobile app (the desktop's package carries its own). + */ +class SearchServices( + private val databasePath: String, + private val repository: SeforimRepository, + private val semanticModels: SemanticModels? = null, +) : AutoCloseable { + /** The full-text index: seforim.db.lucene beside seforim.db. */ + val indexPath: String get() = if (databasePath.endsWith(".db")) "$databasePath.lucene" else "$databasePath.luceneindex" + + /** The lookup index: seforim.db.lookup.lucene beside seforim.db. */ + val lookupIndexPath: String get() = if (databasePath.endsWith(".db")) "$databasePath.lookup.lucene" else "$databasePath.lookupindex" + + /** The lexical dictionary beside the index. */ + val dictionaryPath: String get() = indexPath.substringBeforeLast('/', ".") + "/lexical.db" + + private val textIndex = lazy { openLuceneIndex(indexPath) } + + /** The search engine. */ + val engine: SearchEngine by lazy { + val lexical = + LexicalSearchEngine( + index = textIndex.value, + snippetProvider = RepositorySnippetSourceProvider(repository), + dictionary = lazy { MagicDictionaryIndex.load(HebrewTextUtils::normalizeHebrew, dictionaryPath) }, + ) + withSemanticSearch(lexical, textIndex.value, repository, semanticModels) + } + + /** The acronyms of the books, for the lookup's ranking. */ + val acronyms: AcronymFrequencyCache by lazy { AcronymFrequencyCache { databasePath } } + + /** The lookup of books, authors and tables of contents. */ + val lookup: LuceneLookupSearchService by lazy { + LuceneLookupSearchService(openLuceneIndex(lookupIndexPath, cacheQueries = true), acronyms) + } + + override fun close() { + if (textIndex.isInitialized()) runCatching { engine.close() } + runCatching { lookup.close() } + runCatching { acronyms.close() } + } +} diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/SemanticSearchEngine.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/SemanticSearchEngine.kt similarity index 64% rename from SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/SemanticSearchEngine.kt rename to searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/SemanticSearchEngine.kt index 37e5a3fda..a7d0dbeba 100644 --- a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/SemanticSearchEngine.kt +++ b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/SemanticSearchEngine.kt @@ -1,24 +1,26 @@ package io.github.kdroidfilter.seforimapp.framework.search +import io.github.kdroidfilter.seforimapp.features.search.semantic.SemanticModels import io.github.kdroidfilter.seforimapp.features.search.semantic.installedSemanticSearch import io.github.kdroidfilter.seforimlibrary.dao.repository.SeforimRepository import io.github.kdroidfilter.seforimlibrary.search.LineHit -import io.github.kdroidfilter.seforimlibrary.search.LuceneSearchEngine import io.github.kdroidfilter.seforimlibrary.search.SearchEngine -import java.nio.file.Path +import io.github.kdroidfilter.seforimlibrary.search.index.LuceneIndex /** * Fuses [lexical] with dense semantic search in the official builds ([installedSemanticSearch], open core), whose - * vectors live in the same Lucene index at [indexPath]; returns [lexical] as is in a community build. Shared by the - * app's search engine and `zayit cli`, so both search the same way. + * vectors live in the same Lucene [index]; returns [lexical] as is in a community build. Shared by the app's search + * engine, `zayit cli` and the mobile apps, so all search the same way. [models] is the semantic model (null: the one the + * package carries, on the desktop). */ fun withSemanticSearch( - lexical: LuceneSearchEngine, - indexPath: Path, + lexical: SearchEngine, + index: LuceneIndex, repository: SeforimRepository, + models: SemanticModels? = null, ): SearchEngine { val semantic = installedSemanticSearch ?: return lexical - return semantic.hybrid(lexical, indexPath) { lineId, query -> + return semantic.hybrid(lexical, index, models) { lineId, query -> repository.getLine(lineId)?.let { line -> LineHit( bookId = line.bookId, diff --git a/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/framework/session/SearchPersistedState.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/framework/session/SearchPersistedState.kt new file mode 100644 index 000000000..d6a286def --- /dev/null +++ b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/framework/session/SearchPersistedState.kt @@ -0,0 +1,80 @@ +@file:OptIn(kotlinx.serialization.ExperimentalSerializationApi::class) + +package io.github.kdroidfilter.seforimapp.framework.session + +import io.github.kdroidfilter.seforim.tabs.SearchScope +import io.github.kdroidfilter.seforimapp.features.search.SearchTabCache +import kotlinx.serialization.Serializable + +/** What a search tab keeps across sessions: the desktop session saves it with its tab, a mobile app as it likes. */ +@Serializable +data class SearchPersistedState( + val query: String = "", + val globalExtended: Boolean = false, + val datasetScope: String = "global", + // Unused (a search is no longer scoped to a category), kept for the ProtoBuf field numbers + val filterCategoryId: Long = 0L, + val filterBookId: Long = 0L, + val filterTocId: Long = 0L, + // Unused, as filterCategoryId + val fetchCategoryId: Long = 0L, + val fetchBookId: Long = 0L, + val fetchTocId: Long = 0L, + // View filters (multi-select) + val selectedCategoryIds: Set = emptySet(), + val selectedBookIds: Set = emptySet(), + val selectedTocIds: Set = emptySet(), + // Scroll/anchor persistence + val scrollIndex: Int = 0, + val scrollOffset: Int = 0, + val anchorId: Long = -1L, + val anchorIndex: Int = 0, + // Full search snapshot (results + aggregates) for identical restore + val snapshot: SearchTabCache.Snapshot? = null, + val breadcrumbs: Map> = emptyMap(), + // Appended last: the session is ProtoBuf, numbered by declaration order. + // What the bar holds, typed but not searched yet (restored in the bar; [query] is what is searched) + val draftQuery: String = "", + // The author whose books are searched (none with a book or TOC scope) + val fetchAuthorId: Long = 0L, +) { + /** The scope the search runs in: its fetch scope, else its view filter. */ + val scope: SearchScope + get() { + val bookId = fetchBookId.takeIf { it > 0 } ?: filterBookId.takeIf { it > 0 } + val tocId = fetchTocId.takeIf { it > 0 } ?: filterTocId.takeIf { it > 0 } + return when { + tocId != null && bookId != null -> SearchScope.Toc(bookId = bookId, tocId = tocId) + bookId != null -> SearchScope.Book(bookId) + fetchAuthorId > 0 -> SearchScope.Author(fetchAuthorId) + else -> SearchScope.Global + } + } + + /** Runs the search in [scope]: both its fetch scope and its view filter (a legacy category: everywhere). */ + fun withScope(scope: SearchScope): SearchPersistedState { + val bookId = + when (scope) { + is SearchScope.Book -> scope.bookId + is SearchScope.Toc -> scope.bookId + else -> 0L + } + val tocId = (scope as? SearchScope.Toc)?.tocId ?: 0L + return copy( + datasetScope = + when (scope) { + SearchScope.Global, is SearchScope.Category -> "global" + is SearchScope.Book -> "book" + is SearchScope.Toc -> "toc" + is SearchScope.Author -> "author" + }, + filterCategoryId = 0L, + filterBookId = bookId, + filterTocId = tocId, + fetchCategoryId = 0L, + fetchBookId = bookId, + fetchTocId = tocId, + fetchAuthorId = (scope as? SearchScope.Author)?.authorId ?: 0L, + ) + } +} diff --git a/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/core/presentation/text/DiacriticsModeTest.kt b/searchkit/src/commonTest/kotlin/io/github/kdroidfilter/seforimapp/core/presentation/text/DiacriticsModeTest.kt similarity index 100% rename from SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/core/presentation/text/DiacriticsModeTest.kt rename to searchkit/src/commonTest/kotlin/io/github/kdroidfilter/seforimapp/core/presentation/text/DiacriticsModeTest.kt diff --git a/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/core/presentation/text/HebrewSearchTest.kt b/searchkit/src/commonTest/kotlin/io/github/kdroidfilter/seforimapp/core/presentation/text/HebrewSearchTest.kt similarity index 100% rename from SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/core/presentation/text/HebrewSearchTest.kt rename to searchkit/src/commonTest/kotlin/io/github/kdroidfilter/seforimapp/core/presentation/text/HebrewSearchTest.kt diff --git a/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/usecases/BookFindMatchesTest.kt b/searchkit/src/commonTest/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/usecases/BookFindMatchesTest.kt similarity index 95% rename from SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/usecases/BookFindMatchesTest.kt rename to searchkit/src/commonTest/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/usecases/BookFindMatchesTest.kt index 8da662cb5..cb780a93d 100644 --- a/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/usecases/BookFindMatchesTest.kt +++ b/searchkit/src/commonTest/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/usecases/BookFindMatchesTest.kt @@ -107,7 +107,7 @@ class BookFindMatchesTest { } @Test - fun `confirms candidates literally, nikud ignored, in reading order`() = + fun `confirms candidates literally - nikud ignored - in reading order`() = runTest { val matches = find("ברא") // Twice in line 0 (inside בראשית, then ברא), twice in line 2 @@ -117,14 +117,14 @@ class BookFindMatchesTest { } @Test - fun `matches against the displayed text, not the html`() = + fun `matches against the displayed text - not the html`() = runTest { assertEquals(0, find("").occurrences) assertEquals(1, find("והארץ").occurrences) } @Test - fun `steps through every match, line after line, wrapping both ways`() = + fun `steps through every match - line after line - wrapping both ways`() = runTest { val matches = find("ברא") val first = matches.nextAfterLine(-1, forward = true)!! @@ -137,7 +137,7 @@ class BookFindMatchesTest { } @Test - fun `whole-line matches, one occurrence each`() { + fun `whole-line matches - one occurrence each`() { val matches = BookFindMatches.ofLines(lines.reversed()) assertEquals(3, matches.occurrences) assertEquals(BookFindMatches.Match(1, 11, ordinal = 0, position = 2), matches.nextAfterLine(0, forward = true)) diff --git a/searchkit/src/commonTest/kotlin/io/github/kdroidfilter/seforimapp/framework/search/SearchKitDump.kt b/searchkit/src/commonTest/kotlin/io/github/kdroidfilter/seforimapp/framework/search/SearchKitDump.kt new file mode 100644 index 000000000..f54d9e377 --- /dev/null +++ b/searchkit/src/commonTest/kotlin/io/github/kdroidfilter/seforimapp/framework/search/SearchKitDump.kt @@ -0,0 +1,40 @@ +package io.github.kdroidfilter.seforimapp.framework.search + +import kotlin.math.round + +/** + * What the app's search answers on real data, as text: the lookup (books, authors, tables of contents) and the search + * engine (hybrid in the official builds: hits, snippets, highlights by meaning, smart find in a book). The same data + * gives the same dump on the desktop and the mobile apps, up to Lucene's score noise (see SeforimLibrary's SearchDump). + */ +object SearchKitDump { + private val BOOK_QUERIES = listOf("שוע", "רמבם", "בראשית", "חפץ חיים", "משנה ברורה", "גמ") + private val SEARCH_QUERIES = listOf("מה מברכים על אוכל", "ואהבת לרעך כמוך", "\"קריאת שמע\"", "הלכות שבת") + + suspend fun dump(services: SearchServices): String = + buildString { + val lookup = services.lookup + for (q in BOOK_QUERIES) appendLine("books $q -> ${lookup.suggestBooks(q, 8).map { "${it.id}:${it.title}" }}") + appendLine("authors רמב -> ${lookup.suggestAuthors("רמב", 5).map { it.name }}") + appendLine("toc הלכות שבת -> ${lookup.searchTocPrefix("הלכות שבת", 5).map { "${it.bookId}:${it.text}" }}") + + val engine = services.engine + appendLine("dense ${engine.denseReady()}") + for (q in SEARCH_QUERIES) { + appendLine("## $q (by meaning: ${engine.highlightsByMeaning(q)})") + val page = engine.openSession(q, baseBookOnly = false)?.use { it.nextPage(10) } + appendLine("total ${page?.totalHits}") + val hits = page?.hits.orEmpty() + for (hit in hits) appendLine(" ${hit.lineId} ${hit.bookTitle} ${(round(hit.score * 1e5) / 1e5)} | ${hit.snippet}") + val texts = hits.take(3).map { it.rawText.substring(it.lineStart, it.lineEnd) } + val highlights = engine.highlights(texts, q) + highlights.forEachIndexed { + i, + h, + -> + appendLine(" highlight ${hits[i].lineId} byMeaning=${h.byMeaning} ${h.ranges.map { texts[i].substring(it) }}") + } + hits.firstOrNull()?.let { appendLine(" find in ${it.bookId}: ${engine.semanticFind(q, it.bookId, 5)}") } + } + } +} diff --git a/searchkit/src/iosMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/Repositories.ios.kt b/searchkit/src/iosMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/Repositories.ios.kt new file mode 100644 index 000000000..33e1efe71 --- /dev/null +++ b/searchkit/src/iosMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/Repositories.ios.kt @@ -0,0 +1,25 @@ +package io.github.kdroidfilter.seforimapp.framework.search + +import app.cash.sqldelight.driver.native.NativeSqliteDriver +import co.touchlab.sqliter.DatabaseConfiguration +import io.github.kdroidfilter.seforimlibrary.dao.repository.SeforimRepository + +/** + * The repository of the database at [databasePath] (the downloaded database). Its schema exists already: the driver + * creates and migrates nothing. + */ +fun openSeforimRepository(databasePath: String): SeforimRepository { + val driver = + NativeSqliteDriver( + DatabaseConfiguration( + name = databasePath.substringAfterLast('/'), + version = 1, + create = {}, + upgrade = { _, _, _ -> }, + extendedConfig = DatabaseConfiguration.Extended(basePath = databasePath.substringBeforeLast('/')), + ), + // Several readers at once: the search reads snippets while the screens read books and lines + maxReaderConnections = 4, + ) + return SeforimRepository(databasePath, driver) +} diff --git a/searchkit/src/iosSimulatorArm64Test/kotlin/io/github/kdroidfilter/seforimapp/framework/search/IosSearchKitParityTest.kt b/searchkit/src/iosSimulatorArm64Test/kotlin/io/github/kdroidfilter/seforimapp/framework/search/IosSearchKitParityTest.kt new file mode 100644 index 000000000..25497864a --- /dev/null +++ b/searchkit/src/iosSimulatorArm64Test/kotlin/io/github/kdroidfilter/seforimapp/framework/search/IosSearchKitParityTest.kt @@ -0,0 +1,33 @@ +package io.github.kdroidfilter.seforimapp.framework.search + +import io.github.kdroidfilter.seforimapp.features.search.semantic.SemanticModels +import kotlinx.coroutines.runBlocking +import kotlinx.io.buffered +import kotlinx.io.files.Path +import kotlinx.io.files.SystemFileSystem +import kotlinx.io.writeString +import platform.Foundation.NSProcessInfo +import kotlin.test.Test +import kotlin.time.measureTimedValue + +/** + * The app's search on iOS, end to end in the simulator: the lookup and the hybrid search, Lucene in the embedded Java + * runtime, the semantic model on ONNX Runtime's C API, the database on SQLDelight's native driver. Writes SearchKitDump + * (variables passed by the iosSimulatorArm64Test task); skipped without data. + */ +class IosSearchKitParityTest { + @Test + fun dump() { + val env = NSProcessInfo.processInfo.environment + val db = env["SEFORIM_TEST_DB"] as String? ?: return + val models = + (env["SEFORIM_MODEL_DIR"] as String?)?.let { + SemanticModels("$it/seforim-embed-v5-int8.onnx", "$it/tokenizer.json", "$it/seforim-embed-v5-static.i8") + } + SearchServices(db, openSeforimRepository(db), models).use { services -> + val (dump, took) = measureTimedValue { runBlocking { SearchKitDump.dump(services) } } + println("SearchKit dump in $took") + SystemFileSystem.sink(Path(env["SEFORIM_PARITY_OUT"] as String)).buffered().use { it.writeString(dump) } + } + } +} diff --git a/searchkit/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/CliSemanticSearch.jvm.kt b/searchkit/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/CliSemanticSearch.jvm.kt new file mode 100644 index 000000000..ac0c6e6e0 --- /dev/null +++ b/searchkit/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/CliSemanticSearch.jvm.kt @@ -0,0 +1,13 @@ +package io.github.kdroidfilter.seforimapp.framework.search + +import io.github.kdroidfilter.seforimlibrary.dao.repository.SeforimRepository +import io.github.kdroidfilter.seforimlibrary.search.LuceneSearchEngine +import io.github.kdroidfilter.seforimlibrary.search.SearchEngine +import java.nio.file.Path + +/** [withSemanticSearch] for the engine `zayit cli` builds (see `CliHost`): the semantic search reads its index. */ +fun withSemanticSearch( + lexical: LuceneSearchEngine, + @Suppress("UNUSED_PARAMETER") indexDir: Path, + repository: SeforimRepository, +): SearchEngine = withSemanticSearch(lexical, lexical.index, repository) diff --git a/searchkit/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/Repositories.jvm.kt b/searchkit/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/Repositories.jvm.kt new file mode 100644 index 000000000..47f0e69a4 --- /dev/null +++ b/searchkit/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/Repositories.jvm.kt @@ -0,0 +1,8 @@ +package io.github.kdroidfilter.seforimapp.framework.search + +import app.cash.sqldelight.driver.jdbc.sqlite.JdbcSqliteDriver +import io.github.kdroidfilter.seforimlibrary.dao.repository.SeforimRepository + +/** The repository of the database at [databasePath] (the desktop app keeps its own pooled driver). */ +fun openSeforimRepository(databasePath: String): SeforimRepository = + SeforimRepository(databasePath, JdbcSqliteDriver("jdbc:sqlite:$databasePath")) diff --git a/searchkit/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/framework/search/SearchKitParityTest.kt b/searchkit/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/framework/search/SearchKitParityTest.kt new file mode 100644 index 000000000..45481f778 --- /dev/null +++ b/searchkit/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/framework/search/SearchKitParityTest.kt @@ -0,0 +1,19 @@ +package io.github.kdroidfilter.seforimapp.framework.search + +import kotlinx.coroutines.runBlocking +import java.io.File +import kotlin.test.Test + +/** Writes SearchKitDump on the JVM for the parity with iOS: -PseforimTestDb=, -PparityOut. */ +class SearchKitParityTest { + @Test + fun dump() { + val db = System.getProperty("seforimTestDb") ?: return + val out = File(System.getProperty("parityOut") ?: return) + SearchServices(db, openSeforimRepository(db)).use { services -> + val start = System.nanoTime() + out.writeText(runBlocking { SearchKitDump.dump(services) }) + println("SearchKit dump in ${(System.nanoTime() - start) / 1_000_000} ms") + } + } +} diff --git a/settings.gradle.kts b/settings.gradle.kts index 689746c76..114818d07 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -138,6 +138,8 @@ plugins { } include(":SeforimApp") +include(":searchkit") +include(":searchkit-check") include(":cataloggen") include(":widgetprocessor")