From 7e6d2f03277ba3c40286723207fddf8249303837 Mon Sep 17 00:00:00 2001 From: Elie Gambache Date: Fri, 16 Oct 2026 11:16:40 +0300 Subject: [PATCH 01/12] feat(search): searchkit, the search without its UI for desktop, Android 15+ and iOS The engines (lexical, hybrid with the semantic search), the lookup, the search use cases and the view models move to searchkit, a Kotlin Multiplatform module the desktop app uses as is; a mobile app only adds its screens. - searchkit: SearchServices, openLuceneIndex per platform (in process on the JVM and Android 15+, in the embedded OpenJDK on iOS), the database driver per platform, SearchResultViewModel and SearchHomeViewModel reaching the shell only through SearchHost's ports (SearchTabs, SearchSessionStore, SearchSettings, SearchHistory); the semantic addon on seforim-semantic 0.6.0 (Kotlin Multiplatform). - navigation: Android and iOS targets. - SeforimApp: the desktop adapters of the ports (DesktopSearchTabs, DesktopSearchSessionStore, AppSettings, HistoryStore) and the graph wiring. - SeforimLibrary: the multiplatform search (kdroidFilter/SeforimLibrary#134). Parity on real data (SearchKitDump): the JVM, the iOS simulator and an Android 15 emulator give the same lookup, hybrid search, snippets, highlights and smart find. Signed-off-by: Elie Gambache --- CLAUDE.md | 18 +- .../semantic/InstalledSemanticSearch.kt | 15 -- SeforimApp/build.gradle.kts | 13 +- .../seforimapp/core/history/HistoryStore.kt | 5 +- .../seforimapp/core/settings/AppSettings.kt | 21 +- .../ui/panels/booktoc/BookTocPanel.kt | 7 +- .../search/semantic/SemanticSearchFeature.kt | 22 -- .../framework/di/modules/AppCoreBindings.kt | 30 ++- .../framework/search/AcronymFrequencyCache.kt | 155 ------------- .../framework/search/DesktopSearchHost.kt | 76 +++++++ .../framework/session/SessionModels.kt | 74 ------- .../io/github/kdroidfilter/seforimapp/main.kt | 7 +- .../bookcontent/PostSelectLineTest.kt | 8 +- .../seforimapp/features/search/SearchBench.kt | 18 +- .../search/domain/AuthorScopeSearchTest.kt | 6 +- .../domain/ResultsIndexingUseCaseTest.kt | 3 +- .../settings/SettingsWindowViewModelTest.kt | 8 +- .../framework/desktop/DesktopManagerTest.kt | 8 +- .../search/BookSuggestionsE2ETest.kt | 6 +- .../framework/search/TypedAsWordsOfTest.kt | 3 +- SeforimLibrary | 2 +- build.gradle.kts | 1 + gradle/libs.versions.toml | 20 +- navigation/build.gradle.kts | 9 + .../kdroidfilter/seforim/tabs/TabsState.kt | 9 +- searchkit/README.md | 83 +++++++ .../semantic/InstalledSemanticSearch.kt | 0 .../semantic/BundledSemanticModels.jvm.kt | 8 + .../search/semantic/HybridSearchEngine.kt | 37 ++-- .../semantic/InstalledSemanticSearch.kt | 21 ++ .../search/semantic/LuceneVectorIndex.kt | 67 ++++++ .../semantic/BundledSemanticModels.mobile.kt | 4 + searchkit/build.gradle.kts | 209 ++++++++++++++++++ .../search/AndroidSearchKitParityTest.kt | 40 ++++ .../framework/search/LuceneIndexes.android.kt | 11 + .../framework/search/Repositories.android.kt | 38 ++++ .../core/coroutines/CoroutineUtils.kt | 0 .../seforimapp/core/deeplink/ZayitDeepLink.kt | 45 +++- .../features/bookcontent/state/StateKeys.kt | 0 .../features/search/SearchHomeViewModel.kt | 73 +++--- .../seforimapp/features/search/SearchHost.kt | 100 +++++++++ .../features/search/SearchResultViewModel.kt | 114 +++------- .../features/search/SearchTabCache.kt | 0 .../features/search/domain/AuthorNames.kt | 0 .../search/domain/BookDetailsLoader.kt | 8 +- .../search/domain/BuildSearchTreeUseCase.kt | 0 .../domain/CategoryNavigationUseCase.kt | 0 .../search/domain/ExecuteSearchUseCase.kt | 0 .../domain/GetBreadcrumbPiecesUseCase.kt | 0 .../search/domain/ResultsIndexingUseCase.kt | 18 +- .../search/domain/SearchEntityFinder.kt | 0 .../features/search/domain/SearchKey.kt | 2 +- .../domain/SearchStatePersistenceUseCase.kt | 0 .../search/domain/SearchTocUseCase.kt | 5 +- .../domain/reference/ReferenceParser.kt | 0 .../domain/reference/ReferenceResolver.kt | 0 .../reference/RepositoryReferenceSource.kt | 18 +- .../search/semantic/SemanticSearchFeature.kt | 37 ++++ .../seforimapp/framework/di/AppScope.kt | 0 .../framework/search/AcronymFrequencyCache.kt | 117 ++++++++++ .../search/LuceneLookupSearchService.kt | 169 +++++--------- .../search/RepositorySnippetSourceProvider.kt | 1 + .../framework/search/SearchServices.kt | 67 ++++++ .../framework/search/SemanticSearchEngine.kt | 16 +- .../framework/session/SearchPersistedState.kt | 80 +++++++ .../framework/search/SearchKitDump.kt | 40 ++++ .../framework/search/LuceneIndexes.ios.kt | 43 ++++ .../framework/search/Repositories.ios.kt | 25 +++ .../search/IosSearchKitParityTest.kt | 34 +++ .../framework/search/LuceneIndexes.jvm.kt | 20 ++ .../framework/search/Repositories.jvm.kt | 8 + .../framework/search/SearchKitParityTest.kt | 19 ++ settings.gradle.kts | 1 + 73 files changed, 1515 insertions(+), 607 deletions(-) delete mode 100644 SeforimApp/addons/semantic/enabled/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/InstalledSemanticSearch.kt delete mode 100644 SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/SemanticSearchFeature.kt delete mode 100644 SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/AcronymFrequencyCache.kt create mode 100644 SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/DesktopSearchHost.kt create mode 100644 searchkit/README.md rename {SeforimApp => searchkit}/addons/semantic/disabled/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/InstalledSemanticSearch.kt (100%) create mode 100644 searchkit/addons/semantic/enabled/jvm/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/BundledSemanticModels.jvm.kt rename {SeforimApp => searchkit}/addons/semantic/enabled/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/HybridSearchEngine.kt (88%) create mode 100644 searchkit/addons/semantic/enabled/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/InstalledSemanticSearch.kt create mode 100644 searchkit/addons/semantic/enabled/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/LuceneVectorIndex.kt create mode 100644 searchkit/addons/semantic/enabled/mobile/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/BundledSemanticModels.mobile.kt create mode 100644 searchkit/build.gradle.kts create mode 100644 searchkit/src/androidDeviceTest/kotlin/io/github/kdroidfilter/seforimapp/framework/search/AndroidSearchKitParityTest.kt create mode 100644 searchkit/src/androidMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/LuceneIndexes.android.kt create mode 100644 searchkit/src/androidMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/Repositories.android.kt rename {SeforimApp/src/jvmMain => searchkit/src/commonMain}/kotlin/io/github/kdroidfilter/seforimapp/core/coroutines/CoroutineUtils.kt (100%) rename {SeforimApp/src/jvmMain => searchkit/src/commonMain}/kotlin/io/github/kdroidfilter/seforimapp/core/deeplink/ZayitDeepLink.kt (67%) rename {SeforimApp/src/jvmMain => searchkit/src/commonMain}/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/state/StateKeys.kt (100%) rename {SeforimApp/src/jvmMain => searchkit/src/commonMain}/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchHomeViewModel.kt (94%) create mode 100644 searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchHost.kt rename {SeforimApp/src/jvmMain => searchkit/src/commonMain}/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchResultViewModel.kt (95%) rename {SeforimApp/src/jvmMain => searchkit/src/commonMain}/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchTabCache.kt (100%) rename {SeforimApp/src/jvmMain => searchkit/src/commonMain}/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/AuthorNames.kt (100%) rename {SeforimApp/src/jvmMain => searchkit/src/commonMain}/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/BookDetailsLoader.kt (92%) rename {SeforimApp/src/jvmMain => searchkit/src/commonMain}/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/BuildSearchTreeUseCase.kt (100%) rename {SeforimApp/src/jvmMain => searchkit/src/commonMain}/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/CategoryNavigationUseCase.kt (100%) rename {SeforimApp/src/jvmMain => searchkit/src/commonMain}/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/ExecuteSearchUseCase.kt (100%) rename {SeforimApp/src/jvmMain => searchkit/src/commonMain}/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/GetBreadcrumbPiecesUseCase.kt (100%) rename {SeforimApp/src/jvmMain => searchkit/src/commonMain}/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/ResultsIndexingUseCase.kt (94%) rename {SeforimApp/src/jvmMain => searchkit/src/commonMain}/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/SearchEntityFinder.kt (100%) rename {SeforimApp/src/jvmMain => searchkit/src/commonMain}/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/SearchKey.kt (92%) rename {SeforimApp/src/jvmMain => searchkit/src/commonMain}/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/SearchStatePersistenceUseCase.kt (100%) rename {SeforimApp/src/jvmMain => searchkit/src/commonMain}/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/SearchTocUseCase.kt (98%) rename {SeforimApp/src/jvmMain => searchkit/src/commonMain}/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/reference/ReferenceParser.kt (100%) rename {SeforimApp/src/jvmMain => searchkit/src/commonMain}/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/reference/ReferenceResolver.kt (100%) rename {SeforimApp/src/jvmMain => searchkit/src/commonMain}/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/reference/RepositoryReferenceSource.kt (81%) create mode 100644 searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/SemanticSearchFeature.kt rename {SeforimApp/src/jvmMain => searchkit/src/commonMain}/kotlin/io/github/kdroidfilter/seforimapp/framework/di/AppScope.kt (100%) create mode 100644 searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/AcronymFrequencyCache.kt rename {SeforimApp/src/jvmMain => searchkit/src/commonMain}/kotlin/io/github/kdroidfilter/seforimapp/framework/search/LuceneLookupSearchService.kt (77%) rename {SeforimApp/src/jvmMain => searchkit/src/commonMain}/kotlin/io/github/kdroidfilter/seforimapp/framework/search/RepositorySnippetSourceProvider.kt (97%) create mode 100644 searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/SearchServices.kt rename {SeforimApp/src/jvmMain => searchkit/src/commonMain}/kotlin/io/github/kdroidfilter/seforimapp/framework/search/SemanticSearchEngine.kt (64%) create mode 100644 searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/framework/session/SearchPersistedState.kt create mode 100644 searchkit/src/commonTest/kotlin/io/github/kdroidfilter/seforimapp/framework/search/SearchKitDump.kt create mode 100644 searchkit/src/iosMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/LuceneIndexes.ios.kt create mode 100644 searchkit/src/iosMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/Repositories.ios.kt create mode 100644 searchkit/src/iosSimulatorArm64Test/kotlin/io/github/kdroidfilter/seforimapp/framework/search/IosSearchKitParityTest.kt create mode 100644 searchkit/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/LuceneIndexes.jvm.kt create mode 100644 searchkit/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/Repositories.jvm.kt create mode 100644 searchkit/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/framework/search/SearchKitParityTest.kt diff --git a/CLAUDE.md b/CLAUDE.md index 4f0037a22..540104963 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -117,10 +117,19 @@ through `SiddurFeature`/`installedSiddur`, never by importing `addons/siddur` co The semantic search (own embedding model, dense KNN fused with lexical search by RRF) follows the same pattern: the private `io.github.kdroidfilter:seforim-semantic` package (module `semantic/` of the private `SeforimEmbedding` repo, read with the siddur's token by default, or `semantic.token`/`SEFORIM_SEMANTIC_TOKEN`/`gh`, or `-Psemantic.local=true`), -whose jar carries the int8 model as a resource (never in the DB bundle). `addons/semantic/enabled` holds the -`HybridSearchEngine`; `disabled` makes `installedSemanticSearch` null and the app uses `LuceneSearchEngine` alone. Core -code reaches it only through `SemanticSearchFeature`/`installedSemanticSearch`. With the same token, SeforimLibrary's -generator embeds the corpus into the index's vectors (`:searchindex:embedCorpus`, GPU via CUDA, no Python). +Kotlin Multiplatform since 0.6.0, whose JVM jar carries the int8 model as a resource (never in the DB bundle; mobile +apps download it). `searchkit/addons/semantic/enabled` holds the `HybridSearchEngine`; `disabled` makes +`installedSemanticSearch` null and the app searches lexically alone. Core code reaches it only through +`SemanticSearchFeature`/`installedSemanticSearch`. With the same token, SeforimLibrary's generator embeds the corpus +into the index's vectors (`:searchindex:embedCorpus`, GPU via CUDA, no Python). + +### Search on mobile: `searchkit` +The search without its UI (engines, lookup, use cases, `SearchResultViewModel`/`SearchHomeViewModel`) is the KMP module +`searchkit` (JVM, Android 15+, iOS), shared by the desktop and the mobile apps; see `searchkit/README.md`. Lucene is +reached only through SeforimLibrary's `LuceneIndex` port: in process on the JVM and Android (ART, `lucene-android` +patches), through an embedded OpenJDK (Zero) on iOS (`SeforimLibrary/IOS_RUNTIME.md`). The view models reach the +desktop shell only through the ports of `SearchHost.kt` (`SearchTabs`, `SearchSessionStore`, `SearchSettings`, +`SearchHistory`); keep `java.*`, AWT and desktop types out of `searchkit/src/commonMain`. The verse quote links of the Talmud (temporary, until the DB carries them) follow the same pattern: the private `io.github.kdroidfilter:seforim-quotelinks` package (repo `SeforimQuoteLinks`, read with the siddur's token by default, @@ -157,6 +166,7 @@ See `TAB_SYSTEM_README.md` for complete details. Key points: - `pagination`: Pagination support for large datasets - `texteffects`: Text rendering effects and styling - `network`: Network utilities and Ktor client setup +- `searchkit`: the search without its UI, multiplatform (see "Search on mobile") - `backup`: Backup core, Android-BackupAgent/Transport style: `BackupSource` (what: the app's `UserDataBackup`, user DB + portable prefs), `BackupDestination` (where: `LocalFileDestination`, …), `BackupManager` (the only code linking both) - `backup-drive`: Google Drive `BackupDestination` + `GoogleDriveSync` (OAuth PKCE loopback, hidden `appDataFolder`, append-only with the 10 latest kept as IntelliJ settings sync, never restores without asking, on-quit + daily auto backup paused while a restore is pending) diff --git a/SeforimApp/addons/semantic/enabled/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/InstalledSemanticSearch.kt b/SeforimApp/addons/semantic/enabled/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/InstalledSemanticSearch.kt deleted file mode 100644 index a599730fd..000000000 --- a/SeforimApp/addons/semantic/enabled/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/InstalledSemanticSearch.kt +++ /dev/null @@ -1,15 +0,0 @@ -package io.github.kdroidfilter.seforimapp.features.search.semantic - -import io.github.kdroidfilter.seforimlibrary.search.LineHit -import io.github.kdroidfilter.seforimlibrary.search.SearchEngine -import java.nio.file.Path - -/** An official build: the semantic search, its model inside the SeforimEmbedding package. */ -val installedSemanticSearch: SemanticSearchFeature? = - object : SemanticSearchFeature { - override fun hybrid( - lexical: SearchEngine, - indexDir: Path, - resolveLine: suspend (lineId: Long, query: String) -> LineHit?, - ): SearchEngine = HybridSearchEngine(lexical, indexDir, resolveLine) - } diff --git a/SeforimApp/build.gradle.kts b/SeforimApp/build.gradle.kts index 9363c0aaa..584739e00 100644 --- a/SeforimApp/build.gradle.kts +++ b/SeforimApp/build.gradle.kts @@ -43,10 +43,6 @@ sentry { // (settings.gradle.kts); a community build has none and addons/siddur/disabled says so val withSiddur = gradle.extensions.extraProperties["siddurEnabled"] == true -// The semantic search too (open core): its package (code + model) with its token or its sources beside; without -// them, addons/semantic/disabled leaves the search lexical only -val withSemantic = gradle.extensions.extraProperties["semanticEnabled"] == true - // The verse quote links of the Talmud (open core, temporary until the DB carries them): its package, or // addons/quotelinks/disabled and the reader shows the lines as they are val withQuoteLinks = gradle.extensions.extraProperties["quoteLinksEnabled"] == true @@ -145,11 +141,6 @@ kotlin { } if (withSiddur) jvmTest { kotlin.srcDir("addons/siddur/test/kotlin") } - jvmMain { - kotlin.srcDir(if (withSemantic) "addons/semantic/enabled/kotlin" else "addons/semantic/disabled/kotlin") - if (withSemantic) dependencies { implementation(libs.seforim.semantic) } - } - jvmMain { kotlin.srcDir(if (withQuoteLinks) "addons/quotelinks/enabled/kotlin" else "addons/quotelinks/disabled/kotlin") if (withQuoteLinks) dependencies { implementation(libs.seforim.quotelinks) } @@ -199,6 +190,10 @@ kotlin { // SeforimLibrary search module implementation(libs.seforimlibrary.search) + // The search without its UI (engines, lookup, use cases, view models), shared with the mobile apps; the + // semantic search (open core) comes with it in the official builds + implementation(project(":searchkit")) + // SeforimLibrary CLI: lets the desktop binary run headless search commands // (`zayit cli search ...`) by delegating to its runCli() entry point. implementation(libs.seforimlibrary.cli) diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/core/history/HistoryStore.kt b/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/core/history/HistoryStore.kt index 4af070739..98ba82494 100644 --- a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/core/history/HistoryStore.kt +++ b/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/core/history/HistoryStore.kt @@ -4,6 +4,7 @@ import androidx.compose.runtime.Stable import io.github.kdroidfilter.seforim.tabs.SearchScope import io.github.kdroidfilter.seforimapp.db.UserSettingsDb import io.github.kdroidfilter.seforimapp.db.Visit_history +import io.github.kdroidfilter.seforimapp.features.search.SearchHistory import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow @@ -39,7 +40,7 @@ enum class VisitKind { BOOK, SEARCH } */ class HistoryStore( database: UserSettingsDb, -) { +) : SearchHistory { private val queries = database.visitHistoryQueries private val _revision = MutableStateFlow(0L) @@ -79,7 +80,7 @@ class HistoryStore( * Records an executed search with the [scope] it ran in: one history row per (query, scope, extended), titled * with [scopeLabel] (the scope's name) so a scoped search reads apart from the same query elsewhere. */ - suspend fun recordSearchVisit( + override suspend fun recordSearchVisit( query: String, scope: SearchScope, globalExtended: Boolean, diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/core/settings/AppSettings.kt b/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/core/settings/AppSettings.kt index 6239fc1e5..b782d4456 100644 --- a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/core/settings/AppSettings.kt +++ b/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/core/settings/AppSettings.kt @@ -8,6 +8,7 @@ import dev.zacsweers.metro.SingleIn import io.github.kdroidfilter.seforimapp.core.presentation.theme.AccentColor import io.github.kdroidfilter.seforimapp.core.presentation.theme.IntUiThemes import io.github.kdroidfilter.seforimapp.earthwidget.WidgetAntiAliasing +import io.github.kdroidfilter.seforimapp.features.search.SearchSettings import io.github.kdroidfilter.seforimapp.framework.di.AppScope import io.github.kdroidfilter.seforimapp.framework.platform.PlatformInfo import kotlinx.coroutines.channels.BufferOverflow @@ -28,10 +29,10 @@ import java.util.concurrent.ConcurrentHashMap @SingleIn(AppScope::class) class AppSettings( private val settings: Settings, -) { +) : SearchSettings { // StateFlow to observe text size changes private val _textSizeFlow = MutableStateFlow(getTextSize()) - val textSizeFlow: StateFlow = _textSizeFlow.asStateFlow() + override val textSizeFlow: StateFlow = _textSizeFlow.asStateFlow() // StateFlow to observe line height changes private val _lineHeightFlow = MutableStateFlow(getLineHeight()) @@ -119,7 +120,7 @@ class AppSettings( fun findQueryFlow(tabId: String): StateFlow = queryFlowFor(tabId).asStateFlow() - fun setFindQuery( + override fun setFindQuery( tabId: String, q: String, ) { @@ -128,7 +129,7 @@ class AppSettings( fun findBarOpenFlow(tabId: String): StateFlow = findOpenFlowFor(tabId).asStateFlow() - fun openFindBar(tabId: String) { + override fun openFindBar(tabId: String) { findOpenFlowFor(tabId).value = true } @@ -169,7 +170,7 @@ class AppSettings( fun findSmartModeFlow(tabId: String): StateFlow = smartModeFlowFor(tabId).asStateFlow() - fun setFindSmartMode( + override fun setFindSmartMode( tabId: String, enabled: Boolean, ) { @@ -181,7 +182,7 @@ class AppSettings( flow.value = !flow.value } - fun getTextSize(): Float = settings[KEY_TEXT_SIZE, DEFAULT_TEXT_SIZE] + override fun getTextSize(): Float = settings[KEY_TEXT_SIZE, DEFAULT_TEXT_SIZE] fun setTextSize(size: Float) { settings[KEY_TEXT_SIZE] = size @@ -425,13 +426,13 @@ class AppSettings( // User profile accessors // Reactive flows to observe user identity changes across the app private val _userFirstNameFlow = MutableStateFlow(getUserFirstName() ?: "") - val userFirstNameFlow: StateFlow = _userFirstNameFlow.asStateFlow() + override val userFirstNameFlow: StateFlow = _userFirstNameFlow.asStateFlow() private val _userLastNameFlow = MutableStateFlow(getUserLastName() ?: "") - val userLastNameFlow: StateFlow = _userLastNameFlow.asStateFlow() + override val userLastNameFlow: StateFlow = _userLastNameFlow.asStateFlow() private val _userCommunityCodeFlow = MutableStateFlow(getUserCommunityCode()) - val userCommunityCodeFlow: StateFlow = _userCommunityCodeFlow.asStateFlow() + override val userCommunityCodeFlow: StateFlow = _userCommunityCodeFlow.asStateFlow() fun getUserFirstName(): String? { val value: String = settings[KEY_USER_FIRST_NAME, ""] @@ -631,7 +632,7 @@ class AppSettings( } // Text size constants - const val DEFAULT_TEXT_SIZE = 16f + const val DEFAULT_TEXT_SIZE = SearchSettings.DEFAULT_TEXT_SIZE const val MIN_TEXT_SIZE = 14f const val MAX_TEXT_SIZE = 50f const val TEXT_SIZE_INCREMENT = 2f diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/ui/panels/booktoc/BookTocPanel.kt b/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/ui/panels/booktoc/BookTocPanel.kt index 4e9da0b96..8346f787b 100644 --- a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/ui/panels/booktoc/BookTocPanel.kt +++ b/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/ui/panels/booktoc/BookTocPanel.kt @@ -138,11 +138,12 @@ fun SearchBookTocPanel( onHide = { onEvent(BookContentEvent.ToggleToc) }, ) Column(modifier = Modifier.padding(horizontal = 8.dp)) { - if (searchUi.scopeBook != null) { + val scopeBook = searchUi.scopeBook + if (scopeBook != null) { Box(modifier = Modifier.fillMaxHeight()) { val expanded: Set = uiState.toc.expandedEntries - var autoExpanded by remember(searchUi.scopeBook.id, tocCounts, tocTree) { mutableStateOf(false) } - LaunchedEffect(searchUi.scopeBook.id, tocCounts, tocTree) { + var autoExpanded by remember(scopeBook.id, tocCounts, tocTree) { mutableStateOf(false) } + LaunchedEffect(scopeBook.id, tocCounts, tocTree) { if (!autoExpanded) { val idsWithChildren = tocTree?.children?.keys ?: emptySet() val withResults = tocCounts.keys diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/SemanticSearchFeature.kt b/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/SemanticSearchFeature.kt deleted file mode 100644 index 99d7f3630..000000000 --- a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/SemanticSearchFeature.kt +++ /dev/null @@ -1,22 +0,0 @@ -package io.github.kdroidfilter.seforimapp.features.search.semantic - -import io.github.kdroidfilter.seforimlibrary.search.LineHit -import io.github.kdroidfilter.seforimlibrary.search.SearchEngine -import java.nio.file.Path - -/** - * The semantic search (embedding model + dense search), in Zayit's official builds only (open core): its engine is in - * addons/semantic/enabled, built with the SeforimEmbedding package; a community build compiles addons/semantic/disabled - * instead, where [installedSemanticSearch] is null and the search stays lexical. - */ -interface SemanticSearchFeature { - /** - * Fuses [lexical] with dense search over the vectors of the index at [indexDir]; it degrades to [lexical] when the - * model or the vectors are missing. [resolveLine] turns a dense-only hit into a full [LineHit]. - */ - fun hybrid( - lexical: SearchEngine, - indexDir: Path, - resolveLine: suspend (lineId: Long, query: String) -> LineHit?, - ): SearchEngine -} diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/di/modules/AppCoreBindings.kt b/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/di/modules/AppCoreBindings.kt index e8954640d..d66d96175 100644 --- a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/di/modules/AppCoreBindings.kt +++ b/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/di/modules/AppCoreBindings.kt @@ -28,7 +28,11 @@ import io.github.kdroidfilter.seforimapp.core.settings.AppSettings import io.github.kdroidfilter.seforimapp.core.settings.CategoryDisplaySettingsStore import io.github.kdroidfilter.seforimapp.core.shnayimmikra.ShnayimMikraStore import io.github.kdroidfilter.seforimapp.db.UserSettingsDb +import io.github.kdroidfilter.seforimapp.features.search.SearchHistory import io.github.kdroidfilter.seforimapp.features.search.SearchHomeViewModel +import io.github.kdroidfilter.seforimapp.features.search.SearchSessionStore +import io.github.kdroidfilter.seforimapp.features.search.SearchSettings +import io.github.kdroidfilter.seforimapp.features.search.SearchTabs import io.github.kdroidfilter.seforimapp.framework.backup.UserDataBackup import io.github.kdroidfilter.seforimapp.framework.database.CatalogCache import io.github.kdroidfilter.seforimapp.framework.database.DatabasePathProvider @@ -38,8 +42,11 @@ import io.github.kdroidfilter.seforimapp.framework.database.getUserSettingsDatab import io.github.kdroidfilter.seforimapp.framework.desktop.DesktopManager import io.github.kdroidfilter.seforimapp.framework.di.AppScope import io.github.kdroidfilter.seforimapp.framework.search.AcronymFrequencyCache +import io.github.kdroidfilter.seforimapp.framework.search.DesktopSearchSessionStore +import io.github.kdroidfilter.seforimapp.framework.search.DesktopSearchTabs import io.github.kdroidfilter.seforimapp.framework.search.LuceneLookupSearchService import io.github.kdroidfilter.seforimapp.framework.search.RepositorySnippetSourceProvider +import io.github.kdroidfilter.seforimapp.framework.search.openLuceneIndex import io.github.kdroidfilter.seforimapp.framework.search.withSemanticSearch import io.github.kdroidfilter.seforimapp.framework.session.SessionManager import io.github.kdroidfilter.seforimapp.framework.session.TabPersistedStateStore @@ -190,13 +197,28 @@ object AppCoreBindings { val dictionaryPath = indexPath.resolveSibling("lexical.db") val snippetProvider = RepositorySnippetSourceProvider(repository) val lexical = LuceneSearchEngine(indexPath, snippetProvider, dictionaryPath = dictionaryPath) - return withSemanticSearch(lexical, indexPath, repository) + return withSemanticSearch(lexical, lexical.index, repository) } + // What the search screens (searchkit) ask of the desktop around them + @Provides + @SingleIn(AppScope::class) + fun provideSearchTabs(desktopManager: DesktopManager): SearchTabs = DesktopSearchTabs(desktopManager) + + @Provides + @SingleIn(AppScope::class) + fun provideSearchSessionStore(store: TabPersistedStateStore): SearchSessionStore = DesktopSearchSessionStore(store) + + @Provides + fun provideSearchSettings(appSettings: AppSettings): SearchSettings = appSettings + + @Provides + fun provideSearchHistory(historyStore: HistoryStore): SearchHistory = historyStore + @Provides @SingleIn(AppScope::class) fun provideAcronymFrequencyCache(databasePathProvider: DatabasePathProvider): AcronymFrequencyCache = - AcronymFrequencyCache(databasePathProvider) + AcronymFrequencyCache(databasePathProvider::get) @Provides @SingleIn(AppScope::class) @@ -206,7 +228,7 @@ object AppCoreBindings { ): LuceneLookupSearchService { val dbPath = databasePathProvider.get() val indexPath = if (dbPath.endsWith(".db")) "$dbPath.lookup.lucene" else "$dbPath.lookupindex" - return LuceneLookupSearchService(Paths.get(indexPath), acronymCache = acronymCache) + return LuceneLookupSearchService(openLuceneIndex(indexPath, cacheQueries = true), acronymCache = acronymCache) } @Provides @@ -269,7 +291,7 @@ object AppCoreBindings { // created and disposed by DesktopManager. searchHomeViewModelFactory = { observeProfile -> SearchHomeViewModel( - persistedStore = tabPersistedStateStore, + sessionStore = DesktopSearchSessionStore(tabPersistedStateStore), repository = repository, lookup = lookup, appSettings = appSettings, diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/AcronymFrequencyCache.kt b/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/AcronymFrequencyCache.kt deleted file mode 100644 index 91c727012..000000000 --- a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/AcronymFrequencyCache.kt +++ /dev/null @@ -1,155 +0,0 @@ -package io.github.kdroidfilter.seforimapp.framework.search - -import io.github.kdroidfilter.seforimapp.framework.database.DatabasePathProvider -import io.github.kdroidfilter.seforimapp.logger.infoln -import java.sql.Connection -import java.sql.DriverManager - -/** - * Cache for acronym frequency data from seforim.db. - * Provides fast lookup of how many books use each acronym. - * - * Uses the book_acronym table from seforim.db - no separate database needed! - */ -class AcronymFrequencyCache( - private val databasePath: () -> String, -) { - constructor(databasePathProvider: DatabasePathProvider) : this(databasePathProvider::get) - - private val connection: Connection by lazy { - loadSeforimDb() - } - - // Cache: acronym → frequency (number of books using it) - private val frequencyCache = mutableMapOf() - - // Cache: acronym → list of book titles - private val bookTitlesCache = mutableMapOf>() - - // Cache: book title → list of acronyms - private val titleToAcronymsCache = mutableMapOf>() - - init { - // Pre-load common acronyms into cache - preloadCache() - } - - /** - * Get the frequency (number of books) for an acronym. - */ - fun getFrequency(acronym: String): Int = - frequencyCache.getOrPut(acronym) { - queryFrequency(acronym) - } - - /** - * Get all book titles that use this acronym. - */ - fun getBookTitles(acronym: String): List = - bookTitlesCache.getOrPut(acronym) { - queryBookTitles(acronym) - } - - /** - * Check if an acronym is unique (used by only one book). - */ - fun isUnique(acronym: String): Boolean = getFrequency(acronym) == 1 - - /** - * Get all acronyms for a given book title. - */ - fun getAcronymsForBook(bookTitle: String): List = - titleToAcronymsCache.getOrPut(bookTitle) { - queryAcronymsForTitle(bookTitle) - } - - private fun loadSeforimDb(): Connection { - val dbPath = databasePath() - infoln { "Loading acronym data from seforim.db: $dbPath" } - return DriverManager.getConnection("jdbc:sqlite:$dbPath") - } - - private fun queryFrequency(acronym: String): Int { - val sql = """ - SELECT COUNT(DISTINCT bookId) as frequency - FROM book_acronym - WHERE term = ? - """ - - return connection.prepareStatement(sql).use { stmt -> - stmt.setString(1, acronym) - stmt.executeQuery().use { rs -> - if (rs.next()) rs.getInt("frequency") else 0 - } - } - } - - private fun queryBookTitles(acronym: String): List { - val sql = """ - SELECT DISTINCT b.title - FROM book_acronym ba - JOIN book b ON ba.bookId = b.id - WHERE ba.term = ? - """ - - return connection.prepareStatement(sql).use { stmt -> - stmt.setString(1, acronym) - stmt.executeQuery().use { rs -> - buildList { - while (rs.next()) { - add(rs.getString("title")) - } - } - } - } - } - - private fun queryAcronymsForTitle(bookTitle: String): List { - val sql = """ - SELECT DISTINCT ba.term - FROM book b - JOIN book_acronym ba ON b.id = ba.bookId - WHERE b.title = ? - """ - - return connection.prepareStatement(sql).use { stmt -> - stmt.setString(1, bookTitle) - stmt.executeQuery().use { rs -> - buildList { - while (rs.next()) { - add(rs.getString("term")) - } - } - } - } - } - - private fun preloadCache() { - // Pre-load top 100 most common acronyms - val sql = """ - SELECT - term, - COUNT(DISTINCT bookId) as frequency - FROM book_acronym - GROUP BY term - ORDER BY frequency DESC - LIMIT 100 - """ - - connection.prepareStatement(sql).use { stmt -> - stmt.executeQuery().use { rs -> - while (rs.next()) { - val acronym = rs.getString("term") - val frequency = rs.getInt("frequency") - frequencyCache[acronym] = frequency - } - } - } - - infoln { "Pre-loaded ${frequencyCache.size} acronyms into cache from seforim.db" } - } - - fun close() { - connection.close() - } -} diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/DesktopSearchHost.kt b/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/DesktopSearchHost.kt new file mode 100644 index 000000000..a843422ec --- /dev/null +++ b/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/DesktopSearchHost.kt @@ -0,0 +1,76 @@ +package io.github.kdroidfilter.seforimapp.framework.search + +import io.github.kdroidfilter.seforim.tabs.TabsDestination +import io.github.kdroidfilter.seforimapp.features.search.SearchSessionStore +import io.github.kdroidfilter.seforimapp.features.search.SearchTabs +import io.github.kdroidfilter.seforimapp.framework.desktop.DesktopManager +import io.github.kdroidfilter.seforimapp.framework.session.SearchPersistedState +import io.github.kdroidfilter.seforimapp.framework.session.TabPersistedStateStore +import kotlinx.coroutines.flow.Flow + +/** The search's tabs on the desktop: those of the window that holds the searching tab, across virtual desktops. */ +class DesktopSearchTabs( + private val desktopManager: DesktopManager, +) : SearchTabs { + override fun openTab( + fromTabId: String, + destination: TabsDestination, + ) { + desktopManager.tabsViewModelFor(fromTabId)?.openTab(destination) + } + + override fun replace( + tabId: String, + destination: TabsDestination, + ) { + desktopManager.tabsViewModelFor(tabId)?.replaceCurrentTabDestination(destination) + } + + override fun tabExistsFlow(tabId: String): Flow = desktopManager.tabExistsFlow(tabId) + + override fun isTabOpen(tabId: String): Boolean = desktopManager.isTabOpen(tabId) +} + +/** The search's state in the desktop session: the per-tab state the session saves at close. */ +class DesktopSearchSessionStore( + private val store: TabPersistedStateStore, +) : SearchSessionStore { + override fun search(tabId: String): SearchPersistedState? = store.get(tabId)?.search + + override fun updateSearch( + tabId: String, + transform: (SearchPersistedState?) -> SearchPersistedState?, + ) { + store.update(tabId) { current -> current.copy(search = transform(current.search)) } + } + + override fun selectBook( + tabId: String, + bookId: Long, + ) { + store.update(tabId) { current -> current.copy(bookContent = current.bookContent.copy(selectedBookId = bookId)) } + } + + override fun openBookAtLine( + tabId: String, + bookId: Long, + lineId: Long, + ) { + store.update(tabId) { current -> + current.copy( + bookContent = + current.bookContent.copy( + selectedBookId = bookId, + selectedLineIds = setOf(lineId), + primarySelectedLineId = lineId, + isTocEntrySelection = false, + contentAnchorLineId = lineId, + contentAnchorIndex = 0, + contentScrollIndex = 0, + contentScrollOffset = 0, + isTocVisible = true, + ), + ) + } + } +} diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/session/SessionModels.kt b/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/session/SessionModels.kt index 55090dac0..ab0526e1d 100644 --- a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/session/SessionModels.kt +++ b/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/session/SessionModels.kt @@ -3,11 +3,9 @@ package io.github.kdroidfilter.seforimapp.framework.session import io.github.kdroidfilter.seforim.desktop.VirtualDesktop -import io.github.kdroidfilter.seforim.tabs.SearchScope import io.github.kdroidfilter.seforim.tabs.TabType import io.github.kdroidfilter.seforim.tabs.TabsDestination import io.github.kdroidfilter.seforimapp.features.bookcontent.state.SplitDefaults -import io.github.kdroidfilter.seforimapp.features.search.SearchTabCache import kotlinx.serialization.Serializable /** @@ -183,75 +181,3 @@ data class DesktopsState( .filter { id -> desktops.any { it.id == id } } .distinct() } - -@Serializable -data class SearchPersistedState( - val query: String = "", - val globalExtended: Boolean = false, - val datasetScope: String = "global", - // Unused (a search is no longer scoped to a category), kept for the ProtoBuf field numbers - val filterCategoryId: Long = 0L, - val filterBookId: Long = 0L, - val filterTocId: Long = 0L, - // Unused, as filterCategoryId - val fetchCategoryId: Long = 0L, - val fetchBookId: Long = 0L, - val fetchTocId: Long = 0L, - // View filters (multi-select) - val selectedCategoryIds: Set = emptySet(), - val selectedBookIds: Set = emptySet(), - val selectedTocIds: Set = emptySet(), - // Scroll/anchor persistence - val scrollIndex: Int = 0, - val scrollOffset: Int = 0, - val anchorId: Long = -1L, - val anchorIndex: Int = 0, - // Full search snapshot (results + aggregates) for identical restore - val snapshot: SearchTabCache.Snapshot? = null, - val breadcrumbs: Map> = emptyMap(), - // Appended last: the session is ProtoBuf, numbered by declaration order. - // What the bar holds, typed but not searched yet (restored in the bar; [query] is what is searched) - val draftQuery: String = "", - // The author whose books are searched (none with a book or TOC scope) - val fetchAuthorId: Long = 0L, -) { - /** The scope the search runs in: its fetch scope, else its view filter. */ - val scope: SearchScope - get() { - val bookId = fetchBookId.takeIf { it > 0 } ?: filterBookId.takeIf { it > 0 } - val tocId = fetchTocId.takeIf { it > 0 } ?: filterTocId.takeIf { it > 0 } - return when { - tocId != null && bookId != null -> SearchScope.Toc(bookId = bookId, tocId = tocId) - bookId != null -> SearchScope.Book(bookId) - fetchAuthorId > 0 -> SearchScope.Author(fetchAuthorId) - else -> SearchScope.Global - } - } - - /** Runs the search in [scope]: both its fetch scope and its view filter (a legacy category: everywhere). */ - fun withScope(scope: SearchScope): SearchPersistedState { - val bookId = - when (scope) { - is SearchScope.Book -> scope.bookId - is SearchScope.Toc -> scope.bookId - else -> 0L - } - val tocId = (scope as? SearchScope.Toc)?.tocId ?: 0L - return copy( - datasetScope = - when (scope) { - SearchScope.Global, is SearchScope.Category -> "global" - is SearchScope.Book -> "book" - is SearchScope.Toc -> "toc" - is SearchScope.Author -> "author" - }, - filterCategoryId = 0L, - filterBookId = bookId, - filterTocId = tocId, - fetchCategoryId = 0L, - fetchBookId = bookId, - fetchTocId = tocId, - fetchAuthorId = (scope as? SearchScope.Author)?.authorId ?: 0L, - ) - } -} diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/main.kt b/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/main.kt index ff8423fd3..24fd80d73 100644 --- a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/main.kt +++ b/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/main.kt @@ -59,6 +59,7 @@ import io.github.kdroidfilter.seforimapp.framework.di.LocalAppGraph import io.github.kdroidfilter.seforimapp.framework.platform.PlatformInfo import io.github.kdroidfilter.seforimapp.framework.search.AcronymFrequencyCache import io.github.kdroidfilter.seforimapp.framework.search.LuceneLookupSearchService +import io.github.kdroidfilter.seforimapp.framework.search.openLuceneIndex import io.github.kdroidfilter.seforimapp.framework.search.withSemanticSearch import io.github.kdroidfilter.seforimapp.logger.infoln import io.github.kdroidfilter.seforimapp.logger.isDevEnv @@ -154,7 +155,11 @@ private fun appCliHost() = lookup = { dbPath, query, limit -> val indexPath = Paths.get("$dbPath.lookup.lucene") require(Files.exists(indexPath)) { "Lookup index not found: $indexPath" } - val service = LuceneLookupSearchService(indexPath, acronymCache = AcronymFrequencyCache { dbPath.toString() }) + val service = + LuceneLookupSearchService( + openLuceneIndex(indexPath.toString(), cacheQueries = true), + acronymCache = AcronymFrequencyCache { dbPath.toString() }, + ) LookupOutput( query = query, books = diff --git a/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/PostSelectLineTest.kt b/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/PostSelectLineTest.kt index 75df36fd8..a7a03c2ff 100644 --- a/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/PostSelectLineTest.kt +++ b/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/PostSelectLineTest.kt @@ -16,6 +16,7 @@ import io.github.kdroidfilter.seforimapp.features.bookcontent.usecases.NotesUseC import io.github.kdroidfilter.seforimapp.features.bookcontent.usecases.TocUseCase import io.github.kdroidfilter.seforimapp.features.search.SearchHomeViewModel import io.github.kdroidfilter.seforimapp.framework.desktop.DesktopManager +import io.github.kdroidfilter.seforimapp.framework.search.DesktopSearchSessionStore import io.github.kdroidfilter.seforimapp.framework.session.TabPersistedStateStore import io.github.kdroidfilter.seforimapp.framework.session.TabThumbnailStore import io.github.kdroidfilter.seforimapp.testAppSettings @@ -87,7 +88,12 @@ class PostSelectLineTest { ), titleUpdateManager = titleUpdateManager, searchHomeViewModelFactory = { - SearchHomeViewModel(TabPersistedStateStore(), mockk(relaxed = true), mockk(relaxed = true), testAppSettings()) + SearchHomeViewModel( + DesktopSearchSessionStore(TabPersistedStateStore()), + mockk(relaxed = true), + mockk(relaxed = true), + testAppSettings(), + ) }, defaultDesktopName = "D1", ) diff --git a/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchBench.kt b/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchBench.kt index 8ee87c446..bebac94ab 100644 --- a/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchBench.kt +++ b/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchBench.kt @@ -1,10 +1,9 @@ package io.github.kdroidfilter.seforimapp.features.search -import io.github.kdroidfilter.seforimapp.features.search.semantic.installedSemanticSearch import io.github.kdroidfilter.seforimapp.framework.database.PersistentSqliteDriver import io.github.kdroidfilter.seforimapp.framework.search.RepositorySnippetSourceProvider +import io.github.kdroidfilter.seforimapp.framework.search.withSemanticSearch import io.github.kdroidfilter.seforimlibrary.dao.repository.SeforimRepository -import io.github.kdroidfilter.seforimlibrary.search.LineHit import io.github.kdroidfilter.seforimlibrary.search.LuceneSearchEngine import io.github.kdroidfilter.seforimlibrary.search.SearchEngine import kotlinx.coroutines.runBlocking @@ -117,18 +116,5 @@ private fun buildEngine( val lexical = LuceneSearchEngine(indexPath, RepositorySnippetSourceProvider(repository), dictionaryPath = indexPath.resolveSibling("lexical.db")) if (System.getenv("BENCH_LEXICAL_ONLY") != null) return lexical - val semantic = installedSemanticSearch ?: return lexical - return semantic.hybrid(lexical, indexPath) { lineId, query -> - repository.getLine(lineId)?.let { line -> - LineHit( - bookId = line.bookId, - bookTitle = repository.getBook(line.bookId)?.title.orEmpty(), - lineId = lineId, - lineIndex = line.lineIndex, - snippet = lexical.buildSnippet(line.content, query, 5), - score = 0f, - rawText = line.content, - ) - } - } + return withSemanticSearch(lexical, lexical.index, repository) } diff --git a/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/AuthorScopeSearchTest.kt b/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/AuthorScopeSearchTest.kt index 1f15ce65a..fb9e2677d 100644 --- a/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/AuthorScopeSearchTest.kt +++ b/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/AuthorScopeSearchTest.kt @@ -9,6 +9,8 @@ import io.github.kdroidfilter.seforimapp.db.UserSettingsDb import io.github.kdroidfilter.seforimapp.features.bookcontent.state.StateKeys import io.github.kdroidfilter.seforimapp.features.search.SearchResultViewModel import io.github.kdroidfilter.seforimapp.framework.desktop.DesktopManager +import io.github.kdroidfilter.seforimapp.framework.search.DesktopSearchSessionStore +import io.github.kdroidfilter.seforimapp.framework.search.DesktopSearchTabs import io.github.kdroidfilter.seforimapp.framework.session.SearchPersistedState import io.github.kdroidfilter.seforimapp.framework.session.TabPersistedStateStore import io.github.kdroidfilter.seforimapp.testAppSettings @@ -51,11 +53,11 @@ class AuthorScopeSearchTest : SearchIntegrationTestBase() { every { desktopManager.tabExistsFlow(any()) } returns flowOf(true) return SearchResultViewModel( savedStateHandle = SavedStateHandle(mapOf(StateKeys.TAB_ID to tabId)), - persistedStore = store, + sessionStore = DesktopSearchSessionStore(store), repository = requireRepository(), lucene = requireSearchEngine(), titleUpdateManager = TabTitleUpdateManager(), - desktopManager = desktopManager, + tabs = DesktopSearchTabs(desktopManager), historyStore = HistoryStore(UserSettingsDb(JdbcSqliteDriver(JdbcSqliteDriver.IN_MEMORY).also(UserSettingsDb.Schema::create))), appSettings = testAppSettings(), lookup = mockk(relaxed = true), diff --git a/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/ResultsIndexingUseCaseTest.kt b/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/ResultsIndexingUseCaseTest.kt index b553ceda8..a208f7ed5 100644 --- a/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/ResultsIndexingUseCaseTest.kt +++ b/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/ResultsIndexingUseCaseTest.kt @@ -5,6 +5,7 @@ import kotlinx.coroutines.runBlocking import kotlin.test.Test import kotlin.test.assertEquals import kotlin.test.assertNotNull +import kotlin.test.assertSame import kotlin.test.assertTrue /** @@ -46,7 +47,7 @@ class ResultsIndexingUseCaseTest { val index = useCase.ensureIndex(results) assertNotNull(index) - assertEquals(System.identityHashCode(results), index.identity) + assertSame(results, index.identity) // Check book to indices mapping assertEquals(3, index.bookToIndices.size) diff --git a/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/features/settings/SettingsWindowViewModelTest.kt b/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/features/settings/SettingsWindowViewModelTest.kt index 5f3cf4b2d..63d9301ac 100644 --- a/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/features/settings/SettingsWindowViewModelTest.kt +++ b/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/features/settings/SettingsWindowViewModelTest.kt @@ -3,6 +3,7 @@ package io.github.kdroidfilter.seforimapp.features.settings import io.github.kdroidfilter.seforim.tabs.TabTitleUpdateManager import io.github.kdroidfilter.seforimapp.features.search.SearchHomeViewModel import io.github.kdroidfilter.seforimapp.framework.desktop.DesktopManager +import io.github.kdroidfilter.seforimapp.framework.search.DesktopSearchSessionStore import io.github.kdroidfilter.seforimapp.framework.session.TabPersistedStateStore import io.github.kdroidfilter.seforimapp.framework.session.TabThumbnailStore import io.github.kdroidfilter.seforimapp.testAppSettings @@ -27,7 +28,12 @@ class SettingsWindowViewModelTest { ), titleUpdateManager = TabTitleUpdateManager(), searchHomeViewModelFactory = { - SearchHomeViewModel(TabPersistedStateStore(), mockk(relaxed = true), mockk(relaxed = true), testAppSettings()) + SearchHomeViewModel( + DesktopSearchSessionStore(TabPersistedStateStore()), + mockk(relaxed = true), + mockk(relaxed = true), + testAppSettings(), + ) }, defaultDesktopName = "D1", ) diff --git a/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/framework/desktop/DesktopManagerTest.kt b/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/framework/desktop/DesktopManagerTest.kt index 0714fb9f8..8d4fa52ff 100644 --- a/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/framework/desktop/DesktopManagerTest.kt +++ b/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/framework/desktop/DesktopManagerTest.kt @@ -5,6 +5,7 @@ import io.github.kdroidfilter.seforim.tabs.TabTitleUpdateManager import io.github.kdroidfilter.seforim.tabs.TabType import io.github.kdroidfilter.seforim.tabs.TabsDestination import io.github.kdroidfilter.seforimapp.features.search.SearchHomeViewModel +import io.github.kdroidfilter.seforimapp.framework.search.DesktopSearchSessionStore import io.github.kdroidfilter.seforimapp.framework.session.DesktopTabsSnapshot import io.github.kdroidfilter.seforimapp.framework.session.DesktopsState import io.github.kdroidfilter.seforimapp.framework.session.SerializableTabTitle @@ -57,7 +58,12 @@ class DesktopManagerTest { ), titleUpdateManager = TabTitleUpdateManager(), searchHomeViewModelFactory = { - SearchHomeViewModel(TabPersistedStateStore(), mockk(relaxed = true), mockk(relaxed = true), testAppSettings()) + SearchHomeViewModel( + DesktopSearchSessionStore(TabPersistedStateStore()), + mockk(relaxed = true), + mockk(relaxed = true), + testAppSettings(), + ) }, defaultDesktopName = "D1", bootState = bootState, diff --git a/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/framework/search/BookSuggestionsE2ETest.kt b/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/framework/search/BookSuggestionsE2ETest.kt index 2f050b8bd..87cdb54eb 100644 --- a/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/framework/search/BookSuggestionsE2ETest.kt +++ b/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/framework/search/BookSuggestionsE2ETest.kt @@ -5,7 +5,6 @@ import io.github.kdroidfilter.seforimapp.testAppSettings import io.github.vinceglb.filekit.FileKit import org.junit.Assume.assumeTrue import java.io.File -import java.nio.file.Paths import kotlin.test.Test import kotlin.test.assertEquals import kotlin.test.assertTrue @@ -19,7 +18,10 @@ class BookSuggestionsE2ETest { private val lookup by lazy { FileKit.init("io.github.kdroidfilter.seforimapp") val settings = testAppSettings().apply { setDatabasePath(dbPath) } - LuceneLookupSearchService(Paths.get("$dbPath.lookup.lucene"), acronymCache = AcronymFrequencyCache(DatabasePathProvider(settings))) + LuceneLookupSearchService( + openLuceneIndex("$dbPath.lookup.lucene", cacheQueries = true), + acronymCache = AcronymFrequencyCache(DatabasePathProvider(settings)::get), + ) } private fun titles(query: String): List { diff --git a/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/framework/search/TypedAsWordsOfTest.kt b/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/framework/search/TypedAsWordsOfTest.kt index a48eb3f2d..11bd3e287 100644 --- a/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/framework/search/TypedAsWordsOfTest.kt +++ b/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/framework/search/TypedAsWordsOfTest.kt @@ -1,13 +1,12 @@ package io.github.kdroidfilter.seforimapp.framework.search -import java.nio.file.Paths import kotlin.test.Test import kotlin.test.assertFalse import kotlin.test.assertTrue class TypedAsWordsOfTest { // The index is opened lazily: no lookup here - private val lookup = LuceneLookupSearchService(Paths.get("unused")) + private val lookup = LuceneLookupSearchService(openLuceneIndex("unused")) private fun matches( typed: String, diff --git a/SeforimLibrary b/SeforimLibrary index 7ddf045b9..9c9a921b3 160000 --- a/SeforimLibrary +++ b/SeforimLibrary @@ -1 +1 @@ -Subproject commit 7ddf045b9a2ea9fe4ad5c6a24e0513c2022da67d +Subproject commit 9c9a921b3d5784ed8cf963372a66738144406107 diff --git a/build.gradle.kts b/build.gradle.kts index 1b4dfa8fa..4def02ae8 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -3,6 +3,7 @@ plugins { alias(libs.plugins.compose.compiler).apply(false) alias(libs.plugins.compose).apply(false) alias(libs.plugins.android.application).apply(false) + alias(libs.plugins.android.kmp.library).apply(false) alias(libs.plugins.hotReload).apply(false) alias(libs.plugins.kotlinx.serialization).apply(false) alias(libs.plugins.buildConfig).apply(false) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index fda8c2192..f43367ff9 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -38,7 +38,13 @@ sqlDelight = "2.3.2" adaptiveNavigation = "1.2.0" kosherkotlin = "3.0.5" seforimSiddur = "0.2.2" -seforimSemantic = "0.5.0" +seforimSemantic = "0.6.0" +kermit = "2.1.0" +androidx-test-runner = "1.7.0" +androidx-test-junit = "1.3.0" +cachemap = "0.3.0" +atomicfu = "0.33.0" +kotlinx-io = "0.9.1" seforimQuoteLinks = "0.1.0" zstdJni = "1.5.7-10" metro = "1.4.5" @@ -140,9 +146,20 @@ adaptive-layout = { module = "org.jetbrains.compose.material3.adaptive:adaptive- adaptive-navigation = { module = "org.jetbrains.compose.material3.adaptive:adaptive-navigation", version.ref = "adaptiveNavigation" } split-pane-desktop = { module = "org.jetbrains.compose.components:components-splitpane-desktop", version.ref = "compose"} sqlite-driver = { module = "app.cash.sqldelight:sqlite-driver", version.ref = "sqlDelight" } +sqldelight-android-driver = { module = "app.cash.sqldelight:android-driver", version.ref = "sqlDelight" } +sqldelight-native-driver = { module = "app.cash.sqldelight:native-driver", version.ref = "sqlDelight" } kosherkotlin = { module = "io.github.kdroidfilter:kosherkotlin", version.ref = "kosherkotlin" } seforim-siddur = { module = "io.github.kdroidfilter:seforim-siddur", version.ref = "seforimSiddur" } seforim-semantic = { module = "io.github.kdroidfilter:seforim-semantic", version.ref = "seforimSemantic" } +kermit = { module = "co.touchlab:kermit", version.ref = "kermit" } +androidx-test-runner = { module = "androidx.test:runner", version.ref = "androidx-test-runner" } +androidx-test-junit = { module = "androidx.test.ext:junit", version.ref = "androidx-test-junit" } +cachemap = { module = "io.github.charlietap:cachemap", version.ref = "cachemap" } +atomicfu = { module = "org.jetbrains.kotlinx:atomicfu", version.ref = "atomicfu" } +kotlinx-io-core = { module = "org.jetbrains.kotlinx:kotlinx-io-core", version.ref = "kotlinx-io" } +androidx-lifecycle-viewmodel-core = { module = "org.jetbrains.androidx.lifecycle:lifecycle-viewmodel", version.ref = "androidx-lifecycle" } +androidx-lifecycle-viewmodel-savedstate = { module = "org.jetbrains.androidx.lifecycle:lifecycle-viewmodel-savedstate", version.ref = "androidx-lifecycle" } +metrox-viewmodel = { module = "dev.zacsweers.metro:metrox-viewmodel", version.ref = "metro" } seforim-quotelinks = { module = "io.github.kdroidfilter:seforim-quotelinks", version.ref = "seforimQuoteLinks" } zstd-jni = { module = "com.github.luben:zstd-jni", version.ref = "zstdJni" } lucene-core = { module = "org.apache.lucene:lucene-core", version.ref = "lucene" } @@ -186,6 +203,7 @@ multiplatform = { id = "org.jetbrains.kotlin.multiplatform", version.ref = "kotl compose-compiler = { id = "org.jetbrains.kotlin.plugin.compose", version.ref = "kotlin" } compose = { id = "org.jetbrains.compose", version.ref = "compose" } android-application = { id = "com.android.application", version.ref = "agp" } +android-kmp-library = { id = "com.android.kotlin.multiplatform.library", version.ref = "agp" } hotReload = { id = "org.jetbrains.compose.hot-reload", version.ref = "hotReload" } kotlinx-serialization = { id = "org.jetbrains.kotlin.plugin.serialization", version.ref = "kotlin" } buildConfig = { id = "com.github.gmazzo.buildconfig", version.ref = "buildConfig" } diff --git a/navigation/build.gradle.kts b/navigation/build.gradle.kts index 1eb67f6d7..026c3681a 100644 --- a/navigation/build.gradle.kts +++ b/navigation/build.gradle.kts @@ -2,6 +2,7 @@ import org.jetbrains.compose.reload.gradle.ComposeHotRun plugins { alias(libs.plugins.multiplatform) + alias(libs.plugins.android.kmp.library) alias(libs.plugins.compose.compiler) alias(libs.plugins.compose) alias(libs.plugins.kotlinx.serialization) @@ -15,6 +16,14 @@ kotlin { ) jvm() + // The tabs' destinations are shared with the mobile apps' search (searchkit) + android { + namespace = "io.github.kdroidfilter.seforim.navigation" + compileSdk = 37 + minSdk = 26 + } + iosArm64() + iosSimulatorArm64() sourceSets { commonMain.dependencies { diff --git a/navigation/src/commonMain/kotlin/io/github/kdroidfilter/seforim/tabs/TabsState.kt b/navigation/src/commonMain/kotlin/io/github/kdroidfilter/seforim/tabs/TabsState.kt index 2af6d8d7a..72e466278 100644 --- a/navigation/src/commonMain/kotlin/io/github/kdroidfilter/seforim/tabs/TabsState.kt +++ b/navigation/src/commonMain/kotlin/io/github/kdroidfilter/seforim/tabs/TabsState.kt @@ -1,13 +1,14 @@ package io.github.kdroidfilter.seforim.tabs import androidx.compose.runtime.Immutable -import java.util.UUID +import kotlin.uuid.ExperimentalUuidApi +import kotlin.uuid.Uuid @Immutable data class TabItem( val id: Int, val title: String = "Default Tab", - val destination: TabsDestination = TabsDestination.Home(UUID.randomUUID().toString()), + val destination: TabsDestination = TabsDestination.Home(newTabId()), val tabType: TabType = TabType.SEARCH, // Pinned tabs stay ahead of the others and are not closed by accident. val pinned: Boolean = false, @@ -20,3 +21,7 @@ data class TabsState( val tabs: List, val selectedTabIndex: Int, ) + +/** A fresh tab id. */ +@OptIn(ExperimentalUuidApi::class) +fun newTabId(): String = Uuid.random().toString() diff --git a/searchkit/README.md b/searchkit/README.md new file mode 100644 index 000000000..c67de2f03 --- /dev/null +++ b/searchkit/README.md @@ -0,0 +1,83 @@ +# searchkit + +Zayit's search without its UI, for the desktop app and the mobile ones (Android 15+, iOS): the engines (lexical, fused +with the semantic search in the official builds), the lookup (books, authors, tables of contents), the search use cases +and the view models of the search screens. The desktop app uses it as is; a mobile app only adds its screens. + +Verified on real data: `SearchKitDump` (lookup, hybrid search, snippets, highlights by meaning, smart find) gives the +same text on the JVM, in the iOS simulator and on an Android emulator (`jvmTest`, `iosSimulatorArm64Test`, +`androidDeviceTest`). + +## What runs where + +| | Desktop (JVM) | Android 15+ | iOS | +|---|---|---|---| +| Lucene (full-text and vector index) | in process | in process, on ART (`lucene-android`) | in the embedded Java runtime | +| Database (`SeforimRepository`) | the app's pooled JDBC driver | `openSeforimRepository(context, path)` | `openSeforimRepository(path)` | +| Semantic model (official builds) | ONNX Runtime Java, model in the jar | ONNX Runtime Android | ONNX Runtime C API (`onnxruntime-c` 1.20.0) | +| Model files | bundled | downloaded with the data | downloaded with the data | + +Android before 15 isn't supported: its ART lacks the Java 21 APIs Lucene 10 needs. + +The iOS runtime is OpenJDK's mobile port, built by the CI of +[kdroidFilter/openjdk-mobile](https://github.com/kdroidFilter/openjdk-mobile) and downloaded by the build (see +[IOS_RUNTIME.md](../SeforimLibrary/IOS_RUNTIME.md)). + +Everything else (query building, dictionary, ranking, snippets, highlights, RRF fusion, tokenizer, use cases, view +models) is common code. + +## Using it from an app + +```kotlin +// The data: seforim.db with seforim.db.lucene, seforim.db.lookup.lucene and lexical.db beside it, downloaded into the +// app's storage (like the desktop's data). The semantic model too, in the official builds. +val repository = openSeforimRepository(context, dbPath) // iOS: openSeforimRepository(dbPath) +val services = SearchServices(dbPath, repository, SemanticModels(model, tokenizer, staticTable)) +services.engine // SearchEngine: openSession, computeFacets, highlights, semanticFind... +services.lookup // LuceneLookupSearchService: suggestBooks, suggestAuthors, searchTocPrefix +``` + +On iOS, `IosJavaRuntime` (`classPath`, `options`) configures the embedded runtime before the first search; by default it +runs the jars of `App.app/lib/app`. `./gradlew :searchkit:assembleIosJavaRuntimeDevice` assembles +`static/` (what the app links, options in IOS_RUNTIME.md) and `lib/` (copy it into the bundle as `App.app/lib` in an +Xcode build phase). + +### The view models + +`SearchResultViewModel` (a results tab: search, facets, filters by category/book/TOC, paging, preview, breadcrumbs, +opening a result) and `SearchHomeViewModel` (the search bar: book/author/TOC/reference suggestions, scopes, deep links) +are the desktop's, unchanged. What they need of the app is in `SearchHost.kt`, four small interfaces: + +| Interface | Desktop implementation | What a mobile app provides | +|---|---|---| +| `SearchTabs` | `DesktopSearchTabs` (windows and tabs) | its navigation: open a destination (`TabsDestination`), screen still shown | +| `SearchSessionStore` | `DesktopSearchSessionStore` (the session) | where it keeps a screen's `SearchPersistedState` and opens a book at a line | +| `SearchSettings` | `AppSettings` | text size, find-in-page bar, user's name | +| `SearchHistory` | `HistoryStore` | the search history | + +`SearchResultViewModel` is built by Metro on the desktop (`@AssistedInject`, `SavedStateHandle` with `StateKeys.TAB_ID`); +a mobile app can call its constructor directly. + +## Builds + +- **Community**: `addons/semantic/disabled`, lexical search only. +- **Official**: `addons/semantic/enabled` with the private `io.github.kdroidfilter:seforim-semantic` package (Kotlin + Multiplatform since 0.6.0: publish it before an official CI build), or its sources beside with `-Psemantic.local=true`. + +## Tests on real data + +```bash +./gradlew :searchkit:jvmTest -PseforimTestDb= # build/parity/jvm.txt +./gradlew :searchkit:iosSimulatorArm64Test -PseforimTestDb= \ + -Pseforim.ortXcframework= -Pseforim.semanticModels= # build/parity/ios.txt +``` + +Use a copy of the database (an APFS clone: `cp -c`): `SeforimRepository` sets its PRAGMAs on open. On Android, push the +data into the test app's files dir under `seforim/` and run `AndroidSearchKitParityTest` (see its KDoc). + +## Not here (the UI's) + +- The screens, and the rendering of snippets and lines (`:htmlparser`, Jsoup and Compose `AnnotatedString`: on iOS it + needs Ksoup). +- The reader's find-in-page (`BookFindMatches`): its candidates come from the engine (`findInBookCandidates`), its + matches are confirmed on the displayed text, so it follows `:htmlparser`. diff --git a/SeforimApp/addons/semantic/disabled/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/InstalledSemanticSearch.kt b/searchkit/addons/semantic/disabled/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/InstalledSemanticSearch.kt similarity index 100% rename from SeforimApp/addons/semantic/disabled/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/InstalledSemanticSearch.kt rename to searchkit/addons/semantic/disabled/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/InstalledSemanticSearch.kt diff --git a/searchkit/addons/semantic/enabled/jvm/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/BundledSemanticModels.jvm.kt b/searchkit/addons/semantic/enabled/jvm/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/BundledSemanticModels.jvm.kt new file mode 100644 index 000000000..4aef78165 --- /dev/null +++ b/searchkit/addons/semantic/enabled/jvm/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/BundledSemanticModels.jvm.kt @@ -0,0 +1,8 @@ +package io.github.kdroidfilter.seforimapp.features.search.semantic + +import io.github.kdroidfilter.seforim.semantic.bundledModelFiles + +// The desktop's semantic jar carries the model, extracted on first use +private val bundled = lazy { bundledModelFiles().let { SemanticModels(it.model, it.tokenizer, it.staticTable) } } + +internal actual fun bundledSemanticModels(): Lazy? = bundled diff --git a/SeforimApp/addons/semantic/enabled/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/HybridSearchEngine.kt b/searchkit/addons/semantic/enabled/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/HybridSearchEngine.kt similarity index 88% rename from SeforimApp/addons/semantic/enabled/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/HybridSearchEngine.kt rename to searchkit/addons/semantic/enabled/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/HybridSearchEngine.kt index d9a74ee03..1b6195fe7 100644 --- a/SeforimApp/addons/semantic/enabled/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/HybridSearchEngine.kt +++ b/searchkit/addons/semantic/enabled/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/HybridSearchEngine.kt @@ -1,7 +1,8 @@ package io.github.kdroidfilter.seforimapp.features.search.semantic +import co.touchlab.kermit.Logger +import io.github.kdroidfilter.seforim.semantic.ModelFiles import io.github.kdroidfilter.seforim.semantic.SemanticSearch -import io.github.kdroidfilter.seforimapp.logger.warnln import io.github.kdroidfilter.seforimlibrary.search.LineHit import io.github.kdroidfilter.seforimlibrary.search.SearchEngine import io.github.kdroidfilter.seforimlibrary.search.SearchFacets @@ -12,13 +13,17 @@ import io.github.kdroidfilter.seforimlibrary.search.SnippetSources import io.github.kdroidfilter.seforimlibrary.search.WARMUP_PAGE_SIZE import io.github.kdroidfilter.seforimlibrary.search.WARMUP_QUERIES import io.github.kdroidfilter.seforimlibrary.search.WARMUP_ROUNDS +import io.github.kdroidfilter.seforimlibrary.search.index.LuceneIndex import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.IO import kotlinx.coroutines.async import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import kotlinx.coroutines.withContext -import java.nio.file.Path +import kotlin.concurrent.Volatile + +private val logger = Logger.withTag("HybridSearch") /** * Hybrid search: the lexical engine (BM25 + MagicDictionary) fused with dense semantic search ([SemanticSearch]) by @@ -30,7 +35,9 @@ import java.nio.file.Path */ internal class HybridSearchEngine( private val lexical: SearchEngine, - private val indexDir: Path, + private val index: LuceneIndex, + // Resolved on the first dense search: the desktop's model is extracted from its jar then, not at startup + private val models: Lazy?, private val resolveLine: suspend (lineId: Long, query: String) -> LineHit?, ) : SearchEngine { // The model (heavy ONNX session) loads lazily on the first dense search, off the main thread, so the first query @@ -40,21 +47,20 @@ internal class HybridSearchEngine( @Volatile private var semanticTried = false private val semanticMutex = Mutex() - // Cheap, no-load check: decides whether to take the dense path at all - private val semanticConfigured = SemanticSearch.isAvailable(indexDir) + // Cheap, no-load check: decides whether to take the dense path at all (the vectors are in the index itself) + private val semanticConfigured = models != null private suspend fun ensureSemantic(): SemanticSearch? { if (semanticTried) return semantic semanticMutex.withLock { if (!semanticTried) { semantic = - if (semanticConfigured) { - // The tokenizer library otherwise "calls home" (an EC2 metadata probe, then a tracking ping) - // when it loads: seconds of network timeouts on the first search, and a privacy leak - System.setProperty("OPT_OUT_TRACKING", "true") + if (models != null) { withContext(Dispatchers.IO) { - runCatching { SemanticSearch.open(indexDir) } - .onFailure { warnln(it) { "[HybridSearch] Semantic search unavailable; lexical only" } } + runCatching { + val files = models.value + SemanticSearch.open(LuceneVectorIndex(index), ModelFiles(files.model, files.tokenizer, files.staticTable)) + }.onFailure { logger.w(it) { "Semantic search unavailable; lexical only" } } .getOrNull() } } else { @@ -194,7 +200,7 @@ internal class HybridSearchEngine( try { semantic.search(query, CANDIDATES, baseOnly, bookIds) } catch (e: Exception) { - warnln(e) { "[HybridSearch] Dense KNN failed; lexical only" } + logger.w(e) { "Dense KNN failed; lexical only" } null } } @@ -206,11 +212,12 @@ internal class HybridSearchEngine( val denseHits = dense.await() ?: return@coroutineScope lexHits.map { FusedEntry(it.lineId, it.score, it) } val rrf = HashMap() - lexHits.forEachIndexed { rank, hit -> rrf.merge(hit.lineId, 1.0 / (RRF_K + rank + 1), Double::plus) } - denseHits.forEachIndexed { rank, hit -> rrf.merge(hit.lineId, 1.0 / (RRF_K + rank + 1), Double::plus) } + lexHits.forEachIndexed { rank, hit -> rrf[hit.lineId] = (rrf[hit.lineId] ?: 0.0) + 1.0 / (RRF_K + rank + 1) } + denseHits.forEachIndexed { rank, hit -> rrf[hit.lineId] = (rrf[hit.lineId] ?: 0.0) + 1.0 / (RRF_K + rank + 1) } val lexById = lexHits.associateBy { it.lineId } + // Ties broken by line: the same order on every platform, whatever the map's iteration order rrf.entries - .sortedByDescending { it.value } + .sortedWith(compareByDescending> { it.value }.thenBy { it.key }) .map { (lineId, score) -> FusedEntry(lineId, score.toFloat(), lexById[lineId]) } } diff --git a/searchkit/addons/semantic/enabled/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/InstalledSemanticSearch.kt b/searchkit/addons/semantic/enabled/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/InstalledSemanticSearch.kt new file mode 100644 index 000000000..4a88969bf --- /dev/null +++ b/searchkit/addons/semantic/enabled/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/InstalledSemanticSearch.kt @@ -0,0 +1,21 @@ +package io.github.kdroidfilter.seforimapp.features.search.semantic + +import io.github.kdroidfilter.seforimlibrary.search.LineHit +import io.github.kdroidfilter.seforimlibrary.search.SearchEngine +import io.github.kdroidfilter.seforimlibrary.search.index.LuceneIndex + +/** An official build: the semantic search of the SeforimEmbedding package. */ +val installedSemanticSearch: SemanticSearchFeature? = + object : SemanticSearchFeature { + override val bundledModels: SemanticModels? get() = bundledSemanticModels()?.value + + override fun hybrid( + lexical: SearchEngine, + index: LuceneIndex, + models: SemanticModels?, + resolveLine: suspend (lineId: Long, query: String) -> LineHit?, + ): SearchEngine = HybridSearchEngine(lexical, index, models?.let { lazyOf(it) } ?: bundledSemanticModels(), resolveLine) + } + +/** The model the package carries on this platform (the desktop's jar, extracted on first use), null on mobile. */ +internal expect fun bundledSemanticModels(): Lazy? diff --git a/searchkit/addons/semantic/enabled/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/LuceneVectorIndex.kt b/searchkit/addons/semantic/enabled/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/LuceneVectorIndex.kt new file mode 100644 index 000000000..894a46b6b --- /dev/null +++ b/searchkit/addons/semantic/enabled/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/LuceneVectorIndex.kt @@ -0,0 +1,67 @@ +package io.github.kdroidfilter.seforimapp.features.search.semantic + +import io.github.kdroidfilter.seforim.semantic.DenseHit +import io.github.kdroidfilter.seforim.semantic.VectorIndex +import io.github.kdroidfilter.seforimlibrary.search.index.BoolBuilder +import io.github.kdroidfilter.seforimlibrary.search.index.IndexQuery +import io.github.kdroidfilter.seforimlibrary.search.index.KnnRequest +import io.github.kdroidfilter.seforimlibrary.search.index.LuceneIndex +import io.github.kdroidfilter.seforimlibrary.search.index.Occur +import io.github.kdroidfilter.seforimlibrary.search.index.VectorEncoding + +/** + * The vectors of the app's fused Lucene index for the semantic search: its KNN runs through [index], in process on the + * JVM and Android, in the Java runtime of the iOS app. Not closed with the semantic search: the lexical one shares it. + */ +internal class LuceneVectorIndex( + private val index: LuceneIndex, +) : VectorIndex { + override fun encoding(field: String): VectorIndex.Encoding? = + when (index.vectorEncoding(field)) { + VectorEncoding.BYTE -> VectorIndex.Encoding.BYTE + VectorEncoding.FLOAT32 -> VectorIndex.Encoding.FLOAT32 + null -> null + } + + override fun knn( + field: String, + k: Int, + floatVector: FloatArray?, + byteVector: ByteArray?, + baseBookOnly: Boolean, + bookIds: Collection?, + oversample: Int, + ): List = + index + .knn( + KnnRequest( + field = field, + k = k, + fields = setOf(FIELD_LINE_ID, FIELD_BOOK_ID), + floatVector = floatVector, + byteVector = byteVector, + filter = filter(baseBookOnly, bookIds), + oversample = oversample, + ), + ).map { DenseHit(lineId = it.longs.getValue(FIELD_LINE_ID), bookId = it.longs.getValue(FIELD_BOOK_ID), score = it.score) } + + // Base books and specific books, as a pre-filter: the KNN only considers eligible lines + private fun filter( + baseBookOnly: Boolean, + bookIds: Collection?, + ): IndexQuery? { + if (!baseBookOnly && bookIds.isNullOrEmpty()) return null + val builder = BoolBuilder() + if (baseBookOnly) builder.add(IndexQuery.IntExact(FIELD_IS_BASE_BOOK, 1), Occur.FILTER) + if (!bookIds.isNullOrEmpty()) builder.add(IndexQuery.IntSet(FIELD_BOOK_ID, bookIds.map { it.toInt() }.toIntArray()), Occur.FILTER) + return builder.build() + } + + override fun close() = Unit + + private companion object { + const val FIELD_LINE_ID = "line_id" + const val FIELD_BOOK_ID = "book_id" + const val FIELD_IS_BASE_BOOK = "is_base_book" + } +} diff --git a/searchkit/addons/semantic/enabled/mobile/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/BundledSemanticModels.mobile.kt b/searchkit/addons/semantic/enabled/mobile/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/BundledSemanticModels.mobile.kt new file mode 100644 index 000000000..67d950f24 --- /dev/null +++ b/searchkit/addons/semantic/enabled/mobile/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/BundledSemanticModels.mobile.kt @@ -0,0 +1,4 @@ +package io.github.kdroidfilter.seforimapp.features.search.semantic + +// A mobile app downloads the model with the data (too large for the package): it passes its paths +internal actual fun bundledSemanticModels(): Lazy? = null diff --git a/searchkit/build.gradle.kts b/searchkit/build.gradle.kts new file mode 100644 index 000000000..dbfdb74d1 --- /dev/null +++ b/searchkit/build.gradle.kts @@ -0,0 +1,209 @@ +plugins { + alias(libs.plugins.multiplatform) + alias(libs.plugins.android.kmp.library) + alias(libs.plugins.kotlinx.serialization) + // The view models contribute their factories to the app's graph (the desktop's AppGraph) + alias(libs.plugins.metro) +} + +// The semantic search is in the official builds only (open core): with its package's token or its sources beside +// (settings.gradle.kts); a community build compiles addons/semantic/disabled and the search stays lexical +val withSemantic = gradle.extensions.extraProperties["semanticEnabled"] == true + +/** + * Zayit's search, without its UI: the engines (lexical, hybrid with the semantic search), the lookup, the use cases and + * the view models of the search screens, for the desktop app and the mobile ones (Android 15+, iOS). Lucene runs in + * process on the JVM and Android, in the Java runtime the iOS app embeds (see SeforimLibrary's IOS_RUNTIME.md). + */ +kotlin { + jvmToolchain( + libs.versions.jvmToolchain + .get() + .toInt(), + ) + + jvm() + android { + namespace = "io.github.kdroidfilter.seforimapp.searchkit" + compileSdk = 37 + // Lucene runs on ART from Android 15 only (lucene-android) + minSdk = 35 + // The search on a device or an emulator (SearchKitDump on real data, see AndroidSearchKitParityTest) + withDeviceTestBuilder { sourceSetTreeName = "test" }.configure { + instrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" + } + } + iosArm64() + iosSimulatorArm64() + + sourceSets { + commonMain { + kotlin.srcDir(if (withSemantic) "addons/semantic/enabled/kotlin" else "addons/semantic/disabled/kotlin") + dependencies { + api(libs.seforimlibrary.core) + api(libs.seforimlibrary.dao) + api(libs.seforimlibrary.search) + implementation(libs.kotlinx.coroutines.core) + implementation(libs.kotlinx.serialization.json) + implementation(libs.kotlinx.collections.immutable) + implementation(libs.kotlinx.io.core) + implementation(libs.androidx.lifecycle.viewmodel.core) + implementation(libs.androidx.lifecycle.viewmodel.savedstate) + implementation(libs.metrox.viewmodel) + api(project(":navigation")) + implementation(libs.compose.runtime) + implementation(libs.hebrew.numerals) + implementation(libs.kermit) + implementation(libs.cachemap) + implementation(libs.atomicfu) + if (withSemantic) implementation(libs.seforim.semantic) + } + } + + jvmMain.dependencies { implementation(libs.sqlite.driver) } + androidMain.dependencies { implementation(libs.sqldelight.android.driver) } + iosMain.dependencies { implementation(libs.sqldelight.native.driver) } + + // The model's location differs by platform: the desktop's jar carries it, mobile apps download it + if (withSemantic) { + jvmMain { kotlin.srcDir("addons/semantic/enabled/jvm/kotlin") } + androidMain { kotlin.srcDir("addons/semantic/enabled/mobile/kotlin") } + iosMain { kotlin.srcDir("addons/semantic/enabled/mobile/kotlin") } + } + + commonTest.dependencies { + implementation(kotlin("test")) + implementation(libs.kotlinx.coroutines.test) + } + getByName("androidDeviceTest").dependencies { + implementation(libs.androidx.test.runner) + implementation(libs.androidx.test.junit) + } + } +} + +// --- Tests on real data: the same dump on the JVM and in the iOS simulator (SearchKitDump) --- + +// The Java runtime of iOS (OpenJDK's mobile port), downloaded by SeforimLibrary's search (see its IOS_RUNTIME.md): +// build/javaRuntime/ there +val seforimLibrary = gradle.includedBuild("SeforimLibrary") + +fun javaRuntime(target: String): File = seforimLibrary.projectDir.resolve("search/build/javaRuntime/$target") + +fun javaRuntimeTask(target: String) = + seforimLibrary.task(":search:javaRuntime${target.split("-").joinToString("") { it.replaceFirstChar(Char::uppercase) }}") + +// ONNX Runtime's xcframework, for the semantic search in the iOS tests: -Pseforim.ortXcframework= +val ortXcframework: File? = providers.gradleProperty("seforim.ortXcframework").orNull?.let(::file) +val jdkStaticLibs = + listOf( + "libjvm.a", + "libjava.a", + "libnio.a", + "libnet.a", + "libzip.a", + "libjimage.a", + "libverify.a", + "libextnet.a", + "libfallbackLinker.a", + "libsyslookup.a", + ) + +// What an iOS binary links for the search: the Java runtime (force-loaded: it finds its natives by name), its libffi, +// and ONNX Runtime for the semantic search +fun org.jetbrains.kotlin.gradle.plugin.mpp.NativeBinary.linkJavaRuntime( + target: String, + ortSlice: File?, +) { + val static = javaRuntime(target).resolve("static") + linkerOpts(jdkStaticLibs.flatMap { listOf("-force_load", static.resolve(it).path) }) + linkerOpts(static.resolve("libffi.a").path, "-export_dynamic", "-lz", "-lc++") + listOf("Foundation", "CoreFoundation", "Security", "SystemConfiguration").forEach { linkerOpts("-framework", it) } + if (ortSlice != null) linkerOpts("-F", ortSlice.path, "-framework", "onnxruntime") + linkTaskProvider.configure { dependsOn(javaRuntimeTask(target)) } +} + +kotlin { + iosSimulatorArm64 { + binaries.withType().configureEach { + linkJavaRuntime("ios-simulator", ortXcframework?.resolve("ios-arm64_x86_64-simulator")) + } + } + // A device build of the tests only checks that everything links (no device runs them here) + iosArm64 { + binaries.withType().configureEach { + linkJavaRuntime("ios-device", ortXcframework?.resolve("ios-arm64")) + } + } +} + +val testDb = providers.gradleProperty("seforimTestDb") +val parityDir = layout.buildDirectory.dir("parity") + +tasks.named("jvmTest") { + testDb.orNull?.let { systemProperty("seforimTestDb", it) } + val out = parityDir.get().file("jvm.txt").asFile + systemProperty("parityOut", out.path) + testLogging.showStandardStreams = true + doFirst { out.parentFile.mkdirs() } +} + +// SeforimLibrary's jars for the embedded runtime (its Lucene side, see IOS_RUNTIME.md) +val searchJars = seforimLibrary.projectDir.resolve("search/build/iosJvmClasspath") +val searchJarsTask = seforimLibrary.task(":search:iosJvmClasspath") + +// The Java runtime an iOS app bundles as App.app/lib (its java.home): the runtime's classes, conf and data, and the +// search's jars in lib/app (IosJavaRuntime's default class path) +fun Sync.iosJavaRuntime(target: String) { + dependsOn(searchJarsTask, javaRuntimeTask(target)) + from(javaRuntime(target).resolve("lib")) + from(searchJars) { into("app") } + // Some of the JDK's files are read-only: the next sync couldn't replace them + filePermissions { user { write = true } } +} + +// For the Xcode project: copy build/iosJavaRuntime//lib into the app bundle, link build/iosJavaRuntime// +// static (see IOS_RUNTIME.md) +listOf("simulator", "device").forEach { kind -> + tasks.register("assembleIosJavaRuntime${kind.replaceFirstChar { it.uppercase() }}") { + group = "build" + description = "Assembles the Java runtime the iOS app bundles as App.app/lib, and links ($kind)." + dependsOn(searchJarsTask, javaRuntimeTask("ios-$kind")) + from(javaRuntime("ios-$kind").resolve("static")) { into("static") } + from(javaRuntime("ios-$kind").resolve("lib")) { into("lib") } + from(searchJars) { into("lib/app") } + into(layout.buildDirectory.dir("iosJavaRuntime/$kind")) + filePermissions { user { write = true } } + } +} + +run { + val testBinaryDir = layout.buildDirectory.dir("bin/iosSimulatorArm64/debugTest") + + val iosTestRuntime by tasks.registering(Sync::class) { + iosJavaRuntime("ios-simulator") + into(testBinaryDir.map { it.dir("lib") }) + } + + tasks.withType().configureEach { + dependsOn(iosTestRuntime) + device.set(providers.gradleProperty("seforim.iosSimulator").orElse("iPhone 17 Pro")) + val binaryDir = testBinaryDir.get().asFile + val jarsDir = searchJars + val jars = providers.provider { jarsDir.listFiles().orEmpty().map { binaryDir.resolve("lib/app/${it.name}").path } } + val models = providers.gradleProperty("seforim.semanticModels") + val db = providers.gradleProperty("seforimTestDb") + val out = + parityDir + .get() + .file("ios.txt") + .asFile.path + doFirst { + // The simulator passes the SIMCTL_CHILD_ variables to the test process, without the prefix + environment("SIMCTL_CHILD_SEFORIM_JVM_CLASSPATH", jars.get().joinToString(":")) + db.orNull?.let { environment("SIMCTL_CHILD_SEFORIM_TEST_DB", it) } + models.orNull?.let { environment("SIMCTL_CHILD_SEFORIM_MODEL_DIR", it) } + environment("SIMCTL_CHILD_SEFORIM_PARITY_OUT", out) + } + } +} diff --git a/searchkit/src/androidDeviceTest/kotlin/io/github/kdroidfilter/seforimapp/framework/search/AndroidSearchKitParityTest.kt b/searchkit/src/androidDeviceTest/kotlin/io/github/kdroidfilter/seforimapp/framework/search/AndroidSearchKitParityTest.kt new file mode 100644 index 000000000..62b892af7 --- /dev/null +++ b/searchkit/src/androidDeviceTest/kotlin/io/github/kdroidfilter/seforimapp/framework/search/AndroidSearchKitParityTest.kt @@ -0,0 +1,40 @@ +package io.github.kdroidfilter.seforimapp.framework.search + +import android.os.SystemClock +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import io.github.kdroidfilter.seforimapp.features.search.semantic.SemanticModels +import kotlinx.coroutines.runBlocking +import org.junit.Test +import org.junit.runner.RunWith +import java.io.File + +/** + * The app's search on Android 15+, end to end: the lookup and the hybrid search, Lucene on ART, the semantic model on + * ONNX Runtime for Android, the database on Android's SQLite. The data (seforim.db and its indexes, lexical.db, the + * model in model/) goes in the test app's files dir under seforim/ (adb root, push, chown to the app); SearchKitDump + * comes back as searchkit-android.txt there. Skipped without data. + */ +@RunWith(AndroidJUnit4::class) +class AndroidSearchKitParityTest { + @Test + fun dump() { + val context = InstrumentationRegistry.getInstrumentation().targetContext + val dir = File(context.filesDir, "seforim") + val db = File(dir, "seforim.db").takeIf { it.isFile } ?: return + InstrumentationRegistry.getArguments().getString("luceneQueryCache")?.let { System.setProperty("seforim.lucene.queryCache", it) } + val model = File(dir, "model") + val models = + SemanticModels( + File(model, "seforim-embed-v5-int8.onnx").path, + File(model, "tokenizer.json").path, + File(model, "seforim-embed-v5-static.i8").path, + ).takeIf { model.isDirectory } + SearchServices(db.path, openSeforimRepository(context, db.path), models).use { services -> + val start = SystemClock.elapsedRealtime() + val dump = runBlocking { SearchKitDump.dump(services) } + println("SearchKit dump in ${SystemClock.elapsedRealtime() - start} ms") + File(context.filesDir, "searchkit-android.txt").writeText(dump) + } + } +} diff --git a/searchkit/src/androidMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/LuceneIndexes.android.kt b/searchkit/src/androidMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/LuceneIndexes.android.kt new file mode 100644 index 000000000..53f51ce59 --- /dev/null +++ b/searchkit/src/androidMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/LuceneIndexes.android.kt @@ -0,0 +1,11 @@ +package io.github.kdroidfilter.seforimapp.framework.search + +import io.github.kdroidfilter.seforimlibrary.search.index.LuceneIndex +import io.github.kdroidfilter.seforimlibrary.search.index.LuceneIndexBackend +import java.io.File + +// Lucene on ART, in process (Android 15+; lucene-core patched for it, see SeforimLibrary's lucene-android) +actual fun openLuceneIndex( + indexDir: String, + cacheQueries: Boolean, +): LuceneIndex = LuceneIndexBackend(File(indexDir).toPath(), cacheQueries = cacheQueries) diff --git a/searchkit/src/androidMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/Repositories.android.kt b/searchkit/src/androidMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/Repositories.android.kt new file mode 100644 index 000000000..9ee98ea0b --- /dev/null +++ b/searchkit/src/androidMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/Repositories.android.kt @@ -0,0 +1,38 @@ +package io.github.kdroidfilter.seforimapp.framework.search + +import android.content.Context +import app.cash.sqldelight.db.QueryResult +import app.cash.sqldelight.db.SqlDriver +import app.cash.sqldelight.db.SqlSchema +import app.cash.sqldelight.driver.android.AndroidSqliteDriver +import io.github.kdroidfilter.seforimlibrary.dao.repository.SeforimRepository + +/** + * The repository of the database at [databasePath] (an absolute path: the downloaded database, not one the app ships). + * Its schema exists already: the driver creates and migrates nothing. + */ +fun openSeforimRepository( + context: Context, + databasePath: String, +): SeforimRepository = + SeforimRepository( + databasePath, + AndroidSqliteDriver(ExistingSchema, context, databasePath, windowSizeBytes = CURSOR_WINDOW_BYTES), + ) + +// A book's notes reach 2.5 MB: more than the 2 MB a cursor holds by default on Android +private const val CURSOR_WINDOW_BYTES = 8L * 1024 * 1024 + +// The downloaded database is complete: version 1, never created nor migrated here +private object ExistingSchema : SqlSchema> { + override val version: Long = 1 + + override fun create(driver: SqlDriver) = QueryResult.Unit + + override fun migrate( + driver: SqlDriver, + oldVersion: Long, + newVersion: Long, + vararg callbacks: app.cash.sqldelight.db.AfterVersion, + ) = QueryResult.Unit +} diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/core/coroutines/CoroutineUtils.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/core/coroutines/CoroutineUtils.kt similarity index 100% rename from SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/core/coroutines/CoroutineUtils.kt rename to searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/core/coroutines/CoroutineUtils.kt diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/core/deeplink/ZayitDeepLink.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/core/deeplink/ZayitDeepLink.kt similarity index 67% rename from SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/core/deeplink/ZayitDeepLink.kt rename to searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/core/deeplink/ZayitDeepLink.kt index effb2360d..6774c567c 100644 --- a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/core/deeplink/ZayitDeepLink.kt +++ b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/core/deeplink/ZayitDeepLink.kt @@ -1,10 +1,7 @@ package io.github.kdroidfilter.seforimapp.core.deeplink import io.github.kdroidfilter.seforim.tabs.TabsDestination -import java.net.URLDecoder -import java.net.URLEncoder -import java.nio.charset.StandardCharsets -import java.util.UUID +import io.github.kdroidfilter.seforim.tabs.newTabId /** * Public, shareable deep link scheme for Zayit. @@ -34,7 +31,7 @@ fun bookShareLink( } /** Builds a shareable link to a search query. */ -fun searchShareLink(query: String): String = PREFIX + HOST_SEARCH + "/" + URLEncoder.encode(query, StandardCharsets.UTF_8) +fun searchShareLink(query: String): String = PREFIX + HOST_SEARCH + "/" + formEncode(query) /** * Returns a shareable link for this destination, or null when there is nothing worth sharing @@ -62,7 +59,7 @@ fun parseZayitDeepLink(uri: String): TabsDestination? { val path = uri.removePrefix(PREFIX) val segments = path.split('/').filter { it.isNotEmpty() } if (segments.isEmpty()) return null - val newTabId = UUID.randomUUID().toString() + val newTabId = newTabId() return when (segments[0]) { HOST_BOOK -> { val bookId = segments.getOrNull(1)?.toLongOrNull() ?: return null @@ -77,8 +74,42 @@ fun parseZayitDeepLink(uri: String): TabsDestination? { HOST_SEARCH -> { val encoded = path.substringAfter("$HOST_SEARCH/", "") if (encoded.isEmpty()) return null - TabsDestination.Search(searchQuery = URLDecoder.decode(encoded, StandardCharsets.UTF_8), tabId = newTabId) + TabsDestination.Search(searchQuery = formDecode(encoded), tabId = newTabId) } else -> null } } + +// application/x-www-form-urlencoded in UTF-8, as java.net.URLEncoder writes it and URLDecoder reads it (the links +// already shared must keep working): letters, digits and .-*_ as is, a space as +, any other byte as %XX +private fun formEncode(text: String): String = + buildString { + for (byte in text.encodeToByteArray()) { + val c = (byte.toInt() and 0xFF).toChar() + when { + c in 'a'..'z' || c in 'A'..'Z' || c in '0'..'9' || c in ".-*_" -> append(c) + c == ' ' -> append('+') + else -> append('%').append(HEX[(byte.toInt() shr 4) and 0xF]).append(HEX[byte.toInt() and 0xF]) + } + } + } + +private fun formDecode(text: String): String { + val bytes = ArrayList(text.length) + var i = 0 + while (i < text.length) { + val c = text[i] + when { + c == '+' -> bytes += ' '.code.toByte() + c == '%' && i + 2 <= text.lastIndex -> { + bytes += text.substring(i + 1, i + 3).toInt(16).toByte() + i += 2 + } + else -> bytes += c.toString().encodeToByteArray().toList() + } + i++ + } + return bytes.toByteArray().decodeToString() +} + +private const val HEX = "0123456789ABCDEF" diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/state/StateKeys.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/state/StateKeys.kt similarity index 100% rename from SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/state/StateKeys.kt rename to searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/state/StateKeys.kt diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchHomeViewModel.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchHomeViewModel.kt similarity index 94% rename from SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchHomeViewModel.kt rename to searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchHomeViewModel.kt index 7ffff89d3..ec1c7c26c 100644 --- a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchHomeViewModel.kt +++ b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchHomeViewModel.kt @@ -3,11 +3,11 @@ package io.github.kdroidfilter.seforimapp.features.search import androidx.compose.runtime.Immutable import androidx.lifecycle.ViewModel import androidx.lifecycle.viewModelScope +import io.github.charlietap.cachemap.cacheMapOf import io.github.kdroidfilter.seforim.tabs.SearchScope import io.github.kdroidfilter.seforim.tabs.TabsDestination import io.github.kdroidfilter.seforimapp.core.coroutines.runSuspendCatching import io.github.kdroidfilter.seforimapp.core.deeplink.parseZayitDeepLink -import io.github.kdroidfilter.seforimapp.core.settings.AppSettings import io.github.kdroidfilter.seforimapp.features.search.domain.AuthorNames import io.github.kdroidfilter.seforimapp.features.search.domain.reference.ReferenceParser import io.github.kdroidfilter.seforimapp.features.search.domain.reference.ReferenceResolver @@ -17,12 +17,12 @@ import io.github.kdroidfilter.seforimapp.framework.search.LuceneLookupSearchServ import io.github.kdroidfilter.seforimapp.framework.search.LuceneLookupSearchService.AuthorHit import io.github.kdroidfilter.seforimapp.framework.search.MIN_BOOK_QUERY_LENGTH import io.github.kdroidfilter.seforimapp.framework.session.SearchPersistedState -import io.github.kdroidfilter.seforimapp.framework.session.TabPersistedStateStore import io.github.kdroidfilter.seforimlibrary.core.models.Book import io.github.kdroidfilter.seforimlibrary.core.models.TocEntry import io.github.kdroidfilter.seforimlibrary.dao.repository.SeforimRepository import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.FlowPreview +import kotlinx.coroutines.IO import kotlinx.coroutines.async import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.coroutineScope @@ -40,7 +40,6 @@ import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import kotlinx.coroutines.withContext -import java.util.concurrent.ConcurrentHashMap /** * Navigation events emitted by SearchHomeViewModel. @@ -118,10 +117,10 @@ data class SearchHomeUiState( @OptIn(FlowPreview::class) class SearchHomeViewModel( - private val persistedStore: TabPersistedStateStore, + private val sessionStore: SearchSessionStore, private val repository: SeforimRepository, private val lookup: LuceneLookupSearchService, - private val appSettings: AppSettings, + private val appSettings: SearchSettings, // The home page's greeting needs the user's profile; a results tab's bar doesn't observeProfile: Boolean = true, ) : ViewModel() { @@ -135,7 +134,7 @@ class SearchHomeViewModel( private val referenceResolver = ReferenceResolver(RepositoryReferenceSource(repository)) // Author aliases by id, to tell which one a query was typed as; few authors have any - private val aliasesByAuthor = ConcurrentHashMap>() + private val aliasesByAuthor = cacheMapOf>() private suspend fun authorAliases(authorId: Long): List = aliasesByAuthor[authorId] ?: runSuspendCatching { repository.getAuthorDetails(authorId)?.aliases.orEmpty() } @@ -155,11 +154,23 @@ class SearchHomeViewModel( private val maxBookPredictive = 120 // tighter ceiling to avoid heavy allocations private val maxTocPredictive = 300 // TOC suggestions stay bounded - // Lightweight, thread-safe LRU caches to avoid repeated DB hits when typing fast - private class LruCache( + // Lightweight LRU caches (guarded by their mutex) to avoid repeated DB hits when typing fast + private class LruCache( private val maxSize: Int, - ) : LinkedHashMap(maxSize, 0.75f, true) { - override fun removeEldestEntry(eldest: MutableMap.MutableEntry?): Boolean = size > maxSize + ) { + // Insertion order: a hit moves its entry to the end, so the first entry is the least recently used + private val map = LinkedHashMap() + + operator fun get(key: K): V? = map.remove(key)?.also { map[key] = it } + + operator fun set( + key: K, + value: V, + ) { + map.remove(key) + map[key] = value + if (map.size > maxSize) map.remove(map.keys.first()) + } } private val categoryPathCache = LruCache>(512) @@ -616,22 +627,20 @@ class SearchHomeViewModel( val selected = _uiState.value val scope = barScope() - persistedStore.update(currentTabId) { current -> - val nextSearch = - (current.search ?: SearchPersistedState()).withScope(scope).copy( - query = query, - globalExtended = selected.globalExtended, - selectedCategoryIds = emptySet(), - selectedBookIds = emptySet(), - selectedTocIds = emptySet(), - scrollIndex = 0, - scrollOffset = 0, - anchorId = -1L, - anchorIndex = 0, - snapshot = null, - breadcrumbs = emptyMap(), - ) - current.copy(search = nextSearch) + sessionStore.updateSearch(currentTabId) { current -> + (current ?: SearchPersistedState()).withScope(scope).copy( + query = query, + globalExtended = selected.globalExtended, + selectedCategoryIds = emptySet(), + selectedBookIds = emptySet(), + selectedTocIds = emptySet(), + scrollIndex = 0, + scrollOffset = 0, + anchorId = -1L, + anchorIndex = 0, + snapshot = null, + breadcrumbs = emptyMap(), + ) } // Emit navigation event - UI layer handles actual navigation @@ -664,9 +673,7 @@ class SearchHomeViewModel( } // Pre-seed minimal state so the BookContent shell can show a loader instead of flashing Home. - persistedStore.update(currentTabId) { current -> - current.copy(bookContent = current.bookContent.copy(selectedBookId = book.id)) - } + sessionStore.selectBook(currentTabId, book.id) // Emit navigation event - UI layer handles actual navigation _navigationEvents.send( @@ -683,9 +690,7 @@ class SearchHomeViewModel( book: Book, currentTabId: String, ) { - persistedStore.update(currentTabId) { current -> - current.copy(bookContent = current.bookContent.copy(selectedBookId = book.id)) - } + sessionStore.selectBook(currentTabId, book.id) _uiState.value = _uiState.value.copy(suggestionsVisible = false) _navigationEvents.send(SearchHomeNavigationEvent.NavigateToBookContent(bookId = book.id, tabId = currentTabId, lineId = null)) } @@ -706,9 +711,7 @@ class SearchHomeViewModel( jump: ResolvedReference, currentTabId: String, ) { - persistedStore.update(currentTabId) { current -> - current.copy(bookContent = current.bookContent.copy(selectedBookId = jump.book.id)) - } + sessionStore.selectBook(currentTabId, jump.book.id) _uiState.value = _uiState.value.copy(suggestionsVisible = false) _navigationEvents.send( SearchHomeNavigationEvent.NavigateToBookContent( diff --git a/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchHost.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchHost.kt new file mode 100644 index 000000000..05d73ae8a --- /dev/null +++ b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchHost.kt @@ -0,0 +1,100 @@ +package io.github.kdroidfilter.seforimapp.features.search + +import io.github.kdroidfilter.seforim.tabs.SearchScope +import io.github.kdroidfilter.seforim.tabs.TabsDestination +import io.github.kdroidfilter.seforimapp.framework.session.SearchPersistedState +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.StateFlow + +/* + * What the search screens ask of the app around them: the desktop's windows, tabs and session, a mobile app's + * navigation and storage. Each app implements these; the view models know nothing else of it. + */ + +/** The tabs (or screens) the search opens and watches. */ +interface SearchTabs { + /** Opens [destination] in a new tab, from the tab [fromTabId]. */ + fun openTab( + fromTabId: String, + destination: TabsDestination, + ) + + /** Shows [destination] in the tab [tabId] instead of what it shows. */ + fun replace( + tabId: String, + destination: TabsDestination, + ) + + /** Whether the tab [tabId] still exists, as it changes. */ + fun tabExistsFlow(tabId: String): Flow + + fun isTabOpen(tabId: String): Boolean +} + +/** What the tabs keep across sessions, as far as the search is concerned. */ +interface SearchSessionStore { + /** The search state of the tab [tabId], null when it has none. */ + fun search(tabId: String): SearchPersistedState? + + /** Replaces the search state of the tab [tabId] by [transform]'s (null: the tab has none any more). */ + fun updateSearch( + tabId: String, + transform: (SearchPersistedState?) -> SearchPersistedState?, + ) + + /** Makes [bookId] the book the tab [tabId] shows. */ + fun selectBook( + tabId: String, + bookId: Long, + ) + + /** + * Prepares the tab [tabId] to open [bookId] at [lineId]: the line selected and anchored, the table of contents + * shown, as a found line opens. + */ + fun openBookAtLine( + tabId: String, + bookId: Long, + lineId: Long, + ) +} + +/** The app settings the search reads and sets. */ +interface SearchSettings { + val textSizeFlow: StateFlow + + fun getTextSize(): Float + + /** Pre-fills the find-in-page bar of the tab [tabId] with [q]. */ + fun setFindQuery( + tabId: String, + q: String, + ) + + fun setFindSmartMode( + tabId: String, + enabled: Boolean, + ) + + fun openFindBar(tabId: String) + + val userFirstNameFlow: StateFlow + val userLastNameFlow: StateFlow + val userCommunityCodeFlow: StateFlow + + companion object { + /** The text size before the user changes it. */ + const val DEFAULT_TEXT_SIZE = 16f + } +} + +/** The history of what was searched. */ +interface SearchHistory { + suspend fun recordSearchVisit( + query: String, + scope: SearchScope, + globalExtended: Boolean, + scopeLabel: String?, + timestamp: Long, + ) +} diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchResultViewModel.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchResultViewModel.kt similarity index 95% rename from SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchResultViewModel.kt rename to searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchResultViewModel.kt index 328ee5c76..bac687572 100644 --- a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchResultViewModel.kt +++ b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchResultViewModel.kt @@ -16,8 +16,6 @@ import dev.zacsweers.metrox.viewmodel.ViewModelAssistedFactory import dev.zacsweers.metrox.viewmodel.ViewModelAssistedFactoryKey import io.github.kdroidfilter.seforim.tabs.* import io.github.kdroidfilter.seforimapp.core.coroutines.runSuspendCatching -import io.github.kdroidfilter.seforimapp.core.history.HistoryStore -import io.github.kdroidfilter.seforimapp.core.settings.AppSettings import io.github.kdroidfilter.seforimapp.features.bookcontent.state.StateKeys import io.github.kdroidfilter.seforimapp.features.search.domain.AuthorNames import io.github.kdroidfilter.seforimapp.features.search.domain.BookDetailsLoader @@ -33,11 +31,9 @@ import io.github.kdroidfilter.seforimapp.features.search.domain.SearchStatePersi import io.github.kdroidfilter.seforimapp.features.search.domain.SearchTocUseCase import io.github.kdroidfilter.seforimapp.features.search.domain.TocLineIndex import io.github.kdroidfilter.seforimapp.features.search.domain.TocTree -import io.github.kdroidfilter.seforimapp.framework.desktop.DesktopManager import io.github.kdroidfilter.seforimapp.framework.di.AppScope import io.github.kdroidfilter.seforimapp.framework.search.LuceneLookupSearchService import io.github.kdroidfilter.seforimapp.framework.session.SearchPersistedState -import io.github.kdroidfilter.seforimapp.framework.session.TabPersistedStateStore import io.github.kdroidfilter.seforimlibrary.core.models.Book import io.github.kdroidfilter.seforimlibrary.core.models.Category import io.github.kdroidfilter.seforimlibrary.core.models.Line @@ -59,8 +55,8 @@ import kotlinx.coroutines.flow.* import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import kotlinx.serialization.json.Json -import java.util.UUID import kotlin.collections.ArrayDeque +import kotlin.time.Clock private const val LAZY_PAGE_SIZE = 25 @@ -87,7 +83,7 @@ data class SearchUiState( val anchorId: Long = -1L, val anchorIndex: Int = 0, val scrollToAnchorTimestamp: Long = 0L, - val textSize: Float = AppSettings.DEFAULT_TEXT_SIZE, + val textSize: Float = SearchSettings.DEFAULT_TEXT_SIZE, val hasMore: Boolean = false, val isLoadingMore: Boolean = false, val progressCurrent: Int = 0, @@ -97,13 +93,13 @@ data class SearchUiState( @AssistedInject class SearchResultViewModel( @Assisted savedStateHandle: SavedStateHandle, - private val persistedStore: TabPersistedStateStore, + private val sessionStore: SearchSessionStore, private val repository: SeforimRepository, private val lucene: SearchEngine, private val titleUpdateManager: TabTitleUpdateManager, - private val desktopManager: DesktopManager, - private val historyStore: HistoryStore, - private val appSettings: AppSettings, + private val tabs: SearchTabs, + private val historyStore: SearchHistory, + private val appSettings: SearchSettings, lookup: LuceneLookupSearchService, ) : ViewModel() { @AssistedFactory @@ -117,13 +113,10 @@ class SearchResultViewModel( internal val tabId: String = savedStateHandle.get(StateKeys.TAB_ID) ?: "" - private fun persistedSearchState(): SearchPersistedState = persistedStore.get(tabId)?.search ?: SearchPersistedState() + private fun persistedSearchState(): SearchPersistedState = sessionStore.search(tabId) ?: SearchPersistedState() private fun updatePersistedSearch(transform: (SearchPersistedState) -> SearchPersistedState) { - persistedStore.update(tabId) { current -> - val next = transform(current.search ?: SearchPersistedState()) - current.copy(search = next) - } + sessionStore.updateSearch(tabId) { current -> transform(current ?: SearchPersistedState()) } } private fun seedNavigationScope(savedStateHandle: SavedStateHandle) { @@ -440,13 +433,11 @@ class SearchResultViewModel( .drop(1) // ignore initial state on first subscription .flatMapLatest { // Show overlay until visible results emission changes either size or identity - val initial = Pair(visibleResultsFlow.value.size, System.identityHashCode(visibleResultsFlow.value)) + val initial = visibleResultsFlow.value flow { emit(true) visibleResultsFlow - .map { Pair(it.size, System.identityHashCode(it)) } - .distinctUntilChanged() - .filter { it != initial } + .filter { it !== initial } .first() emit(false) } @@ -501,14 +492,12 @@ class SearchResultViewModel( bookId: Long, lineId: Long, ) { - desktopManager.tabsViewModelFor(tabId)?.openTab( - TabsDestination.BookContent(bookId = bookId, tabId = UUID.randomUUID().toString(), lineId = lineId), - ) + tabs.openTab(tabId, TabsDestination.BookContent(bookId = bookId, tabId = newTabId(), lineId = lineId)) } /** Opens an author's page from the panel, in a new tab. */ fun openAuthor(authorId: Long) { - desktopManager.tabsViewModelFor(tabId)?.openTab(TabsDestination.Author(tabId = UUID.randomUUID().toString(), authorId = authorId)) + tabs.openTab(tabId, TabsDestination.Author(tabId = newTabId(), authorId = authorId)) } private val _searchTree = MutableStateFlow>(persistentListOf()) @@ -787,7 +776,7 @@ class SearchResultViewModel( progressCurrent = cached.results.size, progressTotal = cached.totalHits.takeIf { it > 0 } ?: cached.results.size.toLong(), // trigger scroll restoration once items are present - scrollToAnchorTimestamp = System.currentTimeMillis(), + scrollToAnchorTimestamp = Clock.System.now().toEpochMilliseconds(), ) // Re-open a Lucene session if there are more results to load if (cached.hasMore && initialQuery.isNotBlank()) { @@ -881,7 +870,7 @@ class SearchResultViewModel( viewModelScope.launch { // Observe all windows' tabs and cancel search if this tab gets closed everywhere - desktopManager.tabExistsFlow(tabId).collect { exists -> + tabs.tabExistsFlow(tabId).collect { exists -> if (!exists) { // Tab was closed; stop work. cancelSearch() @@ -933,7 +922,7 @@ class SearchResultViewModel( val persisted = persistedSearchState() val scope = persisted.scope viewModelScope.launch { - historyStore.recordSearchVisit(q, scope, persisted.globalExtended, scopeLabel(scope), System.currentTimeMillis()) + historyStore.recordSearchVisit(q, scope, persisted.globalExtended, scopeLabel(scope), Clock.System.now().toEpochMilliseconds()) } // New search: clear any previous streaming job and reset scroll/anchor state currentJob?.cancel() @@ -956,7 +945,7 @@ class SearchResultViewModel( scrollOffset = 0, anchorId = -1L, anchorIndex = 0, - scrollToAnchorTimestamp = System.currentTimeMillis(), + scrollToAnchorTimestamp = Clock.System.now().toEpochMilliseconds(), ) currentJob = viewModelScope.launch(Dispatchers.Default) { @@ -1123,12 +1112,10 @@ class SearchResultViewModel( // but never block indefinitely (important when final results are empty and identical // to the pre-stream empty list reference). runSuspendCatching { - val initial = Pair(visibleResultsFlow.value.size, System.identityHashCode(visibleResultsFlow.value)) + val initial = visibleResultsFlow.value withTimeoutOrNull(300) { visibleResultsFlow - .map { Pair(it.size, System.identityHashCode(it)) } - .distinctUntilChanged() - .filter { it != initial } + .filter { it !== initial } .first() } } @@ -1282,7 +1269,7 @@ class SearchResultViewModel( // without re-searching. val stillExists = runCatching { - desktopManager.isTabOpen(tabId) + tabs.isTabOpen(tabId) }.getOrDefault(false) if (stillExists) { val snap = buildSnapshot(uiState.value.results) @@ -1357,7 +1344,7 @@ class SearchResultViewModel( scopeTocId = null, scrollIndex = 0, scrollOffset = 0, - scrollToAnchorTimestamp = System.currentTimeMillis(), + scrollToAnchorTimestamp = Clock.System.now().toEpochMilliseconds(), ) if (book != null && currentTocBookId != book.id) { val tree = runSuspendCatching { buildTocTreeForBook(book.id) }.getOrNull() @@ -1398,7 +1385,7 @@ class SearchResultViewModel( scopeTocId = tocId, scrollIndex = 0, scrollOffset = 0, - scrollToAnchorTimestamp = System.currentTimeMillis(), + scrollToAnchorTimestamp = Clock.System.now().toEpochMilliseconds(), ) if (scopeBook != null && currentTocBookId != scopeBook.id) { val tree = runSuspendCatching { buildTocTreeForBook(scopeBook.id) }.getOrNull() @@ -1846,7 +1833,7 @@ class SearchResultViewModel( progressCurrent = results.size, scrollIndex = 0, scrollOffset = 0, - scrollToAnchorTimestamp = System.currentTimeMillis(), + scrollToAnchorTimestamp = Clock.System.now().toEpochMilliseconds(), ) refineSnippets(session, firstPage.hits) } finally { @@ -1904,41 +1891,20 @@ class SearchResultViewModel( fun openResult(result: SearchResult) { viewModelScope.launch { - val newTabId = UUID.randomUUID().toString() + val bookTabId = newTabId() // Seed persisted state so the new tab can restore scroll/anchor deterministically. - persistedStore.update(newTabId) { current -> - current.copy( - bookContent = - current.bookContent.copy( - selectedBookId = result.bookId, - selectedLineIds = setOf(result.lineId), - primarySelectedLineId = result.lineId, - isTocEntrySelection = false, - contentAnchorLineId = result.lineId, - contentAnchorIndex = 0, - contentScrollIndex = 0, - contentScrollOffset = 0, - isTocVisible = true, - ), - search = null, - ) - } + sessionStore.openBookAtLine(bookTabId, result.bookId, result.lineId) + sessionStore.updateSearch(bookTabId) { null } // Pre-configure find-in-page with the search query and smart mode enabled val searchQuery = _uiState.value.query if (searchQuery.length >= 2) { - appSettings.setFindQuery(newTabId, searchQuery) - appSettings.setFindSmartMode(newTabId, true) - appSettings.openFindBar(newTabId) + appSettings.setFindQuery(bookTabId, searchQuery) + appSettings.setFindSmartMode(bookTabId, true) + appSettings.openFindBar(bookTabId) } - desktopManager.tabsViewModelFor(tabId)?.openTab( - TabsDestination.BookContent( - bookId = result.bookId, - tabId = newTabId, - lineId = result.lineId, - ), - ) + tabs.openTab(tabId, TabsDestination.BookContent(bookId = result.bookId, tabId = bookTabId, lineId = result.lineId)) } } @@ -1956,22 +1922,7 @@ class SearchResultViewModel( return } viewModelScope.launch { - persistedStore.update(tabId) { current -> - current.copy( - bookContent = - current.bookContent.copy( - selectedBookId = result.bookId, - selectedLineIds = setOf(result.lineId), - primarySelectedLineId = result.lineId, - isTocEntrySelection = false, - contentAnchorLineId = result.lineId, - contentAnchorIndex = 0, - contentScrollIndex = 0, - contentScrollOffset = 0, - isTocVisible = true, - ), - ) - } + sessionStore.openBookAtLine(tabId, result.bookId, result.lineId) // Pre-configure find-in-page with the search query and smart mode enabled val searchQuery = _uiState.value.query @@ -1982,7 +1933,8 @@ class SearchResultViewModel( } // Swap current tab destination to BookContent while preserving tabId - desktopManager.tabsViewModelFor(tabId)?.replaceCurrentTabDestination( + tabs.replace( + tabId, TabsDestination.BookContent( bookId = result.bookId, tabId = tabId, @@ -2070,4 +2022,4 @@ private const val CONTEXT_MAX_LINES = 30 private const val CONTEXT_CHARS = 700 // An HTML tag, to read a line or snippet as plain text -internal val HTML_TAG = Regex("<[^>]+>") +val HTML_TAG = Regex("<[^>]+>") diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchTabCache.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchTabCache.kt similarity index 100% rename from SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchTabCache.kt rename to searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchTabCache.kt diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/AuthorNames.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/AuthorNames.kt similarity index 100% rename from SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/AuthorNames.kt rename to searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/AuthorNames.kt diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/BookDetailsLoader.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/BookDetailsLoader.kt similarity index 92% rename from SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/BookDetailsLoader.kt rename to searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/BookDetailsLoader.kt index 454b0cf4f..dad27197f 100644 --- a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/BookDetailsLoader.kt +++ b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/BookDetailsLoader.kt @@ -1,10 +1,10 @@ package io.github.kdroidfilter.seforimapp.features.search.domain +import co.touchlab.kermit.Logger +import io.github.charlietap.cachemap.cacheMapOf import io.github.kdroidfilter.seforimapp.core.coroutines.runSuspendCatching -import io.github.kdroidfilter.seforimapp.logger.warnln import io.github.kdroidfilter.seforimlibrary.core.models.Book import io.github.kdroidfilter.seforimlibrary.dao.repository.SeforimRepository -import java.util.concurrent.ConcurrentHashMap /** * A book's details as the book-details pane shows them (categories, authors with their years, main @@ -13,12 +13,12 @@ import java.util.concurrent.ConcurrentHashMap class BookDetailsLoader( private val repository: SeforimRepository, ) { - private val cache = ConcurrentHashMap() + private val cache = cacheMapOf() /** A book with its categories, authors and main parts, once per book; null (logged) when it fails. */ suspend fun load(bookId: Long): SearchEntity.BookEntity? = cache[bookId] ?: runSuspendCatching { describe(bookId) } - .onFailure { warnln(it) { "Book details of $bookId failed to load" } } + .onFailure { Logger.withTag("BookDetailsLoader").w(it) { "Book details of $bookId failed to load" } } .getOrNull() ?.also { cache[bookId] = it } diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/BuildSearchTreeUseCase.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/BuildSearchTreeUseCase.kt similarity index 100% rename from SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/BuildSearchTreeUseCase.kt rename to searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/BuildSearchTreeUseCase.kt diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/CategoryNavigationUseCase.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/CategoryNavigationUseCase.kt similarity index 100% rename from SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/CategoryNavigationUseCase.kt rename to searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/CategoryNavigationUseCase.kt diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/ExecuteSearchUseCase.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/ExecuteSearchUseCase.kt similarity index 100% rename from SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/ExecuteSearchUseCase.kt rename to searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/ExecuteSearchUseCase.kt diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/GetBreadcrumbPiecesUseCase.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/GetBreadcrumbPiecesUseCase.kt similarity index 100% rename from SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/GetBreadcrumbPiecesUseCase.kt rename to searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/GetBreadcrumbPiecesUseCase.kt diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/ResultsIndexingUseCase.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/ResultsIndexingUseCase.kt similarity index 94% rename from SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/ResultsIndexingUseCase.kt rename to searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/ResultsIndexingUseCase.kt index fe0138438..2abd4412e 100644 --- a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/ResultsIndexingUseCase.kt +++ b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/ResultsIndexingUseCase.kt @@ -10,12 +10,12 @@ import kotlinx.coroutines.sync.withLock /** * Index structure for fast filtering of search results. * - * @property identity Identity hash code of the results list this index was built from + * @property identity the results list this index was built from, compared by identity (===) * @property bookToIndices Map from book ID to sorted array of result indices * @property lineIdToIndex Map from line ID to result index */ data class ResultsIndex( - val identity: Int, + val identity: Any, val bookToIndices: Map, val lineIdToIndex: Map, ) @@ -31,16 +31,16 @@ data class ResultsIndex( * - Parallel processing for large result sets * * The use case is stateful - it maintains a cached index that is invalidated when - * the results list changes (detected by identity hash code). + * the results list changes (detected by identity). */ class ResultsIndexingUseCase { private val indexMutex = Mutex() - @Volatile + @kotlin.concurrent.Volatile private var cachedIndex: ResultsIndex? = null // TOC indices cache - invalidated when results change - private var tocIndicesIdentity: Int = -1 + private var tocIndicesIdentity: Any? = null private val tocIndicesCache: MutableMap = mutableMapOf() /** @@ -51,14 +51,14 @@ class ResultsIndexingUseCase { * @return The built index */ suspend fun ensureIndex(results: List): ResultsIndex { - val identity = System.identityHashCode(results) + val identity: Any = results val current = cachedIndex - if (current != null && current.identity == identity) return current + if (current != null && current.identity === identity) return current indexMutex.withLock { // Double-check after acquiring lock val cur2 = cachedIndex - if (cur2 != null && cur2.identity == identity) return cur2 + if (cur2 != null && cur2.identity === identity) return cur2 val lineMap = HashMap(results.size * 2) @@ -121,7 +121,7 @@ class ResultsIndexingUseCase { index: ResultsIndex, ): IntArray { // Invalidate cache if results changed - if (tocIndicesIdentity != index.identity) { + if (tocIndicesIdentity !== index.identity) { tocIndicesIdentity = index.identity tocIndicesCache.clear() } diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/SearchEntityFinder.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/SearchEntityFinder.kt similarity index 100% rename from SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/SearchEntityFinder.kt rename to searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/SearchEntityFinder.kt diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/SearchKey.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/SearchKey.kt similarity index 92% rename from SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/SearchKey.kt rename to searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/SearchKey.kt index 3e33058d9..7f090792d 100644 --- a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/SearchKey.kt +++ b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/SearchKey.kt @@ -5,7 +5,7 @@ private val QUOTES = Regex("[\"'״׳]") private val SPACES = Regex("\\s+") /** A name as typed, to compare: without nikud, quotes and extra spaces (`שו"ע` = `שוע`). */ -internal fun String.searchKey(): String = +fun String.searchKey(): String = // The maqaf joins words (שולחן־ערוך): a space, before the nikud range (which holds it) goes replace('\u05BE', ' ') .replace(NIKUD, "") diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/SearchStatePersistenceUseCase.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/SearchStatePersistenceUseCase.kt similarity index 100% rename from SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/SearchStatePersistenceUseCase.kt rename to searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/SearchStatePersistenceUseCase.kt diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/SearchTocUseCase.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/SearchTocUseCase.kt similarity index 98% rename from SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/SearchTocUseCase.kt rename to searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/SearchTocUseCase.kt index ae75f8ca4..a33652792 100644 --- a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/SearchTocUseCase.kt +++ b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/SearchTocUseCase.kt @@ -7,7 +7,6 @@ import kotlinx.collections.immutable.ImmutableList import kotlinx.collections.immutable.toImmutableList import kotlinx.coroutines.currentCoroutineContext import kotlinx.coroutines.ensureActive -import java.util.Arrays /** * Use case for TOC-related operations in search. @@ -228,10 +227,10 @@ data class TocLineIndex( val out = LongArray(total) var pos = 0 for (a in arrays) { - System.arraycopy(a, 0, out, pos, a.size) + a.copyInto(out, pos) pos += a.size } - Arrays.parallelSort(out) + out.sort() var write = 0 var i = 0 while (i < out.size) { diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/reference/ReferenceParser.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/reference/ReferenceParser.kt similarity index 100% rename from SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/reference/ReferenceParser.kt rename to searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/reference/ReferenceParser.kt diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/reference/ReferenceResolver.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/reference/ReferenceResolver.kt similarity index 100% rename from SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/reference/ReferenceResolver.kt rename to searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/reference/ReferenceResolver.kt diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/reference/RepositoryReferenceSource.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/reference/RepositoryReferenceSource.kt similarity index 81% rename from SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/reference/RepositoryReferenceSource.kt rename to searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/reference/RepositoryReferenceSource.kt index 6c29052cd..8792fac86 100644 --- a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/reference/RepositoryReferenceSource.kt +++ b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/reference/RepositoryReferenceSource.kt @@ -3,6 +3,8 @@ package io.github.kdroidfilter.seforimapp.features.search.domain.reference import io.github.kdroidfilter.seforimlibrary.core.models.Book import io.github.kdroidfilter.seforimlibrary.core.models.Line import io.github.kdroidfilter.seforimlibrary.dao.repository.SeforimRepository +import kotlinx.atomicfu.locks.SynchronizedObject +import kotlinx.atomicfu.locks.synchronized /** * [ReferenceSource] over the seforim database. The TOCs of the last books looked up are kept, @@ -43,18 +45,22 @@ class RepositoryReferenceSource( private class LruCache( private val maxSize: Int, ) { - private val map = - object : LinkedHashMap(maxSize, 0.75f, true) { - override fun removeEldestEntry(eldest: MutableMap.MutableEntry?): Boolean = size > maxSize - } + private val lock = SynchronizedObject() + + // Insertion order: a hit moves its entry to the end, so the first entry is the least recently used + private val map = LinkedHashMap() suspend fun getOrPut( key: K, load: suspend () -> V, ): V { - synchronized(map) { map[key] }?.let { return it } + synchronized(lock) { map.remove(key)?.also { map[key] = it } }?.let { return it } val value = load() - synchronized(map) { map[key] = value } + synchronized(lock) { + map.remove(key) + map[key] = value + if (map.size > maxSize) map.remove(map.keys.first()) + } return value } } diff --git a/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/SemanticSearchFeature.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/SemanticSearchFeature.kt new file mode 100644 index 000000000..3a16aceda --- /dev/null +++ b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/SemanticSearchFeature.kt @@ -0,0 +1,37 @@ +package io.github.kdroidfilter.seforimapp.features.search.semantic + +import io.github.kdroidfilter.seforimlibrary.search.LineHit +import io.github.kdroidfilter.seforimlibrary.search.SearchEngine +import io.github.kdroidfilter.seforimlibrary.search.index.LuceneIndex + +/** + * The files of the semantic search's model: the ONNX model, its tokenizer and its static distillation. The desktop's + * semantic package carries them; a mobile app downloads them with the data and passes their paths. + */ +data class SemanticModels( + val model: String, + val tokenizer: String, + val staticTable: String, +) + +/** + * The semantic search (embedding model + dense search), in Zayit's official builds only (open core): its engine is in + * addons/semantic/enabled, built with the SeforimEmbedding package; a community build compiles addons/semantic/disabled + * instead, where [installedSemanticSearch] is null and the search stays lexical. + */ +interface SemanticSearchFeature { + /** The model the package carries (the desktop's), null where the app provides it (mobile). */ + val bundledModels: SemanticModels? + + /** + * Fuses [lexical] with dense search over the vectors of [index] (the fused Lucene index), with the model of + * [models] ([bundledModels] when null); it degrades to [lexical] when the model or the vectors are missing. + * [resolveLine] turns a dense-only hit into a full [LineHit]. + */ + fun hybrid( + lexical: SearchEngine, + index: LuceneIndex, + models: SemanticModels? = null, + resolveLine: suspend (lineId: Long, query: String) -> LineHit?, + ): SearchEngine +} diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/di/AppScope.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/framework/di/AppScope.kt similarity index 100% rename from SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/di/AppScope.kt rename to searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/framework/di/AppScope.kt diff --git a/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/AcronymFrequencyCache.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/AcronymFrequencyCache.kt new file mode 100644 index 000000000..29cb5dc4e --- /dev/null +++ b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/AcronymFrequencyCache.kt @@ -0,0 +1,117 @@ +package io.github.kdroidfilter.seforimapp.framework.search + +import co.touchlab.kermit.Logger +import io.github.charlietap.cachemap.cacheMapOf +import io.github.kdroidfilter.seforimlibrary.search.SqliteReader +import io.github.kdroidfilter.seforimlibrary.search.openReadOnlySqlite + +/** + * Cache for acronym frequency data from seforim.db. + * Provides fast lookup of how many books use each acronym. + * + * Uses the book_acronym table from seforim.db - no separate database needed! + */ +class AcronymFrequencyCache( + private val databasePath: () -> String, +) { + private val logger = Logger.withTag("AcronymFrequencyCache") + + private val db: SqliteReader by lazy { + val dbPath = databasePath() + logger.i { "Loading acronym data from seforim.db: $dbPath" } + openReadOnlySqlite(dbPath) + } + + // Cache: acronym → frequency (number of books using it) + private val frequencyCache = cacheMapOf() + + // Cache: acronym → list of book titles + private val bookTitlesCache = cacheMapOf>() + + // Cache: book title → list of acronyms + private val titleToAcronymsCache = cacheMapOf>() + + init { + // Pre-load common acronyms into cache + preloadCache() + } + + /** + * Get the frequency (number of books) for an acronym. + */ + fun getFrequency(acronym: String): Int = frequencyCache[acronym] ?: queryFrequency(acronym).also { frequencyCache[acronym] = it } + + /** + * Get all book titles that use this acronym. + */ + fun getBookTitles(acronym: String): List = + bookTitlesCache[acronym] ?: queryBookTitles(acronym).also { bookTitlesCache[acronym] = it } + + /** + * Check if an acronym is unique (used by only one book). + */ + fun isUnique(acronym: String): Boolean = getFrequency(acronym) == 1 + + /** + * Get all acronyms for a given book title. + */ + fun getAcronymsForBook(bookTitle: String): List = + titleToAcronymsCache[bookTitle] ?: queryAcronymsForTitle(bookTitle).also { titleToAcronymsCache[bookTitle] = it } + + private fun queryFrequency(acronym: String): Int = + db + .query("SELECT COUNT(DISTINCT bookId) as frequency FROM book_acronym WHERE term = ?", listOf(acronym)) + .firstOrNull() + ?.get("frequency") + ?.toIntOrNull() ?: 0 + + private fun queryBookTitles(acronym: String): List = + db + .query( + """ + SELECT DISTINCT b.title + FROM book_acronym ba + JOIN book b ON ba.bookId = b.id + WHERE ba.term = ? + """.trimIndent(), + listOf(acronym), + ).mapNotNull { it["title"] } + + private fun queryAcronymsForTitle(bookTitle: String): List = + db + .query( + """ + SELECT DISTINCT ba.term + FROM book b + JOIN book_acronym ba ON b.id = ba.bookId + WHERE b.title = ? + """.trimIndent(), + listOf(bookTitle), + ).mapNotNull { it["term"] } + + private fun preloadCache() { + // Pre-load top 100 most common acronyms + val rows = + db.query( + """ + SELECT + term, + COUNT(DISTINCT bookId) as frequency + FROM book_acronym + GROUP BY term + ORDER BY frequency DESC + LIMIT 100 + """.trimIndent(), + ) + for (row in rows) { + val acronym = row["term"] ?: continue + frequencyCache[acronym] = row["frequency"]?.toIntOrNull() ?: continue + } + + logger.i { "Pre-loaded ${rows.size} acronyms into cache from seforim.db" } + } + + fun close() { + db.close() + } +} diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/LuceneLookupSearchService.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/LuceneLookupSearchService.kt similarity index 77% rename from SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/LuceneLookupSearchService.kt rename to searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/LuceneLookupSearchService.kt index d6d91283c..eb1487168 100644 --- a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/LuceneLookupSearchService.kt +++ b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/LuceneLookupSearchService.kt @@ -2,63 +2,27 @@ package io.github.kdroidfilter.seforimapp.framework.search import io.github.kdroidfilter.seforimapp.features.search.domain.AuthorNames import io.github.kdroidfilter.seforimlibrary.core.models.Book -import org.apache.lucene.analysis.Analyzer -import org.apache.lucene.analysis.standard.StandardAnalyzer -import org.apache.lucene.index.IndexReader -import org.apache.lucene.index.Term -import org.apache.lucene.search.BooleanClause -import org.apache.lucene.search.BooleanQuery -import org.apache.lucene.search.IndexSearcher -import org.apache.lucene.search.LRUQueryCache -import org.apache.lucene.search.PrefixQuery -import org.apache.lucene.search.SearcherFactory -import org.apache.lucene.search.SearcherManager -import org.apache.lucene.search.Sort -import org.apache.lucene.search.SortField -import org.apache.lucene.search.TermQuery -import org.apache.lucene.search.UsageTrackingQueryCachingPolicy -import org.apache.lucene.store.FSDirectory -import org.apache.lucene.store.NIOFSDirectory -import java.nio.file.Path - +import io.github.kdroidfilter.seforimlibrary.search.index.BoolBuilder +import io.github.kdroidfilter.seforimlibrary.search.index.IndexDoc +import io.github.kdroidfilter.seforimlibrary.search.index.IndexQuery +import io.github.kdroidfilter.seforimlibrary.search.index.LuceneIndex +import io.github.kdroidfilter.seforimlibrary.search.index.Occur +import io.github.kdroidfilter.seforimlibrary.search.index.SortField + +/** + * Book, author and table-of-contents lookup over the lookup index (seforim.db.lookup.lucene), with acronym-aware + * ranking of the books: [index] is that index (opened with its query cache, see `LuceneIndexBackend`), read in process on + * the JVM and Android, through the embedded Java runtime on iOS. + */ class LuceneLookupSearchService( - indexDir: Path, - private val analyzer: Analyzer = StandardAnalyzer(), + private val index: LuceneIndex, private val acronymCache: AcronymFrequencyCache? = null, ) { - // Open Lucene directory lazily to avoid any I/O at app startup. - // GraalVM native image does not support MMapDirectory (Panama foreign downcalls), use NIOFSDirectory instead. - private val dir by lazy { - if (System.getProperty("org.graalvm.nativeimage.imagecode") != null) { - NIOFSDirectory(indexDir) - } else { - FSDirectory.open(indexDir) - } - } - - // Cache searchers; refresh explicitly when index changes - private val searcherManager by lazy { - SearcherManager( - dir, - object : SearcherFactory() { - override fun newSearcher( - reader: IndexReader, - previousReader: IndexReader?, - ): IndexSearcher { - val s = IndexSearcher(reader) - s.setQueryCache(LRUQueryCache(1024, 64L * 1024 * 1024)) // up to 64MiB cache - s.setQueryCachingPolicy(UsageTrackingQueryCachingPolicy()) - return s - } - }, - ) - } - - private val authorSort = Sort(SortField("book_count", SortField.Type.LONG, true)) + private val authorSort = listOf(SortField("book_count", SortField.Type.LONG, reverse = true)) private val bookSort = - Sort( - SortField("is_base_book", SortField.Type.INT, true), + listOf( + SortField("is_base_book", SortField.Type.INT, reverse = true), SortField("order_index", SortField.Type.INT), SortField("book_id", SortField.Type.LONG), ) @@ -113,19 +77,23 @@ class LuceneLookupSearchService( } fun close() { - kotlin.runCatching { searcherManager.close() } - kotlin.runCatching { dir.close() } + runCatching { index.close() } } - private inline fun withSearcher(block: (IndexSearcher) -> T): T { - // Index is static during app run; refresh is cheap if no changes - searcherManager.maybeRefresh() - val searcher = searcherManager.acquire() - return try { - block(searcher) - } finally { - searcherManager.release(searcher) - } + // Documents of [type] whose words start with each of [tokens], sorted by [sort] + private fun prefixSearch( + type: String, + tokens: List, + limit: Int, + sort: List, + fields: Set, + ): List { + val query = + BoolBuilder() + .add(IndexQuery.Term("type", type), Occur.FILTER) + .apply { tokens.forEach { t -> add(IndexQuery.Prefix("q", t), Occur.MUST) } } + .build() + return index.search(query, limit, fields, sort).docs } fun searchBooksPrefix( @@ -136,22 +104,14 @@ class LuceneLookupSearchService( if (q.isBlank()) return emptyList() val tokens = q.split("\\s+".toRegex()).map { it.trim() }.filter { it.isNotEmpty() } if (tokens.isEmpty()) return emptyList() - return withSearcher { searcher -> - val b = BooleanQuery.Builder() - b.add(TermQuery(Term("type", "book")), BooleanClause.Occur.FILTER) - tokens.forEach { t -> b.add(PrefixQuery(Term("q", t)), BooleanClause.Occur.MUST) } - val top = searcher.search(b.build(), limit, bookSort) - val stored = searcher.storedFields() - top.scoreDocs.map { sd -> - val doc = stored.document(sd.doc) - BookHit( - id = doc.getField("book_id").numericValue().toLong(), - categoryId = doc.getField("category_id").numericValue().toLong(), - title = doc.getField("book_title").stringValue(), - isBaseBook = doc.getField("is_base_book")?.numericValue()?.toInt() == 1, - orderIndex = doc.getField("order_index")?.numericValue()?.toInt() ?: Int.MAX_VALUE, - ) - } + return prefixSearch("book", tokens, limit, bookSort, BOOK_FIELDS).map { doc -> + BookHit( + id = doc.longs.getValue("book_id"), + categoryId = doc.longs.getValue("category_id"), + title = doc.strings.getValue("book_title"), + isBaseBook = doc.longs["is_base_book"]?.toInt() == 1, + orderIndex = doc.longs["order_index"]?.toInt() ?: Int.MAX_VALUE, + ) } } @@ -190,20 +150,12 @@ class LuceneLookupSearchService( ): List { val tokens = normalized.split("\\s+".toRegex()).filter { it.isNotEmpty() } if (tokens.isEmpty()) return emptyList() - return withSearcher { searcher -> - val query = BooleanQuery.Builder() - query.add(TermQuery(Term("type", "author")), BooleanClause.Occur.FILTER) - tokens.forEach { t -> query.add(PrefixQuery(Term("q", t)), BooleanClause.Occur.MUST) } - val top = searcher.search(query.build(), limit, authorSort) - val stored = searcher.storedFields() - top.scoreDocs.map { sd -> - val doc = stored.document(sd.doc) - AuthorHit( - id = doc.getField("author_id").numericValue().toLong(), - name = doc.getField("author_name").stringValue(), - bookCount = doc.getField("book_count").numericValue().toInt(), - ) - } + return prefixSearch("author", tokens, limit, authorSort, AUTHOR_FIELDS).map { doc -> + AuthorHit( + id = doc.longs.getValue("author_id"), + name = doc.strings.getValue("author_name"), + bookCount = doc.longs.getValue("book_count").toInt(), + ) } } @@ -215,23 +167,15 @@ class LuceneLookupSearchService( if (q.isBlank()) return emptyList() val tokens = q.split("\\s+".toRegex()).map { it.trim() }.filter { it.isNotEmpty() } if (tokens.isEmpty()) return emptyList() - return withSearcher { searcher -> - val b = BooleanQuery.Builder() - b.add(TermQuery(Term("type", "toc")), BooleanClause.Occur.FILTER) - tokens.forEach { t -> b.add(PrefixQuery(Term("q", t)), BooleanClause.Occur.MUST) } - val top = searcher.search(b.build(), limit, bookSort) - val stored = searcher.storedFields() - top.scoreDocs.map { sd -> - val doc = stored.document(sd.doc) - TocHit( - tocId = doc.getField("toc_id").numericValue().toLong(), - bookId = doc.getField("book_id").numericValue().toLong(), - bookTitle = doc.getField("book_title").stringValue(), - text = doc.getField("toc_text").stringValue(), - level = doc.getField("toc_level").numericValue().toInt(), - score = sd.score, - ) - } + return prefixSearch("toc", tokens, limit, bookSort, TOC_FIELDS).map { doc -> + TocHit( + tocId = doc.longs.getValue("toc_id"), + bookId = doc.longs.getValue("book_id"), + bookTitle = doc.strings.getValue("book_title"), + text = doc.strings.getValue("toc_text"), + level = doc.longs.getValue("toc_level").toInt(), + score = doc.score, + ) } } @@ -595,3 +539,8 @@ private fun sanitizeHebrewForAcronym(input: String): String { s = s.replace("\\s+".toRegex(), " ").trim() return s } + +// The stored fields each kind of lookup document is read from +private val BOOK_FIELDS = setOf("book_id", "category_id", "book_title", "is_base_book", "order_index") +private val AUTHOR_FIELDS = setOf("author_id", "author_name", "book_count") +private val TOC_FIELDS = setOf("toc_id", "book_id", "book_title", "toc_text", "toc_level") diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/RepositorySnippetSourceProvider.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/RepositorySnippetSourceProvider.kt similarity index 97% rename from SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/RepositorySnippetSourceProvider.kt rename to searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/RepositorySnippetSourceProvider.kt index 12e5795d9..b95cc382e 100644 --- a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/RepositorySnippetSourceProvider.kt +++ b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/RepositorySnippetSourceProvider.kt @@ -6,6 +6,7 @@ import io.github.kdroidfilter.seforimlibrary.search.SnippetProvider import io.github.kdroidfilter.seforimlibrary.search.SnippetSource import io.github.kdroidfilter.seforimlibrary.search.SnippetSources import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.IO import kotlinx.coroutines.withContext /** [SnippetProvider] over the app's database: the lines come from [repository]. */ diff --git a/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/SearchServices.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/SearchServices.kt new file mode 100644 index 000000000..b2ad2d051 --- /dev/null +++ b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/SearchServices.kt @@ -0,0 +1,67 @@ +package io.github.kdroidfilter.seforimapp.framework.search + +import io.github.kdroidfilter.seforimapp.features.search.semantic.SemanticModels +import io.github.kdroidfilter.seforimlibrary.dao.repository.SeforimRepository +import io.github.kdroidfilter.seforimlibrary.search.HebrewTextUtils +import io.github.kdroidfilter.seforimlibrary.search.LexicalSearchEngine +import io.github.kdroidfilter.seforimlibrary.search.MagicDictionaryIndex +import io.github.kdroidfilter.seforimlibrary.search.SearchEngine +import io.github.kdroidfilter.seforimlibrary.search.index.LuceneIndex + +/** + * The Lucene index at [indexDir] on this platform: read in process on the JVM and Android (Lucene on ART, API 35+), + * through the Java runtime the app embeds on iOS (see [IosJavaRuntime][io.github.kdroidfilter.seforimapp.framework.search]). + * [cacheQueries]: cache any query seen often (the small lookup index). + */ +expect fun openLuceneIndex( + indexDir: String, + cacheQueries: Boolean = false, +): LuceneIndex + +/** + * The search of the database at [databasePath] and of the indexes beside it, as the desktop and the mobile apps set it + * up: the search engine (lexical, fused with the semantic search in the official builds) and the lookup (books, + * authors, tables of contents). Everything opens lazily, on first use; [semanticModels] is the semantic model of a + * mobile app (the desktop's package carries its own). + */ +class SearchServices( + private val databasePath: String, + private val repository: SeforimRepository, + private val semanticModels: SemanticModels? = null, +) : AutoCloseable { + /** The full-text index: seforim.db.lucene beside seforim.db. */ + val indexPath: String get() = if (databasePath.endsWith(".db")) "$databasePath.lucene" else "$databasePath.luceneindex" + + /** The lookup index: seforim.db.lookup.lucene beside seforim.db. */ + val lookupIndexPath: String get() = if (databasePath.endsWith(".db")) "$databasePath.lookup.lucene" else "$databasePath.lookupindex" + + /** The lexical dictionary beside the index. */ + val dictionaryPath: String get() = indexPath.substringBeforeLast('/', ".") + "/lexical.db" + + private val textIndex = lazy { openLuceneIndex(indexPath) } + + /** The search engine. */ + val engine: SearchEngine by lazy { + val lexical = + LexicalSearchEngine( + index = textIndex.value, + snippetProvider = RepositorySnippetSourceProvider(repository), + dictionary = lazy { MagicDictionaryIndex.load(HebrewTextUtils::normalizeHebrew, dictionaryPath) }, + ) + withSemanticSearch(lexical, textIndex.value, repository, semanticModels) + } + + /** The acronyms of the books, for the lookup's ranking. */ + val acronyms: AcronymFrequencyCache by lazy { AcronymFrequencyCache { databasePath } } + + /** The lookup of books, authors and tables of contents. */ + val lookup: LuceneLookupSearchService by lazy { + LuceneLookupSearchService(openLuceneIndex(lookupIndexPath, cacheQueries = true), acronyms) + } + + override fun close() { + if (textIndex.isInitialized()) runCatching { engine.close() } + runCatching { lookup.close() } + runCatching { acronyms.close() } + } +} diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/SemanticSearchEngine.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/SemanticSearchEngine.kt similarity index 64% rename from SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/SemanticSearchEngine.kt rename to searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/SemanticSearchEngine.kt index 37e5a3fda..a7d0dbeba 100644 --- a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/SemanticSearchEngine.kt +++ b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/SemanticSearchEngine.kt @@ -1,24 +1,26 @@ package io.github.kdroidfilter.seforimapp.framework.search +import io.github.kdroidfilter.seforimapp.features.search.semantic.SemanticModels import io.github.kdroidfilter.seforimapp.features.search.semantic.installedSemanticSearch import io.github.kdroidfilter.seforimlibrary.dao.repository.SeforimRepository import io.github.kdroidfilter.seforimlibrary.search.LineHit -import io.github.kdroidfilter.seforimlibrary.search.LuceneSearchEngine import io.github.kdroidfilter.seforimlibrary.search.SearchEngine -import java.nio.file.Path +import io.github.kdroidfilter.seforimlibrary.search.index.LuceneIndex /** * Fuses [lexical] with dense semantic search in the official builds ([installedSemanticSearch], open core), whose - * vectors live in the same Lucene index at [indexPath]; returns [lexical] as is in a community build. Shared by the - * app's search engine and `zayit cli`, so both search the same way. + * vectors live in the same Lucene [index]; returns [lexical] as is in a community build. Shared by the app's search + * engine, `zayit cli` and the mobile apps, so all search the same way. [models] is the semantic model (null: the one the + * package carries, on the desktop). */ fun withSemanticSearch( - lexical: LuceneSearchEngine, - indexPath: Path, + lexical: SearchEngine, + index: LuceneIndex, repository: SeforimRepository, + models: SemanticModels? = null, ): SearchEngine { val semantic = installedSemanticSearch ?: return lexical - return semantic.hybrid(lexical, indexPath) { lineId, query -> + return semantic.hybrid(lexical, index, models) { lineId, query -> repository.getLine(lineId)?.let { line -> LineHit( bookId = line.bookId, diff --git a/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/framework/session/SearchPersistedState.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/framework/session/SearchPersistedState.kt new file mode 100644 index 000000000..d6a286def --- /dev/null +++ b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/framework/session/SearchPersistedState.kt @@ -0,0 +1,80 @@ +@file:OptIn(kotlinx.serialization.ExperimentalSerializationApi::class) + +package io.github.kdroidfilter.seforimapp.framework.session + +import io.github.kdroidfilter.seforim.tabs.SearchScope +import io.github.kdroidfilter.seforimapp.features.search.SearchTabCache +import kotlinx.serialization.Serializable + +/** What a search tab keeps across sessions: the desktop session saves it with its tab, a mobile app as it likes. */ +@Serializable +data class SearchPersistedState( + val query: String = "", + val globalExtended: Boolean = false, + val datasetScope: String = "global", + // Unused (a search is no longer scoped to a category), kept for the ProtoBuf field numbers + val filterCategoryId: Long = 0L, + val filterBookId: Long = 0L, + val filterTocId: Long = 0L, + // Unused, as filterCategoryId + val fetchCategoryId: Long = 0L, + val fetchBookId: Long = 0L, + val fetchTocId: Long = 0L, + // View filters (multi-select) + val selectedCategoryIds: Set = emptySet(), + val selectedBookIds: Set = emptySet(), + val selectedTocIds: Set = emptySet(), + // Scroll/anchor persistence + val scrollIndex: Int = 0, + val scrollOffset: Int = 0, + val anchorId: Long = -1L, + val anchorIndex: Int = 0, + // Full search snapshot (results + aggregates) for identical restore + val snapshot: SearchTabCache.Snapshot? = null, + val breadcrumbs: Map> = emptyMap(), + // Appended last: the session is ProtoBuf, numbered by declaration order. + // What the bar holds, typed but not searched yet (restored in the bar; [query] is what is searched) + val draftQuery: String = "", + // The author whose books are searched (none with a book or TOC scope) + val fetchAuthorId: Long = 0L, +) { + /** The scope the search runs in: its fetch scope, else its view filter. */ + val scope: SearchScope + get() { + val bookId = fetchBookId.takeIf { it > 0 } ?: filterBookId.takeIf { it > 0 } + val tocId = fetchTocId.takeIf { it > 0 } ?: filterTocId.takeIf { it > 0 } + return when { + tocId != null && bookId != null -> SearchScope.Toc(bookId = bookId, tocId = tocId) + bookId != null -> SearchScope.Book(bookId) + fetchAuthorId > 0 -> SearchScope.Author(fetchAuthorId) + else -> SearchScope.Global + } + } + + /** Runs the search in [scope]: both its fetch scope and its view filter (a legacy category: everywhere). */ + fun withScope(scope: SearchScope): SearchPersistedState { + val bookId = + when (scope) { + is SearchScope.Book -> scope.bookId + is SearchScope.Toc -> scope.bookId + else -> 0L + } + val tocId = (scope as? SearchScope.Toc)?.tocId ?: 0L + return copy( + datasetScope = + when (scope) { + SearchScope.Global, is SearchScope.Category -> "global" + is SearchScope.Book -> "book" + is SearchScope.Toc -> "toc" + is SearchScope.Author -> "author" + }, + filterCategoryId = 0L, + filterBookId = bookId, + filterTocId = tocId, + fetchCategoryId = 0L, + fetchBookId = bookId, + fetchTocId = tocId, + fetchAuthorId = (scope as? SearchScope.Author)?.authorId ?: 0L, + ) + } +} diff --git a/searchkit/src/commonTest/kotlin/io/github/kdroidfilter/seforimapp/framework/search/SearchKitDump.kt b/searchkit/src/commonTest/kotlin/io/github/kdroidfilter/seforimapp/framework/search/SearchKitDump.kt new file mode 100644 index 000000000..f54d9e377 --- /dev/null +++ b/searchkit/src/commonTest/kotlin/io/github/kdroidfilter/seforimapp/framework/search/SearchKitDump.kt @@ -0,0 +1,40 @@ +package io.github.kdroidfilter.seforimapp.framework.search + +import kotlin.math.round + +/** + * What the app's search answers on real data, as text: the lookup (books, authors, tables of contents) and the search + * engine (hybrid in the official builds: hits, snippets, highlights by meaning, smart find in a book). The same data + * gives the same dump on the desktop and the mobile apps, up to Lucene's score noise (see SeforimLibrary's SearchDump). + */ +object SearchKitDump { + private val BOOK_QUERIES = listOf("שוע", "רמבם", "בראשית", "חפץ חיים", "משנה ברורה", "גמ") + private val SEARCH_QUERIES = listOf("מה מברכים על אוכל", "ואהבת לרעך כמוך", "\"קריאת שמע\"", "הלכות שבת") + + suspend fun dump(services: SearchServices): String = + buildString { + val lookup = services.lookup + for (q in BOOK_QUERIES) appendLine("books $q -> ${lookup.suggestBooks(q, 8).map { "${it.id}:${it.title}" }}") + appendLine("authors רמב -> ${lookup.suggestAuthors("רמב", 5).map { it.name }}") + appendLine("toc הלכות שבת -> ${lookup.searchTocPrefix("הלכות שבת", 5).map { "${it.bookId}:${it.text}" }}") + + val engine = services.engine + appendLine("dense ${engine.denseReady()}") + for (q in SEARCH_QUERIES) { + appendLine("## $q (by meaning: ${engine.highlightsByMeaning(q)})") + val page = engine.openSession(q, baseBookOnly = false)?.use { it.nextPage(10) } + appendLine("total ${page?.totalHits}") + val hits = page?.hits.orEmpty() + for (hit in hits) appendLine(" ${hit.lineId} ${hit.bookTitle} ${(round(hit.score * 1e5) / 1e5)} | ${hit.snippet}") + val texts = hits.take(3).map { it.rawText.substring(it.lineStart, it.lineEnd) } + val highlights = engine.highlights(texts, q) + highlights.forEachIndexed { + i, + h, + -> + appendLine(" highlight ${hits[i].lineId} byMeaning=${h.byMeaning} ${h.ranges.map { texts[i].substring(it) }}") + } + hits.firstOrNull()?.let { appendLine(" find in ${it.bookId}: ${engine.semanticFind(q, it.bookId, 5)}") } + } + } +} diff --git a/searchkit/src/iosMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/LuceneIndexes.ios.kt b/searchkit/src/iosMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/LuceneIndexes.ios.kt new file mode 100644 index 000000000..5ca541c4c --- /dev/null +++ b/searchkit/src/iosMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/LuceneIndexes.ios.kt @@ -0,0 +1,43 @@ +@file:OptIn(ExperimentalForeignApi::class) + +package io.github.kdroidfilter.seforimapp.framework.search + +import io.github.kdroidfilter.seforimlibrary.search.index.LuceneIndex +import io.github.kdroidfilter.seforimlibrary.search.index.bridge.EmbeddedJvm +import io.github.kdroidfilter.seforimlibrary.search.index.bridge.JvmLuceneIndex +import kotlinx.cinterop.ExperimentalForeignApi +import platform.Foundation.NSBundle +import platform.Foundation.NSFileManager + +/** + * The Java runtime the iOS app embeds for Lucene (OpenJDK's mobile port, statically linked: see SeforimLibrary's + * IOS_RUNTIME.md). It starts with the first index opened; set these before. + */ +object IosJavaRuntime { + /** The jars of the search (SeforimLibrary's `iosJvmClasspath`): by default those of `/lib/app`. */ + var classPath: List = bundledJars() + + /** + * The runtime's options: its heap above all (Lucene's caches live there). 128 MB keeps the app's footprint around + * 160 MB on real data; a larger heap only grows it (the GC runs less), 32 MB fails. + */ + var options: List = listOf("-Xmx128m") + + val jvm: EmbeddedJvm by lazy { EmbeddedJvm.start(classPath, options) } + + private fun bundledJars(): List { + val dir = NSBundle.mainBundle.bundlePath + "/lib/app" + return NSFileManager.defaultManager + .contentsOfDirectoryAtPath(dir, null) + .orEmpty() + .map { it.toString() } + .filter { it.endsWith(".jar") } + .sorted() + .map { "$dir/$it" } + } +} + +actual fun openLuceneIndex( + indexDir: String, + cacheQueries: Boolean, +): LuceneIndex = JvmLuceneIndex(IosJavaRuntime.jvm, indexDir, cacheQueries) diff --git a/searchkit/src/iosMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/Repositories.ios.kt b/searchkit/src/iosMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/Repositories.ios.kt new file mode 100644 index 000000000..33e1efe71 --- /dev/null +++ b/searchkit/src/iosMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/Repositories.ios.kt @@ -0,0 +1,25 @@ +package io.github.kdroidfilter.seforimapp.framework.search + +import app.cash.sqldelight.driver.native.NativeSqliteDriver +import co.touchlab.sqliter.DatabaseConfiguration +import io.github.kdroidfilter.seforimlibrary.dao.repository.SeforimRepository + +/** + * The repository of the database at [databasePath] (the downloaded database). Its schema exists already: the driver + * creates and migrates nothing. + */ +fun openSeforimRepository(databasePath: String): SeforimRepository { + val driver = + NativeSqliteDriver( + DatabaseConfiguration( + name = databasePath.substringAfterLast('/'), + version = 1, + create = {}, + upgrade = { _, _, _ -> }, + extendedConfig = DatabaseConfiguration.Extended(basePath = databasePath.substringBeforeLast('/')), + ), + // Several readers at once: the search reads snippets while the screens read books and lines + maxReaderConnections = 4, + ) + return SeforimRepository(databasePath, driver) +} diff --git a/searchkit/src/iosSimulatorArm64Test/kotlin/io/github/kdroidfilter/seforimapp/framework/search/IosSearchKitParityTest.kt b/searchkit/src/iosSimulatorArm64Test/kotlin/io/github/kdroidfilter/seforimapp/framework/search/IosSearchKitParityTest.kt new file mode 100644 index 000000000..ec1c31582 --- /dev/null +++ b/searchkit/src/iosSimulatorArm64Test/kotlin/io/github/kdroidfilter/seforimapp/framework/search/IosSearchKitParityTest.kt @@ -0,0 +1,34 @@ +package io.github.kdroidfilter.seforimapp.framework.search + +import io.github.kdroidfilter.seforimapp.features.search.semantic.SemanticModels +import kotlinx.coroutines.runBlocking +import kotlinx.io.buffered +import kotlinx.io.files.Path +import kotlinx.io.files.SystemFileSystem +import kotlinx.io.writeString +import platform.Foundation.NSProcessInfo +import kotlin.test.Test +import kotlin.time.measureTimedValue + +/** + * The app's search on iOS, end to end in the simulator: the lookup and the hybrid search, Lucene in the embedded Java + * runtime, the semantic model on ONNX Runtime's C API, the database on SQLDelight's native driver. Writes SearchKitDump + * (variables passed by the iosSimulatorArm64Test task); skipped without data. + */ +class IosSearchKitParityTest { + @Test + fun dump() { + val env = NSProcessInfo.processInfo.environment + val db = env["SEFORIM_TEST_DB"] as String? ?: return + IosJavaRuntime.classPath = (env["SEFORIM_JVM_CLASSPATH"] as String).split(':') + val models = + (env["SEFORIM_MODEL_DIR"] as String?)?.let { + SemanticModels("$it/seforim-embed-v5-int8.onnx", "$it/tokenizer.json", "$it/seforim-embed-v5-static.i8") + } + SearchServices(db, openSeforimRepository(db), models).use { services -> + val (dump, took) = measureTimedValue { runBlocking { SearchKitDump.dump(services) } } + println("SearchKit dump in $took") + SystemFileSystem.sink(Path(env["SEFORIM_PARITY_OUT"] as String)).buffered().use { it.writeString(dump) } + } + } +} diff --git a/searchkit/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/LuceneIndexes.jvm.kt b/searchkit/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/LuceneIndexes.jvm.kt new file mode 100644 index 000000000..fc8ff7b67 --- /dev/null +++ b/searchkit/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/LuceneIndexes.jvm.kt @@ -0,0 +1,20 @@ +package io.github.kdroidfilter.seforimapp.framework.search + +import io.github.kdroidfilter.seforimlibrary.dao.repository.SeforimRepository +import io.github.kdroidfilter.seforimlibrary.search.LuceneSearchEngine +import io.github.kdroidfilter.seforimlibrary.search.SearchEngine +import io.github.kdroidfilter.seforimlibrary.search.index.LuceneIndex +import io.github.kdroidfilter.seforimlibrary.search.index.LuceneIndexBackend +import java.nio.file.Path + +actual fun openLuceneIndex( + indexDir: String, + cacheQueries: Boolean, +): LuceneIndex = LuceneIndexBackend(Path.of(indexDir), cacheQueries = cacheQueries) + +/** [withSemanticSearch] for the engine `zayit cli` builds (see `CliHost`): the semantic search reads its index. */ +fun withSemanticSearch( + lexical: LuceneSearchEngine, + @Suppress("UNUSED_PARAMETER") indexDir: Path, + repository: SeforimRepository, +): SearchEngine = withSemanticSearch(lexical, lexical.index, repository) diff --git a/searchkit/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/Repositories.jvm.kt b/searchkit/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/Repositories.jvm.kt new file mode 100644 index 000000000..47f0e69a4 --- /dev/null +++ b/searchkit/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/Repositories.jvm.kt @@ -0,0 +1,8 @@ +package io.github.kdroidfilter.seforimapp.framework.search + +import app.cash.sqldelight.driver.jdbc.sqlite.JdbcSqliteDriver +import io.github.kdroidfilter.seforimlibrary.dao.repository.SeforimRepository + +/** The repository of the database at [databasePath] (the desktop app keeps its own pooled driver). */ +fun openSeforimRepository(databasePath: String): SeforimRepository = + SeforimRepository(databasePath, JdbcSqliteDriver("jdbc:sqlite:$databasePath")) diff --git a/searchkit/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/framework/search/SearchKitParityTest.kt b/searchkit/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/framework/search/SearchKitParityTest.kt new file mode 100644 index 000000000..45481f778 --- /dev/null +++ b/searchkit/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/framework/search/SearchKitParityTest.kt @@ -0,0 +1,19 @@ +package io.github.kdroidfilter.seforimapp.framework.search + +import kotlinx.coroutines.runBlocking +import java.io.File +import kotlin.test.Test + +/** Writes SearchKitDump on the JVM for the parity with iOS: -PseforimTestDb=, -PparityOut. */ +class SearchKitParityTest { + @Test + fun dump() { + val db = System.getProperty("seforimTestDb") ?: return + val out = File(System.getProperty("parityOut") ?: return) + SearchServices(db, openSeforimRepository(db)).use { services -> + val start = System.nanoTime() + out.writeText(runBlocking { SearchKitDump.dump(services) }) + println("SearchKit dump in ${(System.nanoTime() - start) / 1_000_000} ms") + } + } +} diff --git a/settings.gradle.kts b/settings.gradle.kts index 689746c76..d4e056193 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -138,6 +138,7 @@ plugins { } include(":SeforimApp") +include(":searchkit") include(":cataloggen") include(":widgetprocessor") From 9a96582022a5a0bcf6fd20eb717be542a8dabe4b Mon Sep 17 00:00:00 2001 From: Elie Gambache Date: Fri, 16 Oct 2026 12:56:03 +0300 Subject: [PATCH 02/12] feat(search): Lucene in a Java process on Android, from Android 10 As SeforimLibrary now runs Lucene on a real JVM only (ART can't keep up with Lucene's Java versions), Android reaches it through the bridge, in a Java process of the app's own. - searchkit: AndroidJavaRuntime (configure(context), a 128 MB heap: ~140 MB for the process), openLuceneIndex always through it, assembleAndroidJavaRuntime (the launcher, the JDK, the search's jars), minSdk 29. - searchkit-check back: the app that runs the search on a device (a library's test APK can't hold the launcher). - SeforimLibrary at e4c1cad (kdroidFilter/SeforimLibrary#134). Verified on Android 15 and Android 11 emulators: SearchKitDump as on the desktop, in 2.3 to 4.5 s. Signed-off-by: Elie Gambache --- CLAUDE.md | 8 +-- SeforimLibrary | 2 +- searchkit-check/build.gradle.kts | 40 ++++++++++++++ .../searchkit/check/SearchParityTest.kt | 20 ++++--- searchkit-check/src/main/AndroidManifest.xml | 4 ++ searchkit/README.md | 26 +++++---- searchkit/build.gradle.kts | 37 +++++++------ .../framework/search/LuceneIndexes.android.kt | 53 +++++++++++++++++-- .../framework/search/LuceneIndexes.ios.kt | 2 +- settings.gradle.kts | 1 + 10 files changed, 152 insertions(+), 41 deletions(-) create mode 100644 searchkit-check/build.gradle.kts rename searchkit/src/androidDeviceTest/kotlin/io/github/kdroidfilter/seforimapp/framework/search/AndroidSearchKitParityTest.kt => searchkit-check/src/androidTest/kotlin/io/github/kdroidfilter/seforimapp/searchkit/check/SearchParityTest.kt (59%) create mode 100644 searchkit-check/src/main/AndroidManifest.xml diff --git a/CLAUDE.md b/CLAUDE.md index 540104963..d43477188 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -125,9 +125,11 @@ into the index's vectors (`:searchindex:embedCorpus`, GPU via CUDA, no Python). ### Search on mobile: `searchkit` The search without its UI (engines, lookup, use cases, `SearchResultViewModel`/`SearchHomeViewModel`) is the KMP module -`searchkit` (JVM, Android 15+, iOS), shared by the desktop and the mobile apps; see `searchkit/README.md`. Lucene is -reached only through SeforimLibrary's `LuceneIndex` port: in process on the JVM and Android (ART, `lucene-android` -patches), through an embedded OpenJDK (Zero) on iOS (`SeforimLibrary/IOS_RUNTIME.md`). The view models reach the +`searchkit` (JVM, Android 10+, iOS), shared by the desktop and the mobile apps; see `searchkit/README.md`. Lucene is +reached only through SeforimLibrary's `LuceneIndex` port and runs on a real JVM only (ART can't keep up with Lucene's +Java versions): in process on the desktop, in an embedded OpenJDK 27 elsewhere, behind the same bridge: in the app on +iOS (Zero, JNI), in a process of the app's own on Android (server VM, `AndroidJavaRuntime`); see +`SeforimLibrary/IOS_RUNTIME.md`. The view models reach the desktop shell only through the ports of `SearchHost.kt` (`SearchTabs`, `SearchSessionStore`, `SearchSettings`, `SearchHistory`); keep `java.*`, AWT and desktop types out of `searchkit/src/commonMain`. diff --git a/SeforimLibrary b/SeforimLibrary index 9c9a921b3..e4c1cada2 160000 --- a/SeforimLibrary +++ b/SeforimLibrary @@ -1 +1 @@ -Subproject commit 9c9a921b3d5784ed8cf963372a66738144406107 +Subproject commit e4c1cada2630f5f791dd2ac59598fdf65fb337d4 diff --git a/searchkit-check/build.gradle.kts b/searchkit-check/build.gradle.kts new file mode 100644 index 000000000..3509005b0 --- /dev/null +++ b/searchkit-check/build.gradle.kts @@ -0,0 +1,40 @@ +// A test harness, no UI: an Android app that holds searchkit as a real app does (the launcher of the Java process among +// its native libraries, which a library's test APK can't hold) and runs the search on a device or an emulator through +// its instrumented test. +plugins { + alias(libs.plugins.android.application) +} + +android { + namespace = "io.github.kdroidfilter.seforimapp.searchkit.check" + compileSdk = 37 + + defaultConfig { + applicationId = "io.github.kdroidfilter.seforimapp.searchkit.check" + minSdk = 29 + targetSdk = 36 + testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" + } + + // The launcher is an executable: installed as a file in the native library dir, to be run from there + packaging { jniLibs { useLegacyPackaging = true } } + + sourceSets { + // The launcher of the Java runtime (searchkit's assembleAndroidJavaRuntime) + getByName("main").jniLibs.directories.add("../searchkit/build/androidJavaRuntime/jniLibs") + // The dump compared across platforms + getByName("androidTest").kotlin.directories.add("../searchkit/src/commonTest/kotlin") + } +} + +tasks.matching { it.name.startsWith("merge") && it.name.endsWith("JniLibFolders") }.configureEach { + dependsOn(":searchkit:assembleAndroidJavaRuntime") +} + +dependencies { + implementation(project(":searchkit")) + androidTestImplementation(libs.androidx.test.runner) + androidTestImplementation(libs.androidx.test.junit) + androidTestImplementation(libs.kotlinx.coroutines.test) + androidTestImplementation(kotlin("test")) +} diff --git a/searchkit/src/androidDeviceTest/kotlin/io/github/kdroidfilter/seforimapp/framework/search/AndroidSearchKitParityTest.kt b/searchkit-check/src/androidTest/kotlin/io/github/kdroidfilter/seforimapp/searchkit/check/SearchParityTest.kt similarity index 59% rename from searchkit/src/androidDeviceTest/kotlin/io/github/kdroidfilter/seforimapp/framework/search/AndroidSearchKitParityTest.kt rename to searchkit-check/src/androidTest/kotlin/io/github/kdroidfilter/seforimapp/searchkit/check/SearchParityTest.kt index 62b892af7..526c99406 100644 --- a/searchkit/src/androidDeviceTest/kotlin/io/github/kdroidfilter/seforimapp/framework/search/AndroidSearchKitParityTest.kt +++ b/searchkit-check/src/androidTest/kotlin/io/github/kdroidfilter/seforimapp/searchkit/check/SearchParityTest.kt @@ -1,28 +1,32 @@ -package io.github.kdroidfilter.seforimapp.framework.search +package io.github.kdroidfilter.seforimapp.searchkit.check import android.os.SystemClock import androidx.test.ext.junit.runners.AndroidJUnit4 import androidx.test.platform.app.InstrumentationRegistry import io.github.kdroidfilter.seforimapp.features.search.semantic.SemanticModels +import io.github.kdroidfilter.seforimapp.framework.search.AndroidJavaRuntime +import io.github.kdroidfilter.seforimapp.framework.search.SearchKitDump +import io.github.kdroidfilter.seforimapp.framework.search.SearchServices +import io.github.kdroidfilter.seforimapp.framework.search.openSeforimRepository import kotlinx.coroutines.runBlocking import org.junit.Test import org.junit.runner.RunWith import java.io.File /** - * The app's search on Android 15+, end to end: the lookup and the hybrid search, Lucene on ART, the semantic model on - * ONNX Runtime for Android, the database on Android's SQLite. The data (seforim.db and its indexes, lexical.db, the - * model in model/) goes in the test app's files dir under seforim/ (adb root, push, chown to the app); SearchKitDump - * comes back as searchkit-android.txt there. Skipped without data. + * The app's search on this device, as an app runs it: Lucene in the app's Java process. The data goes in the app's + * files dir (adb root, push, chown to the app): `seforim/` (seforim.db, its indexes, lexical.db, `model/`), `jdk/` and + * `jdk-app/` (searchkit's assembleAndroidJavaRuntime). Writes SearchKitDump as `searchkit.txt` there; skipped without + * data. */ @RunWith(AndroidJUnit4::class) -class AndroidSearchKitParityTest { +class SearchParityTest { @Test fun dump() { val context = InstrumentationRegistry.getInstrumentation().targetContext val dir = File(context.filesDir, "seforim") val db = File(dir, "seforim.db").takeIf { it.isFile } ?: return - InstrumentationRegistry.getArguments().getString("luceneQueryCache")?.let { System.setProperty("seforim.lucene.queryCache", it) } + AndroidJavaRuntime.configure(context) val model = File(dir, "model") val models = SemanticModels( @@ -34,7 +38,7 @@ class AndroidSearchKitParityTest { val start = SystemClock.elapsedRealtime() val dump = runBlocking { SearchKitDump.dump(services) } println("SearchKit dump in ${SystemClock.elapsedRealtime() - start} ms") - File(context.filesDir, "searchkit-android.txt").writeText(dump) + File(context.filesDir, "searchkit.txt").writeText(dump) } } } diff --git a/searchkit-check/src/main/AndroidManifest.xml b/searchkit-check/src/main/AndroidManifest.xml new file mode 100644 index 000000000..9917f7cb8 --- /dev/null +++ b/searchkit-check/src/main/AndroidManifest.xml @@ -0,0 +1,4 @@ + + + + diff --git a/searchkit/README.md b/searchkit/README.md index c67de2f03..b41635a15 100644 --- a/searchkit/README.md +++ b/searchkit/README.md @@ -1,25 +1,24 @@ # searchkit -Zayit's search without its UI, for the desktop app and the mobile ones (Android 15+, iOS): the engines (lexical, fused +Zayit's search without its UI, for the desktop app and the mobile ones (Android 10+, iOS): the engines (lexical, fused with the semantic search in the official builds), the lookup (books, authors, tables of contents), the search use cases and the view models of the search screens. The desktop app uses it as is; a mobile app only adds its screens. Verified on real data: `SearchKitDump` (lookup, hybrid search, snippets, highlights by meaning, smart find) gives the -same text on the JVM, in the iOS simulator and on an Android emulator (`jvmTest`, `iosSimulatorArm64Test`, -`androidDeviceTest`). +same text on the JVM, in the iOS simulator and on Android 11 and 15 emulators (`jvmTest`, `iosSimulatorArm64Test`, +`searchkit-check`). ## What runs where -| | Desktop (JVM) | Android 15+ | iOS | +| | Desktop (JVM) | Android 10+ | iOS | |---|---|---|---| -| Lucene (full-text and vector index) | in process | in process, on ART (`lucene-android`) | in the embedded Java runtime | +| Lucene (full-text and vector index) | in process | in a Java process of the app's own (server VM) | in the embedded Java runtime (Zero) | | Database (`SeforimRepository`) | the app's pooled JDBC driver | `openSeforimRepository(context, path)` | `openSeforimRepository(path)` | | Semantic model (official builds) | ONNX Runtime Java, model in the jar | ONNX Runtime Android | ONNX Runtime C API (`onnxruntime-c` 1.20.0) | | Model files | bundled | downloaded with the data | downloaded with the data | -Android before 15 isn't supported: its ART lacks the Java 21 APIs Lucene 10 needs. - -The iOS runtime is OpenJDK's mobile port, built by the CI of +Lucene runs on a real JVM only: ART can't keep up with the Java versions Lucene requires (21 for Lucene 10, 25 for 11). +The mobile runtimes are OpenJDK 27 with openjdk/mobile's port, built by the CI of [kdroidFilter/openjdk-mobile](https://github.com/kdroidFilter/openjdk-mobile) and downloaded by the build (see [IOS_RUNTIME.md](../SeforimLibrary/IOS_RUNTIME.md)). @@ -37,6 +36,12 @@ services.engine // SearchEngine: openSession, computeFacets, highlights, sema services.lookup // LuceneLookupSearchService: suggestBooks, suggestAuthors, searchTocPrefix ``` +On Android, call `AndroidJavaRuntime.configure(context)` before the first search: Lucene runs in a Java process, from +the runtime `./gradlew :searchkit:assembleAndroidJavaRuntime` assembles: `jniLibs/` (the launcher, among the app's +native libraries, with `packaging.jniLibs.useLegacyPackaging = true`), `jdk/` and `jdk-app/` (delivered into its files). +`searchkit-check` is such an app (no UI). On a recent emulator, the process takes ~140 MB of memory with its default +128 MB heap (plus the index pages it maps, which Android reclaims). + On iOS, `IosJavaRuntime` (`classPath`, `options`) configures the embedded runtime before the first search; by default it runs the jars of `App.app/lib/app`. `./gradlew :searchkit:assembleIosJavaRuntimeDevice` assembles `static/` (what the app links, options in IOS_RUNTIME.md) and `lib/` (copy it into the bundle as `App.app/lib` in an @@ -72,8 +77,9 @@ a mobile app can call its constructor directly. -Pseforim.ortXcframework= -Pseforim.semanticModels= # build/parity/ios.txt ``` -Use a copy of the database (an APFS clone: `cp -c`): `SeforimRepository` sets its PRAGMAs on open. On Android, push the -data into the test app's files dir under `seforim/` and run `AndroidSearchKitParityTest` (see its KDoc). +Use a copy of the database (an APFS clone: `cp -c`): `SeforimRepository` sets its PRAGMAs on open. On Android, install +`searchkit-check` and its test APK, push the data and the runtime into its files dir and run its `SearchParityTest` +(see its KDoc). ## Not here (the UI's) diff --git a/searchkit/build.gradle.kts b/searchkit/build.gradle.kts index dbfdb74d1..e13682fa7 100644 --- a/searchkit/build.gradle.kts +++ b/searchkit/build.gradle.kts @@ -12,8 +12,8 @@ val withSemantic = gradle.extensions.extraProperties["semanticEnabled"] == true /** * Zayit's search, without its UI: the engines (lexical, hybrid with the semantic search), the lookup, the use cases and - * the view models of the search screens, for the desktop app and the mobile ones (Android 15+, iOS). Lucene runs in - * process on the JVM and Android, in the Java runtime the iOS app embeds (see SeforimLibrary's IOS_RUNTIME.md). + * the view models of the search screens, for the desktop app and the mobile ones (Android 10+, iOS). Lucene runs in + * process on the JVM, in the Java runtime the mobile apps embed (see SeforimLibrary's IOS_RUNTIME.md). */ kotlin { jvmToolchain( @@ -26,12 +26,8 @@ kotlin { android { namespace = "io.github.kdroidfilter.seforimapp.searchkit" compileSdk = 37 - // Lucene runs on ART from Android 15 only (lucene-android) - minSdk = 35 - // The search on a device or an emulator (SearchKitDump on real data, see AndroidSearchKitParityTest) - withDeviceTestBuilder { sourceSetTreeName = "test" }.configure { - instrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" - } + // Lucene runs in a Java process of the app's own (AndroidJavaRuntime), from Android 10 + minSdk = 29 } iosArm64() iosSimulatorArm64() @@ -75,16 +71,13 @@ kotlin { implementation(kotlin("test")) implementation(libs.kotlinx.coroutines.test) } - getByName("androidDeviceTest").dependencies { - implementation(libs.androidx.test.runner) - implementation(libs.androidx.test.junit) - } } } // --- Tests on real data: the same dump on the JVM and in the iOS simulator (SearchKitDump) --- -// The Java runtime of iOS (OpenJDK's mobile port), downloaded by SeforimLibrary's search (see its IOS_RUNTIME.md): +// The Java runtimes of the mobile apps (OpenJDK's mobile port), downloaded by SeforimLibrary's search (see its +// IOS_RUNTIME.md): // build/javaRuntime/ there val seforimLibrary = gradle.includedBuild("SeforimLibrary") @@ -149,8 +142,8 @@ tasks.named("jvmTest") { } // SeforimLibrary's jars for the embedded runtime (its Lucene side, see IOS_RUNTIME.md) -val searchJars = seforimLibrary.projectDir.resolve("search/build/iosJvmClasspath") -val searchJarsTask = seforimLibrary.task(":search:iosJvmClasspath") +val searchJars = seforimLibrary.projectDir.resolve("search/build/mobileJvmClasspath") +val searchJarsTask = seforimLibrary.task(":search:mobileJvmClasspath") // The Java runtime an iOS app bundles as App.app/lib (its java.home): the runtime's classes, conf and data, and the // search's jars in lib/app (IosJavaRuntime's default class path) @@ -177,6 +170,20 @@ listOf("simulator", "device").forEach { kind -> } } +// For an Android app (Lucene in a Java process, see AndroidJavaRuntime): build/androidJavaRuntime/jniLibs (the launcher, +// among the app's native libraries), jdk (the JDK's classes and data, its java.home) and jdk-app (the search's jars), +// both delivered into the app's files +tasks.register("assembleAndroidJavaRuntime") { + group = "build" + description = "Assembles the Java runtime an Android app delivers for Lucene." + dependsOn(searchJarsTask, javaRuntimeTask("android")) + from(javaRuntime("android").resolve("jniLibs")) { into("jniLibs") } + from(javaRuntime("android").resolve("lib")) { into("jdk") } + from(searchJars) { into("jdk-app") } + into(layout.buildDirectory.dir("androidJavaRuntime")) + filePermissions { user { write = true } } +} + run { val testBinaryDir = layout.buildDirectory.dir("bin/iosSimulatorArm64/debugTest") diff --git a/searchkit/src/androidMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/LuceneIndexes.android.kt b/searchkit/src/androidMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/LuceneIndexes.android.kt index 53f51ce59..4acf1d9d9 100644 --- a/searchkit/src/androidMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/LuceneIndexes.android.kt +++ b/searchkit/src/androidMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/LuceneIndexes.android.kt @@ -1,11 +1,58 @@ package io.github.kdroidfilter.seforimapp.framework.search +import android.content.Context import io.github.kdroidfilter.seforimlibrary.search.index.LuceneIndex -import io.github.kdroidfilter.seforimlibrary.search.index.LuceneIndexBackend +import io.github.kdroidfilter.seforimlibrary.search.index.bridge.BridgedLuceneIndex +import io.github.kdroidfilter.seforimlibrary.search.index.bridge.JavaProcessTransport import java.io.File -// Lucene on ART, in process (Android 15+; lucene-core patched for it, see SeforimLibrary's lucene-android) +/** + * The Java runtime Lucene runs in on Android, whose ART can't keep up with the Java versions Lucene requires: a process + * of the app's own (see SeforimLibrary's IOS_RUNTIME.md, "Android"). + * + * Call [configure] before the first search; the app delivers the runtime (`./gradlew :searchkit:assembleAndroidJavaRuntime`): + * the launcher in its native libraries (`lib/arm64-v8a/libjvmlauncher.so`, installed as a file: `useLegacyPackaging`), + * the JDK's classes and data in [javaHome] and the search's jars in [classPath] (both downloaded or extracted from the + * app's assets). + */ +object AndroidJavaRuntime { + /** The JDK's classes and data (an exploded image: `modules/`, `conf/`, `lib/`). */ + var javaHome: File? = null + + /** The search's jars (SeforimLibrary's `mobileJvmClasspath`, the same as iOS). */ + var classPath: List = emptyList() + + /** The Java process's options: its heap above all. */ + var options: List = listOf("-Xmx128m", "-XX:+UseSerialGC", "-XX:UseSVE=0", "--enable-native-access=ALL-UNNAMED") + + private var launcher: File? = null + + /** Uses the runtime the app delivers in [context]'s files: `jdk/` and the jars of `jdk-app/`, unless set otherwise. */ + fun configure( + context: Context, + javaHome: File = File(context.filesDir, "jdk"), + classPath: List = File(context.filesDir, "jdk-app").listFiles { file -> file.extension == "jar" }.orEmpty().sorted(), + ) { + launcher = File(context.applicationInfo.nativeLibraryDir, "libjvmlauncher.so") + // Android has no /tmp: the app's cache instead + val tmp = context.cacheDir.path + options = options.filterNot { it.startsWith("-Djava.io.tmpdir=") || it.startsWith("-XX:AltTempDir=") } + + listOf("-Djava.io.tmpdir=$tmp", "-XX:AltTempDir=$tmp") + this.javaHome = javaHome + this.classPath = classPath + } + + val transport: JavaProcessTransport by lazy { + JavaProcessTransport( + launcher = checkNotNull(launcher) { "AndroidJavaRuntime.configure(context) first: Lucene runs in its Java process" }, + javaHome = checkNotNull(javaHome), + classPath = classPath, + options = options, + ) + } +} + actual fun openLuceneIndex( indexDir: String, cacheQueries: Boolean, -): LuceneIndex = LuceneIndexBackend(File(indexDir).toPath(), cacheQueries = cacheQueries) +): LuceneIndex = BridgedLuceneIndex(AndroidJavaRuntime.transport, indexDir, cacheQueries) diff --git a/searchkit/src/iosMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/LuceneIndexes.ios.kt b/searchkit/src/iosMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/LuceneIndexes.ios.kt index 5ca541c4c..01927f039 100644 --- a/searchkit/src/iosMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/LuceneIndexes.ios.kt +++ b/searchkit/src/iosMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/LuceneIndexes.ios.kt @@ -14,7 +14,7 @@ import platform.Foundation.NSFileManager * IOS_RUNTIME.md). It starts with the first index opened; set these before. */ object IosJavaRuntime { - /** The jars of the search (SeforimLibrary's `iosJvmClasspath`): by default those of `/lib/app`. */ + /** The jars of the search (SeforimLibrary's `mobileJvmClasspath`): by default those of `/lib/app`. */ var classPath: List = bundledJars() /** diff --git a/settings.gradle.kts b/settings.gradle.kts index d4e056193..114818d07 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -139,6 +139,7 @@ plugins { include(":SeforimApp") include(":searchkit") +include(":searchkit-check") include(":cataloggen") include(":widgetprocessor") From 7f38382aa8833d3ee841d89ae430ec6f0afaf33c Mon Sep 17 00:00:00 2001 From: Elie Gambache Date: Fri, 16 Oct 2026 13:31:27 +0300 Subject: [PATCH 03/12] feat(search): the Android runtime for arm64 and x86_64 assembleAndroidJavaRuntime puts both launchers among the native libraries and a java.home per ABI (jdk/: some of its classes depend on the architecture); AndroidJavaRuntime picks the one of the launcher Android installed. SeforimLibrary at its jdk-mobile-27b35.2 commit. Signed-off-by: Elie Gambache --- SeforimLibrary | 2 +- .../searchkit/check/SearchParityTest.kt | 4 ++-- searchkit/README.md | 5 +++-- searchkit/build.gradle.kts | 15 ++++++++------ .../framework/search/LuceneIndexes.android.kt | 20 ++++++++++++++----- 5 files changed, 30 insertions(+), 16 deletions(-) diff --git a/SeforimLibrary b/SeforimLibrary index e4c1cada2..043e435ee 160000 --- a/SeforimLibrary +++ b/SeforimLibrary @@ -1 +1 @@ -Subproject commit e4c1cada2630f5f791dd2ac59598fdf65fb337d4 +Subproject commit 043e435ee8269ff038409d290d0f89d08bc1bf73 diff --git a/searchkit-check/src/androidTest/kotlin/io/github/kdroidfilter/seforimapp/searchkit/check/SearchParityTest.kt b/searchkit-check/src/androidTest/kotlin/io/github/kdroidfilter/seforimapp/searchkit/check/SearchParityTest.kt index 526c99406..e6e4aadbc 100644 --- a/searchkit-check/src/androidTest/kotlin/io/github/kdroidfilter/seforimapp/searchkit/check/SearchParityTest.kt +++ b/searchkit-check/src/androidTest/kotlin/io/github/kdroidfilter/seforimapp/searchkit/check/SearchParityTest.kt @@ -15,8 +15,8 @@ import java.io.File /** * The app's search on this device, as an app runs it: Lucene in the app's Java process. The data goes in the app's - * files dir (adb root, push, chown to the app): `seforim/` (seforim.db, its indexes, lexical.db, `model/`), `jdk/` and - * `jdk-app/` (searchkit's assembleAndroidJavaRuntime). Writes SearchKitDump as `searchkit.txt` there; skipped without + * files dir (adb root, push, chown to the app): `seforim/` (seforim.db, its indexes, lexical.db, `model/`), + * `jdk//` and `jdk-app/` (searchkit's assembleAndroidJavaRuntime). Writes SearchKitDump as `searchkit.txt` there; skipped without * data. */ @RunWith(AndroidJUnit4::class) diff --git a/searchkit/README.md b/searchkit/README.md index b41635a15..0316f6f7d 100644 --- a/searchkit/README.md +++ b/searchkit/README.md @@ -37,8 +37,9 @@ services.lookup // LuceneLookupSearchService: suggestBooks, suggestAuthors, s ``` On Android, call `AndroidJavaRuntime.configure(context)` before the first search: Lucene runs in a Java process, from -the runtime `./gradlew :searchkit:assembleAndroidJavaRuntime` assembles: `jniLibs/` (the launcher, among the app's -native libraries, with `packaging.jniLibs.useLegacyPackaging = true`), `jdk/` and `jdk-app/` (delivered into its files). +the runtime `./gradlew :searchkit:assembleAndroidJavaRuntime` assembles: `jniLibs/` (the launchers, arm64-v8a and +x86_64, among the app's native libraries, with `packaging.jniLibs.useLegacyPackaging = true`), `jdk//` (the JDK of +an ABI: deliver the device's alone) and `jdk-app/` (delivered into its files). `searchkit-check` is such an app (no UI). On a recent emulator, the process takes ~140 MB of memory with its default 128 MB heap (plus the index pages it maps, which Android reclaims). diff --git a/searchkit/build.gradle.kts b/searchkit/build.gradle.kts index e13682fa7..6a1a10cfe 100644 --- a/searchkit/build.gradle.kts +++ b/searchkit/build.gradle.kts @@ -170,15 +170,18 @@ listOf("simulator", "device").forEach { kind -> } } -// For an Android app (Lucene in a Java process, see AndroidJavaRuntime): build/androidJavaRuntime/jniLibs (the launcher, -// among the app's native libraries), jdk (the JDK's classes and data, its java.home) and jdk-app (the search's jars), -// both delivered into the app's files +// For an Android app (Lucene in a Java process, see AndroidJavaRuntime): build/androidJavaRuntime/jniLibs (the launchers, +// arm64-v8a and x86_64, among the app's native libraries), jdk/ (the JDK's classes and data, its java.home: some of +// its classes depend on the architecture) and jdk-app (the search's jars), delivered into the app's files (jdk/ of +// the device's ABI alone) tasks.register("assembleAndroidJavaRuntime") { group = "build" description = "Assembles the Java runtime an Android app delivers for Lucene." - dependsOn(searchJarsTask, javaRuntimeTask("android")) - from(javaRuntime("android").resolve("jniLibs")) { into("jniLibs") } - from(javaRuntime("android").resolve("lib")) { into("jdk") } + dependsOn(searchJarsTask, javaRuntimeTask("android-aarch64"), javaRuntimeTask("android-x64")) + from(javaRuntime("android-aarch64").resolve("jniLibs")) { into("jniLibs") } + from(javaRuntime("android-x64").resolve("jniLibs")) { into("jniLibs") } + from(javaRuntime("android-aarch64").resolve("lib")) { into("jdk/arm64-v8a") } + from(javaRuntime("android-x64").resolve("lib")) { into("jdk/x86_64") } from(searchJars) { into("jdk-app") } into(layout.buildDirectory.dir("androidJavaRuntime")) filePermissions { user { write = true } } diff --git a/searchkit/src/androidMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/LuceneIndexes.android.kt b/searchkit/src/androidMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/LuceneIndexes.android.kt index 4acf1d9d9..0c58202fd 100644 --- a/searchkit/src/androidMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/LuceneIndexes.android.kt +++ b/searchkit/src/androidMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/LuceneIndexes.android.kt @@ -11,9 +11,9 @@ import java.io.File * of the app's own (see SeforimLibrary's IOS_RUNTIME.md, "Android"). * * Call [configure] before the first search; the app delivers the runtime (`./gradlew :searchkit:assembleAndroidJavaRuntime`): - * the launcher in its native libraries (`lib/arm64-v8a/libjvmlauncher.so`, installed as a file: `useLegacyPackaging`), - * the JDK's classes and data in [javaHome] and the search's jars in [classPath] (both downloaded or extracted from the - * app's assets). + * the launchers in its native libraries (`lib//libjvmlauncher.so`, arm64-v8a and x86_64, installed as files: + * `useLegacyPackaging`), the JDK's classes and data of the device's ABI in [javaHome] and the search's jars in + * [classPath] (both downloaded or extracted from the app's assets). */ object AndroidJavaRuntime { /** The JDK's classes and data (an exploded image: `modules/`, `conf/`, `lib/`). */ @@ -27,10 +27,13 @@ object AndroidJavaRuntime { private var launcher: File? = null - /** Uses the runtime the app delivers in [context]'s files: `jdk/` and the jars of `jdk-app/`, unless set otherwise. */ + /** + * Uses the runtime the app delivers in [context]'s files: `jdk//` (the ABI of the launcher Android installed) + * and the jars of `jdk-app/`, unless set otherwise. + */ fun configure( context: Context, - javaHome: File = File(context.filesDir, "jdk"), + javaHome: File = File(context.filesDir, "jdk/${launcherAbi(context)}"), classPath: List = File(context.filesDir, "jdk-app").listFiles { file -> file.extension == "jar" }.orEmpty().sorted(), ) { launcher = File(context.applicationInfo.nativeLibraryDir, "libjvmlauncher.so") @@ -42,6 +45,13 @@ object AndroidJavaRuntime { this.classPath = classPath } + // The native library dir is named after the ABI Android installed the app's libraries for (lib/arm64, lib/x86_64) + private fun launcherAbi(context: Context): String = + when (val dir = File(context.applicationInfo.nativeLibraryDir).name) { + "arm64" -> "arm64-v8a" + else -> dir + } + val transport: JavaProcessTransport by lazy { JavaProcessTransport( launcher = checkNotNull(launcher) { "AndroidJavaRuntime.configure(context) first: Lucene runs in its Java process" }, From ca7dcf2d4136d808df9dcad47ee10f004c9720cd Mon Sep 17 00:00:00 2001 From: Elie Gambache Date: Fri, 16 Oct 2026 17:42:03 +0300 Subject: [PATCH 04/12] refactor(search): the search's JVM through Nucleus JVM Access MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SeforimLibrary's search now runs Lucene through NJA (kdroidFilter/SeforimLibrary#134), with the JVM in the app's process on every platform: - searchkit: openLuceneIndex is the façade's constructor, common to all platforms; AndroidJavaRuntime, IosJavaRuntime and assembleAndroidJavaRuntime go (Android gets the JVM with search's library, nothing to configure); iOS binaries link the staged runtime and carry NJA's bundle, assembleIosJavaRuntime* assembles both for an app. - searchkit-check: no runtime to deliver anymore. - SeforimLibrary at 81d9725. The same SearchKitDump as before: the desktop byte for byte, iOS identical to it, Android 11 but for one hit total (Lucene's lower bound), in 1.9 s once the runtime is extracted. Signed-off-by: Elie Gambache --- CLAUDE.md | 10 +- SeforimLibrary | 2 +- searchkit-check/build.gradle.kts | 15 +-- .../searchkit/check/SearchParityTest.kt | 9 +- searchkit/README.md | 26 ++--- searchkit/build.gradle.kts | 103 +++++------------- .../framework/search/LuceneIndexes.android.kt | 68 ------------ .../search/LuceneLookupSearchService.kt | 13 ++- .../framework/search/SearchServices.kt | 10 +- .../framework/search/LuceneIndexes.ios.kt | 43 -------- .../search/IosSearchKitParityTest.kt | 1 - ...ndexes.jvm.kt => CliSemanticSearch.jvm.kt} | 7 -- 12 files changed, 64 insertions(+), 243 deletions(-) delete mode 100644 searchkit/src/androidMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/LuceneIndexes.android.kt delete mode 100644 searchkit/src/iosMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/LuceneIndexes.ios.kt rename searchkit/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/{LuceneIndexes.jvm.kt => CliSemanticSearch.jvm.kt} (67%) diff --git a/CLAUDE.md b/CLAUDE.md index d43477188..30be39515 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -125,11 +125,11 @@ into the index's vectors (`:searchindex:embedCorpus`, GPU via CUDA, no Python). ### Search on mobile: `searchkit` The search without its UI (engines, lookup, use cases, `SearchResultViewModel`/`SearchHomeViewModel`) is the KMP module -`searchkit` (JVM, Android 10+, iOS), shared by the desktop and the mobile apps; see `searchkit/README.md`. Lucene is -reached only through SeforimLibrary's `LuceneIndex` port and runs on a real JVM only (ART can't keep up with Lucene's -Java versions): in process on the desktop, in an embedded OpenJDK 27 elsewhere, behind the same bridge: in the app on -iOS (Zero, JNI), in a process of the app's own on Android (server VM, `AndroidJavaRuntime`); see -`SeforimLibrary/IOS_RUNTIME.md`. The view models reach the +`searchkit` (JVM, Android 10+, iOS), shared by the desktop and the mobile apps; see `searchkit/README.md`. Lucene runs +on a real JVM only (ART can't keep up with Lucene's Java versions), through Nucleus JVM Access: SeforimLibrary's +`search-facade` (Kotlin/JVM) is the façade, its common code uses the `LuceneIndex`/`IndexQuery`... NJA generates; the +desktop's own JVM, HotSpot in the app's process on Android, Zero linked into the app on iOS (jdk-mobile's OpenJDK 27); +see `SeforimLibrary/JVM_RUNTIME.md`. Until an NJA release has what it needs, build with `-Pnja.local=`. The view models reach the desktop shell only through the ports of `SearchHost.kt` (`SearchTabs`, `SearchSessionStore`, `SearchSettings`, `SearchHistory`); keep `java.*`, AWT and desktop types out of `searchkit/src/commonMain`. diff --git a/SeforimLibrary b/SeforimLibrary index 043e435ee..81d97254c 160000 --- a/SeforimLibrary +++ b/SeforimLibrary @@ -1 +1 @@ -Subproject commit 043e435ee8269ff038409d290d0f89d08bc1bf73 +Subproject commit 81d97254c65fe17a80f54ea2148f1305d590cb6a diff --git a/searchkit-check/build.gradle.kts b/searchkit-check/build.gradle.kts index 3509005b0..bc0cbbd09 100644 --- a/searchkit-check/build.gradle.kts +++ b/searchkit-check/build.gradle.kts @@ -1,6 +1,6 @@ -// A test harness, no UI: an Android app that holds searchkit as a real app does (the launcher of the Java process among -// its native libraries, which a library's test APK can't hold) and runs the search on a device or an emulator through -// its instrumented test. +// A test harness, no UI: an Android app that holds searchkit as a real app does (the Java runtime NJA brings with +// SeforimLibrary's search: its native libraries and assets) and runs the search on a device or an emulator through its +// instrumented test. plugins { alias(libs.plugins.android.application) } @@ -16,21 +16,12 @@ android { testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" } - // The launcher is an executable: installed as a file in the native library dir, to be run from there - packaging { jniLibs { useLegacyPackaging = true } } - sourceSets { - // The launcher of the Java runtime (searchkit's assembleAndroidJavaRuntime) - getByName("main").jniLibs.directories.add("../searchkit/build/androidJavaRuntime/jniLibs") // The dump compared across platforms getByName("androidTest").kotlin.directories.add("../searchkit/src/commonTest/kotlin") } } -tasks.matching { it.name.startsWith("merge") && it.name.endsWith("JniLibFolders") }.configureEach { - dependsOn(":searchkit:assembleAndroidJavaRuntime") -} - dependencies { implementation(project(":searchkit")) androidTestImplementation(libs.androidx.test.runner) diff --git a/searchkit-check/src/androidTest/kotlin/io/github/kdroidfilter/seforimapp/searchkit/check/SearchParityTest.kt b/searchkit-check/src/androidTest/kotlin/io/github/kdroidfilter/seforimapp/searchkit/check/SearchParityTest.kt index e6e4aadbc..6ca4fe792 100644 --- a/searchkit-check/src/androidTest/kotlin/io/github/kdroidfilter/seforimapp/searchkit/check/SearchParityTest.kt +++ b/searchkit-check/src/androidTest/kotlin/io/github/kdroidfilter/seforimapp/searchkit/check/SearchParityTest.kt @@ -4,7 +4,6 @@ import android.os.SystemClock import androidx.test.ext.junit.runners.AndroidJUnit4 import androidx.test.platform.app.InstrumentationRegistry import io.github.kdroidfilter.seforimapp.features.search.semantic.SemanticModels -import io.github.kdroidfilter.seforimapp.framework.search.AndroidJavaRuntime import io.github.kdroidfilter.seforimapp.framework.search.SearchKitDump import io.github.kdroidfilter.seforimapp.framework.search.SearchServices import io.github.kdroidfilter.seforimapp.framework.search.openSeforimRepository @@ -14,10 +13,9 @@ import org.junit.runner.RunWith import java.io.File /** - * The app's search on this device, as an app runs it: Lucene in the app's Java process. The data goes in the app's - * files dir (adb root, push, chown to the app): `seforim/` (seforim.db, its indexes, lexical.db, `model/`), - * `jdk//` and `jdk-app/` (searchkit's assembleAndroidJavaRuntime). Writes SearchKitDump as `searchkit.txt` there; skipped without - * data. + * The app's search on this device, as an app runs it: Lucene on the JVM NJA runs in the app's process. The data goes + * in the app's files dir (adb root, push, chown to the app): `seforim/` (seforim.db, its indexes, lexical.db, + * `model/`). Writes SearchKitDump as `searchkit.txt` there; skipped without data. */ @RunWith(AndroidJUnit4::class) class SearchParityTest { @@ -26,7 +24,6 @@ class SearchParityTest { val context = InstrumentationRegistry.getInstrumentation().targetContext val dir = File(context.filesDir, "seforim") val db = File(dir, "seforim.db").takeIf { it.isFile } ?: return - AndroidJavaRuntime.configure(context) val model = File(dir, "model") val models = SemanticModels( diff --git a/searchkit/README.md b/searchkit/README.md index 0316f6f7d..a235436d2 100644 --- a/searchkit/README.md +++ b/searchkit/README.md @@ -12,15 +12,15 @@ same text on the JVM, in the iOS simulator and on Android 11 and 15 emulators (` | | Desktop (JVM) | Android 10+ | iOS | |---|---|---|---| -| Lucene (full-text and vector index) | in process | in a Java process of the app's own (server VM) | in the embedded Java runtime (Zero) | +| Lucene (full-text and vector index) | in the app's JVM | on HotSpot in the app's process (NJA) | on Zero linked into the app (NJA) | | Database (`SeforimRepository`) | the app's pooled JDBC driver | `openSeforimRepository(context, path)` | `openSeforimRepository(path)` | | Semantic model (official builds) | ONNX Runtime Java, model in the jar | ONNX Runtime Android | ONNX Runtime C API (`onnxruntime-c` 1.20.0) | | Model files | bundled | downloaded with the data | downloaded with the data | Lucene runs on a real JVM only: ART can't keep up with the Java versions Lucene requires (21 for Lucene 10, 25 for 11). -The mobile runtimes are OpenJDK 27 with openjdk/mobile's port, built by the CI of -[kdroidFilter/openjdk-mobile](https://github.com/kdroidFilter/openjdk-mobile) and downloaded by the build (see -[IOS_RUNTIME.md](../SeforimLibrary/IOS_RUNTIME.md)). +SeforimLibrary's search reaches it through [Nucleus JVM Access](https://github.com/NucleusFramework/NucleusJvmAccess): +its façade (`search-facade`) runs on the JVM, its common code calls the proxies NJA generates; the mobile JVMs are +jdk-mobile's OpenJDK 27 (see [JVM_RUNTIME.md](../SeforimLibrary/JVM_RUNTIME.md)). Everything else (query building, dictionary, ranking, snippets, highlights, RRF fusion, tokenizer, use cases, view models) is common code. @@ -36,17 +36,13 @@ services.engine // SearchEngine: openSession, computeFacets, highlights, sema services.lookup // LuceneLookupSearchService: suggestBooks, suggestAuthors, searchTocPrefix ``` -On Android, call `AndroidJavaRuntime.configure(context)` before the first search: Lucene runs in a Java process, from -the runtime `./gradlew :searchkit:assembleAndroidJavaRuntime` assembles: `jniLibs/` (the launchers, arm64-v8a and -x86_64, among the app's native libraries, with `packaging.jniLibs.useLegacyPackaging = true`), `jdk//` (the JDK of -an ABI: deliver the device's alone) and `jdk-app/` (delivered into its files). -`searchkit-check` is such an app (no UI). On a recent emulator, the process takes ~140 MB of memory with its default -128 MB heap (plus the index pages it maps, which Android reclaims). - -On iOS, `IosJavaRuntime` (`classPath`, `options`) configures the embedded runtime before the first search; by default it -runs the jars of `App.app/lib/app`. `./gradlew :searchkit:assembleIosJavaRuntimeDevice` assembles -`static/` (what the app links, options in IOS_RUNTIME.md) and `lib/` (copy it into the bundle as `App.app/lib` in an -Xcode build phase). +On Android there is nothing to configure: the JVM (`libnja.so`, arm64-v8a and x86_64) and its runtime come with +SeforimLibrary's search library (its native libraries and assets), and start on the first search. `searchkit-check` is +such an app (no UI). + +On iOS, `./gradlew :searchkit:assembleIosJavaRuntimeDevice` assembles `static/` (what the app links: the options are in +JVM_RUNTIME.md) and `bundle/` (copy its content into `App.app` in an Xcode build phase: the VM's `lib/` and NJA's +`nja/`). ### The view models diff --git a/searchkit/build.gradle.kts b/searchkit/build.gradle.kts index 6a1a10cfe..d76f45565 100644 --- a/searchkit/build.gradle.kts +++ b/searchkit/build.gradle.kts @@ -13,7 +13,7 @@ val withSemantic = gradle.extensions.extraProperties["semanticEnabled"] == true /** * Zayit's search, without its UI: the engines (lexical, hybrid with the semantic search), the lookup, the use cases and * the view models of the search screens, for the desktop app and the mobile ones (Android 10+, iOS). Lucene runs in - * process on the JVM, in the Java runtime the mobile apps embed (see SeforimLibrary's IOS_RUNTIME.md). + * process on the JVM, in the Java runtime the mobile apps embed (see SeforimLibrary's JVM_RUNTIME.md). */ kotlin { jvmToolchain( @@ -26,7 +26,7 @@ kotlin { android { namespace = "io.github.kdroidfilter.seforimapp.searchkit" compileSdk = 37 - // Lucene runs in a Java process of the app's own (AndroidJavaRuntime), from Android 10 + // NJA's bridge runs from Android 10 minSdk = 29 } iosArm64() @@ -76,56 +76,43 @@ kotlin { // --- Tests on real data: the same dump on the JVM and in the iOS simulator (SearchKitDump) --- -// The Java runtimes of the mobile apps (OpenJDK's mobile port), downloaded by SeforimLibrary's search (see its -// IOS_RUNTIME.md): -// build/javaRuntime/ there +// The Java runtime of the mobile apps comes with SeforimLibrary's search, through Nucleus JVM Access (see its +// JVM_RUNTIME.md): Android gets it from search's library (its native libraries and assets); an iOS binary links its +// static libraries (search/build/jdkMobile/) and carries NJA's bundle (search/build/nja/ios/) val seforimLibrary = gradle.includedBuild("SeforimLibrary") +val searchBuild = seforimLibrary.projectDir.resolve("search/build") -fun javaRuntime(target: String): File = seforimLibrary.projectDir.resolve("search/build/javaRuntime/$target") - -fun javaRuntimeTask(target: String) = - seforimLibrary.task(":search:javaRuntime${target.split("-").joinToString("") { it.replaceFirstChar(Char::uppercase) }}") +fun searchTask(name: String) = seforimLibrary.task(":search:$name") // ONNX Runtime's xcframework, for the semantic search in the iOS tests: -Pseforim.ortXcframework= val ortXcframework: File? = providers.gradleProperty("seforim.ortXcframework").orNull?.let(::file) -val jdkStaticLibs = - listOf( - "libjvm.a", - "libjava.a", - "libnio.a", - "libnet.a", - "libzip.a", - "libjimage.a", - "libverify.a", - "libextnet.a", - "libfallbackLinker.a", - "libsyslookup.a", - ) +val jdkStaticLibs = listOf("jvm", "java", "nio", "net", "zip", "jimage", "verify", "extnet", "fallbackLinker", "syslookup") // What an iOS binary links for the search: the Java runtime (force-loaded: it finds its natives by name), its libffi, // and ONNX Runtime for the semantic search fun org.jetbrains.kotlin.gradle.plugin.mpp.NativeBinary.linkJavaRuntime( - target: String, + runtime: String, ortSlice: File?, ) { - val static = javaRuntime(target).resolve("static") - linkerOpts(jdkStaticLibs.flatMap { listOf("-force_load", static.resolve(it).path) }) + val static = searchBuild.resolve("jdkMobile/$runtime/static") + linkerOpts(jdkStaticLibs.flatMap { listOf("-force_load", static.resolve("lib$it.a").path) }) linkerOpts(static.resolve("libffi.a").path, "-export_dynamic", "-lz", "-lc++") listOf("Foundation", "CoreFoundation", "Security", "SystemConfiguration").forEach { linkerOpts("-framework", it) } if (ortSlice != null) linkerOpts("-F", ortSlice.path, "-framework", "onnxruntime") - linkTaskProvider.configure { dependsOn(javaRuntimeTask(target)) } + val stage = runtime.split("-").joinToString("") { it.replaceFirstChar(Char::uppercase) } + linkTaskProvider.configure { dependsOn(searchTask("stageJdkMobile$stage")) } } kotlin { iosSimulatorArm64 { binaries.withType().configureEach { - linkJavaRuntime("ios-simulator", ortXcframework?.resolve("ios-arm64_x86_64-simulator")) + linkJavaRuntime("ios-simulator-aarch64", ortXcframework?.resolve("ios-arm64_x86_64-simulator")) } } // A device build of the tests only checks that everything links (no device runs them here) iosArm64 { binaries.withType().configureEach { - linkJavaRuntime("ios-device", ortXcframework?.resolve("ios-arm64")) + linkJavaRuntime("ios-device-aarch64", ortXcframework?.resolve("ios-arm64")) } } } @@ -141,66 +128,35 @@ tasks.named("jvmTest") { doFirst { out.parentFile.mkdirs() } } -// SeforimLibrary's jars for the embedded runtime (its Lucene side, see IOS_RUNTIME.md) -val searchJars = seforimLibrary.projectDir.resolve("search/build/mobileJvmClasspath") -val searchJarsTask = seforimLibrary.task(":search:mobileJvmClasspath") - -// The Java runtime an iOS app bundles as App.app/lib (its java.home): the runtime's classes, conf and data, and the -// search's jars in lib/app (IosJavaRuntime's default class path) -fun Sync.iosJavaRuntime(target: String) { - dependsOn(searchJarsTask, javaRuntimeTask(target)) - from(javaRuntime(target).resolve("lib")) - from(searchJars) { into("app") } - // Some of the JDK's files are read-only: the next sync couldn't replace them - filePermissions { user { write = true } } -} - -// For the Xcode project: copy build/iosJavaRuntime//lib into the app bundle, link build/iosJavaRuntime// -// static (see IOS_RUNTIME.md) -listOf("simulator", "device").forEach { kind -> +// For the Xcode project: build/iosJavaRuntime/: static/ (what the app links, see linkJavaRuntime) and bundle/ +// (lib/, the VM's java.home, and nja/: the façades' class path and the VM's options), copied into App.app +listOf("simulator" to "IosSimulatorArm64", "device" to "IosArm64").forEach { (kind, konan) -> tasks.register("assembleIosJavaRuntime${kind.replaceFirstChar { it.uppercase() }}") { group = "build" - description = "Assembles the Java runtime the iOS app bundles as App.app/lib, and links ($kind)." - dependsOn(searchJarsTask, javaRuntimeTask("ios-$kind")) - from(javaRuntime("ios-$kind").resolve("static")) { into("static") } - from(javaRuntime("ios-$kind").resolve("lib")) { into("lib") } - from(searchJars) { into("lib/app") } + description = "Assembles what the iOS app links and bundles for the search's Java runtime ($kind)." + dependsOn(searchTask("stageJdkMobile${if (kind == "simulator") "IosSimulatorAarch64" else "IosDeviceAarch64"}")) + dependsOn(searchTask("njaIosBundle$konan")) + from(searchBuild.resolve("jdkMobile/ios-$kind-aarch64/static")) { into("static") } + from(searchBuild.resolve("nja/ios/${konan.replaceFirstChar(Char::lowercase)}")) { into("bundle") } into(layout.buildDirectory.dir("iosJavaRuntime/$kind")) filePermissions { user { write = true } } } } -// For an Android app (Lucene in a Java process, see AndroidJavaRuntime): build/androidJavaRuntime/jniLibs (the launchers, -// arm64-v8a and x86_64, among the app's native libraries), jdk/ (the JDK's classes and data, its java.home: some of -// its classes depend on the architecture) and jdk-app (the search's jars), delivered into the app's files (jdk/ of -// the device's ABI alone) -tasks.register("assembleAndroidJavaRuntime") { - group = "build" - description = "Assembles the Java runtime an Android app delivers for Lucene." - dependsOn(searchJarsTask, javaRuntimeTask("android-aarch64"), javaRuntimeTask("android-x64")) - from(javaRuntime("android-aarch64").resolve("jniLibs")) { into("jniLibs") } - from(javaRuntime("android-x64").resolve("jniLibs")) { into("jniLibs") } - from(javaRuntime("android-aarch64").resolve("lib")) { into("jdk/arm64-v8a") } - from(javaRuntime("android-x64").resolve("lib")) { into("jdk/x86_64") } - from(searchJars) { into("jdk-app") } - into(layout.buildDirectory.dir("androidJavaRuntime")) - filePermissions { user { write = true } } -} - run { val testBinaryDir = layout.buildDirectory.dir("bin/iosSimulatorArm64/debugTest") - val iosTestRuntime by tasks.registering(Sync::class) { - iosJavaRuntime("ios-simulator") - into(testBinaryDir.map { it.dir("lib") }) + // NJA's bundle next to the test binary: the VM's java.home is /lib + val iosTestRuntime by tasks.registering(Copy::class) { + dependsOn(searchTask("njaIosBundleIosSimulatorArm64")) + from(searchBuild.resolve("nja/ios/iosSimulatorArm64")) + into(testBinaryDir) + filePermissions { user { write = true } } } tasks.withType().configureEach { dependsOn(iosTestRuntime) device.set(providers.gradleProperty("seforim.iosSimulator").orElse("iPhone 17 Pro")) - val binaryDir = testBinaryDir.get().asFile - val jarsDir = searchJars - val jars = providers.provider { jarsDir.listFiles().orEmpty().map { binaryDir.resolve("lib/app/${it.name}").path } } val models = providers.gradleProperty("seforim.semanticModels") val db = providers.gradleProperty("seforimTestDb") val out = @@ -210,7 +166,6 @@ run { .asFile.path doFirst { // The simulator passes the SIMCTL_CHILD_ variables to the test process, without the prefix - environment("SIMCTL_CHILD_SEFORIM_JVM_CLASSPATH", jars.get().joinToString(":")) db.orNull?.let { environment("SIMCTL_CHILD_SEFORIM_TEST_DB", it) } models.orNull?.let { environment("SIMCTL_CHILD_SEFORIM_MODEL_DIR", it) } environment("SIMCTL_CHILD_SEFORIM_PARITY_OUT", out) diff --git a/searchkit/src/androidMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/LuceneIndexes.android.kt b/searchkit/src/androidMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/LuceneIndexes.android.kt deleted file mode 100644 index 0c58202fd..000000000 --- a/searchkit/src/androidMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/LuceneIndexes.android.kt +++ /dev/null @@ -1,68 +0,0 @@ -package io.github.kdroidfilter.seforimapp.framework.search - -import android.content.Context -import io.github.kdroidfilter.seforimlibrary.search.index.LuceneIndex -import io.github.kdroidfilter.seforimlibrary.search.index.bridge.BridgedLuceneIndex -import io.github.kdroidfilter.seforimlibrary.search.index.bridge.JavaProcessTransport -import java.io.File - -/** - * The Java runtime Lucene runs in on Android, whose ART can't keep up with the Java versions Lucene requires: a process - * of the app's own (see SeforimLibrary's IOS_RUNTIME.md, "Android"). - * - * Call [configure] before the first search; the app delivers the runtime (`./gradlew :searchkit:assembleAndroidJavaRuntime`): - * the launchers in its native libraries (`lib//libjvmlauncher.so`, arm64-v8a and x86_64, installed as files: - * `useLegacyPackaging`), the JDK's classes and data of the device's ABI in [javaHome] and the search's jars in - * [classPath] (both downloaded or extracted from the app's assets). - */ -object AndroidJavaRuntime { - /** The JDK's classes and data (an exploded image: `modules/`, `conf/`, `lib/`). */ - var javaHome: File? = null - - /** The search's jars (SeforimLibrary's `mobileJvmClasspath`, the same as iOS). */ - var classPath: List = emptyList() - - /** The Java process's options: its heap above all. */ - var options: List = listOf("-Xmx128m", "-XX:+UseSerialGC", "-XX:UseSVE=0", "--enable-native-access=ALL-UNNAMED") - - private var launcher: File? = null - - /** - * Uses the runtime the app delivers in [context]'s files: `jdk//` (the ABI of the launcher Android installed) - * and the jars of `jdk-app/`, unless set otherwise. - */ - fun configure( - context: Context, - javaHome: File = File(context.filesDir, "jdk/${launcherAbi(context)}"), - classPath: List = File(context.filesDir, "jdk-app").listFiles { file -> file.extension == "jar" }.orEmpty().sorted(), - ) { - launcher = File(context.applicationInfo.nativeLibraryDir, "libjvmlauncher.so") - // Android has no /tmp: the app's cache instead - val tmp = context.cacheDir.path - options = options.filterNot { it.startsWith("-Djava.io.tmpdir=") || it.startsWith("-XX:AltTempDir=") } + - listOf("-Djava.io.tmpdir=$tmp", "-XX:AltTempDir=$tmp") - this.javaHome = javaHome - this.classPath = classPath - } - - // The native library dir is named after the ABI Android installed the app's libraries for (lib/arm64, lib/x86_64) - private fun launcherAbi(context: Context): String = - when (val dir = File(context.applicationInfo.nativeLibraryDir).name) { - "arm64" -> "arm64-v8a" - else -> dir - } - - val transport: JavaProcessTransport by lazy { - JavaProcessTransport( - launcher = checkNotNull(launcher) { "AndroidJavaRuntime.configure(context) first: Lucene runs in its Java process" }, - javaHome = checkNotNull(javaHome), - classPath = classPath, - options = options, - ) - } -} - -actual fun openLuceneIndex( - indexDir: String, - cacheQueries: Boolean, -): LuceneIndex = BridgedLuceneIndex(AndroidJavaRuntime.transport, indexDir, cacheQueries) diff --git a/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/LuceneLookupSearchService.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/LuceneLookupSearchService.kt index eb1487168..e9d00391e 100644 --- a/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/LuceneLookupSearchService.kt +++ b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/LuceneLookupSearchService.kt @@ -8,23 +8,24 @@ import io.github.kdroidfilter.seforimlibrary.search.index.IndexQuery import io.github.kdroidfilter.seforimlibrary.search.index.LuceneIndex import io.github.kdroidfilter.seforimlibrary.search.index.Occur import io.github.kdroidfilter.seforimlibrary.search.index.SortField +import io.github.kdroidfilter.seforimlibrary.search.index.SortType /** * Book, author and table-of-contents lookup over the lookup index (seforim.db.lookup.lucene), with acronym-aware - * ranking of the books: [index] is that index (opened with its query cache, see `LuceneIndexBackend`), read in process on - * the JVM and Android, through the embedded Java runtime on iOS. + * ranking of the books: [index] is that index (opened with its query cache: [openLuceneIndex]), read on a JVM like the + * others. */ class LuceneLookupSearchService( private val index: LuceneIndex, private val acronymCache: AcronymFrequencyCache? = null, ) { - private val authorSort = listOf(SortField("book_count", SortField.Type.LONG, reverse = true)) + private val authorSort = listOf(SortField("book_count", SortType.LONG, reverse = true)) private val bookSort = listOf( - SortField("is_base_book", SortField.Type.INT, reverse = true), - SortField("order_index", SortField.Type.INT), - SortField("book_id", SortField.Type.LONG), + SortField("is_base_book", SortType.INT, reverse = true), + SortField("order_index", SortType.INT), + SortField("book_id", SortType.LONG), ) data class TocHit( diff --git a/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/SearchServices.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/SearchServices.kt index b2ad2d051..3528eab4f 100644 --- a/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/SearchServices.kt +++ b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/SearchServices.kt @@ -9,14 +9,14 @@ import io.github.kdroidfilter.seforimlibrary.search.SearchEngine import io.github.kdroidfilter.seforimlibrary.search.index.LuceneIndex /** - * The Lucene index at [indexDir] on this platform: read in process on the JVM and Android (Lucene on ART, API 35+), - * through the Java runtime the app embeds on iOS (see [IosJavaRuntime][io.github.kdroidfilter.seforimapp.framework.search]). - * [cacheQueries]: cache any query seen often (the small lookup index). + * The Lucene index at [indexDir], read on a JVM through Nucleus JVM Access (SeforimLibrary's search-facade): the + * desktop's own, the one the mobile apps embed (in the app's process on Android and iOS). [cacheQueries]: cache any + * query seen often (the small lookup index). */ -expect fun openLuceneIndex( +fun openLuceneIndex( indexDir: String, cacheQueries: Boolean = false, -): LuceneIndex +): LuceneIndex = LuceneIndex(indexDir, cacheQueries) /** * The search of the database at [databasePath] and of the indexes beside it, as the desktop and the mobile apps set it diff --git a/searchkit/src/iosMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/LuceneIndexes.ios.kt b/searchkit/src/iosMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/LuceneIndexes.ios.kt deleted file mode 100644 index 01927f039..000000000 --- a/searchkit/src/iosMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/LuceneIndexes.ios.kt +++ /dev/null @@ -1,43 +0,0 @@ -@file:OptIn(ExperimentalForeignApi::class) - -package io.github.kdroidfilter.seforimapp.framework.search - -import io.github.kdroidfilter.seforimlibrary.search.index.LuceneIndex -import io.github.kdroidfilter.seforimlibrary.search.index.bridge.EmbeddedJvm -import io.github.kdroidfilter.seforimlibrary.search.index.bridge.JvmLuceneIndex -import kotlinx.cinterop.ExperimentalForeignApi -import platform.Foundation.NSBundle -import platform.Foundation.NSFileManager - -/** - * The Java runtime the iOS app embeds for Lucene (OpenJDK's mobile port, statically linked: see SeforimLibrary's - * IOS_RUNTIME.md). It starts with the first index opened; set these before. - */ -object IosJavaRuntime { - /** The jars of the search (SeforimLibrary's `mobileJvmClasspath`): by default those of `/lib/app`. */ - var classPath: List = bundledJars() - - /** - * The runtime's options: its heap above all (Lucene's caches live there). 128 MB keeps the app's footprint around - * 160 MB on real data; a larger heap only grows it (the GC runs less), 32 MB fails. - */ - var options: List = listOf("-Xmx128m") - - val jvm: EmbeddedJvm by lazy { EmbeddedJvm.start(classPath, options) } - - private fun bundledJars(): List { - val dir = NSBundle.mainBundle.bundlePath + "/lib/app" - return NSFileManager.defaultManager - .contentsOfDirectoryAtPath(dir, null) - .orEmpty() - .map { it.toString() } - .filter { it.endsWith(".jar") } - .sorted() - .map { "$dir/$it" } - } -} - -actual fun openLuceneIndex( - indexDir: String, - cacheQueries: Boolean, -): LuceneIndex = JvmLuceneIndex(IosJavaRuntime.jvm, indexDir, cacheQueries) diff --git a/searchkit/src/iosSimulatorArm64Test/kotlin/io/github/kdroidfilter/seforimapp/framework/search/IosSearchKitParityTest.kt b/searchkit/src/iosSimulatorArm64Test/kotlin/io/github/kdroidfilter/seforimapp/framework/search/IosSearchKitParityTest.kt index ec1c31582..25497864a 100644 --- a/searchkit/src/iosSimulatorArm64Test/kotlin/io/github/kdroidfilter/seforimapp/framework/search/IosSearchKitParityTest.kt +++ b/searchkit/src/iosSimulatorArm64Test/kotlin/io/github/kdroidfilter/seforimapp/framework/search/IosSearchKitParityTest.kt @@ -20,7 +20,6 @@ class IosSearchKitParityTest { fun dump() { val env = NSProcessInfo.processInfo.environment val db = env["SEFORIM_TEST_DB"] as String? ?: return - IosJavaRuntime.classPath = (env["SEFORIM_JVM_CLASSPATH"] as String).split(':') val models = (env["SEFORIM_MODEL_DIR"] as String?)?.let { SemanticModels("$it/seforim-embed-v5-int8.onnx", "$it/tokenizer.json", "$it/seforim-embed-v5-static.i8") diff --git a/searchkit/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/LuceneIndexes.jvm.kt b/searchkit/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/CliSemanticSearch.jvm.kt similarity index 67% rename from searchkit/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/LuceneIndexes.jvm.kt rename to searchkit/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/CliSemanticSearch.jvm.kt index fc8ff7b67..ac0c6e6e0 100644 --- a/searchkit/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/LuceneIndexes.jvm.kt +++ b/searchkit/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/CliSemanticSearch.jvm.kt @@ -3,15 +3,8 @@ package io.github.kdroidfilter.seforimapp.framework.search import io.github.kdroidfilter.seforimlibrary.dao.repository.SeforimRepository import io.github.kdroidfilter.seforimlibrary.search.LuceneSearchEngine import io.github.kdroidfilter.seforimlibrary.search.SearchEngine -import io.github.kdroidfilter.seforimlibrary.search.index.LuceneIndex -import io.github.kdroidfilter.seforimlibrary.search.index.LuceneIndexBackend import java.nio.file.Path -actual fun openLuceneIndex( - indexDir: String, - cacheQueries: Boolean, -): LuceneIndex = LuceneIndexBackend(Path.of(indexDir), cacheQueries = cacheQueries) - /** [withSemanticSearch] for the engine `zayit cli` builds (see `CliHost`): the semantic search reads its index. */ fun withSemanticSearch( lexical: LuceneSearchEngine, From 16f49d62ace5e248d4d39fb21cfd23186b185674 Mon Sep 17 00:00:00 2001 From: Elie Gambache Date: Sat, 17 Oct 2026 19:05:58 +0300 Subject: [PATCH 05/12] build: SeforimLibrary on Nucleus JVM Access 0.2.0 No more -Pnja.local: the search uses NJA's release. Signed-off-by: Elie Gambache --- CLAUDE.md | 2 +- SeforimLibrary | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 30be39515..4aabf1ebc 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -129,7 +129,7 @@ The search without its UI (engines, lookup, use cases, `SearchResultViewModel`/` on a real JVM only (ART can't keep up with Lucene's Java versions), through Nucleus JVM Access: SeforimLibrary's `search-facade` (Kotlin/JVM) is the façade, its common code uses the `LuceneIndex`/`IndexQuery`... NJA generates; the desktop's own JVM, HotSpot in the app's process on Android, Zero linked into the app on iOS (jdk-mobile's OpenJDK 27); -see `SeforimLibrary/JVM_RUNTIME.md`. Until an NJA release has what it needs, build with `-Pnja.local=`. The view models reach the +see `SeforimLibrary/JVM_RUNTIME.md`. The view models reach the desktop shell only through the ports of `SearchHost.kt` (`SearchTabs`, `SearchSessionStore`, `SearchSettings`, `SearchHistory`); keep `java.*`, AWT and desktop types out of `searchkit/src/commonMain`. diff --git a/SeforimLibrary b/SeforimLibrary index 81d97254c..cc1198daf 160000 --- a/SeforimLibrary +++ b/SeforimLibrary @@ -1 +1 @@ -Subproject commit 81d97254c65fe17a80f54ea2148f1305d590cb6a +Subproject commit cc1198daf3f0112d515817b12e0d3f218c89cefa From c394a3699d52e52eeecee86a6bb0f5509784f81d Mon Sep 17 00:00:00 2001 From: Elie Gambache Date: Sat, 17 Oct 2026 19:31:35 +0300 Subject: [PATCH 06/12] build: searchkit's iOS binaries on the runtimes NJA downloads The runtimes are in search/build/nja/jdkMobile now, behind NJA's njaJdkMobile tasks (NJA 0.3.0); SeforimLibrary at its commit using them. Signed-off-by: Elie Gambache --- SeforimLibrary | 2 +- searchkit/build.gradle.kts | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/SeforimLibrary b/SeforimLibrary index cc1198daf..33ea06e3e 160000 --- a/SeforimLibrary +++ b/SeforimLibrary @@ -1 +1 @@ -Subproject commit cc1198daf3f0112d515817b12e0d3f218c89cefa +Subproject commit 33ea06e3e5d161b46e5fa20d6d05b6ace5d0fefc diff --git a/searchkit/build.gradle.kts b/searchkit/build.gradle.kts index d76f45565..f9f15ea52 100644 --- a/searchkit/build.gradle.kts +++ b/searchkit/build.gradle.kts @@ -78,7 +78,8 @@ kotlin { // The Java runtime of the mobile apps comes with SeforimLibrary's search, through Nucleus JVM Access (see its // JVM_RUNTIME.md): Android gets it from search's library (its native libraries and assets); an iOS binary links its -// static libraries (search/build/jdkMobile/) and carries NJA's bundle (search/build/nja/ios/) +// static libraries (search/build/nja/jdkMobile/, which NJA downloads) and carries NJA's bundle +// (search/build/nja/ios/) val seforimLibrary = gradle.includedBuild("SeforimLibrary") val searchBuild = seforimLibrary.projectDir.resolve("search/build") @@ -94,13 +95,13 @@ fun org.jetbrains.kotlin.gradle.plugin.mpp.NativeBinary.linkJavaRuntime( runtime: String, ortSlice: File?, ) { - val static = searchBuild.resolve("jdkMobile/$runtime/static") + val static = searchBuild.resolve("nja/jdkMobile/$runtime/static") linkerOpts(jdkStaticLibs.flatMap { listOf("-force_load", static.resolve("lib$it.a").path) }) linkerOpts(static.resolve("libffi.a").path, "-export_dynamic", "-lz", "-lc++") listOf("Foundation", "CoreFoundation", "Security", "SystemConfiguration").forEach { linkerOpts("-framework", it) } if (ortSlice != null) linkerOpts("-F", ortSlice.path, "-framework", "onnxruntime") val stage = runtime.split("-").joinToString("") { it.replaceFirstChar(Char::uppercase) } - linkTaskProvider.configure { dependsOn(searchTask("stageJdkMobile$stage")) } + linkTaskProvider.configure { dependsOn(searchTask("njaJdkMobile$stage")) } } kotlin { @@ -134,9 +135,9 @@ listOf("simulator" to "IosSimulatorArm64", "device" to "IosArm64").forEach { (ki tasks.register("assembleIosJavaRuntime${kind.replaceFirstChar { it.uppercase() }}") { group = "build" description = "Assembles what the iOS app links and bundles for the search's Java runtime ($kind)." - dependsOn(searchTask("stageJdkMobile${if (kind == "simulator") "IosSimulatorAarch64" else "IosDeviceAarch64"}")) + dependsOn(searchTask("njaJdkMobile${if (kind == "simulator") "IosSimulatorAarch64" else "IosDeviceAarch64"}")) dependsOn(searchTask("njaIosBundle$konan")) - from(searchBuild.resolve("jdkMobile/ios-$kind-aarch64/static")) { into("static") } + from(searchBuild.resolve("nja/jdkMobile/ios-$kind-aarch64/static")) { into("static") } from(searchBuild.resolve("nja/ios/${konan.replaceFirstChar(Char::lowercase)}")) { into("bundle") } into(layout.buildDirectory.dir("iosJavaRuntime/$kind")) filePermissions { user { write = true } } From 670dfab75396edd9e1b83d7ba050a023bf6c0b95 Mon Sep 17 00:00:00 2001 From: Elie Gambache Date: Sat, 17 Oct 2026 21:58:37 +0300 Subject: [PATCH 07/12] build: SeforimLibrary reading the index through NIO on Android Signed-off-by: Elie Gambache --- SeforimLibrary | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SeforimLibrary b/SeforimLibrary index 33ea06e3e..d313b0dad 160000 --- a/SeforimLibrary +++ b/SeforimLibrary @@ -1 +1 @@ -Subproject commit 33ea06e3e5d161b46e5fa20d6d05b6ace5d0fefc +Subproject commit d313b0dad5f271cf428193eedf3d195806dc331b From 2b0ca4d14dcc833a116cc2d345e3a848925df0ca Mon Sep 17 00:00:00 2001 From: Elie Gambache Date: Sun, 18 Oct 2026 00:28:04 +0300 Subject: [PATCH 08/12] build: searchkit's iOS binaries on SeforimLibrary's GraalVM native image The search's Lucene runs compiled ahead of time on iOS (Nucleus JVM Access 0.4.0): about 8x faster than on Zero, nothing to bundle. assembleIosJavaRuntime* assembles the image and graalvm-mobile's libraries. Signed-off-by: Elie Gambache --- CLAUDE.md | 3 ++- SeforimLibrary | 2 +- searchkit/README.md | 16 +++++------ searchkit/build.gradle.kts | 54 +++++++++++++++++--------------------- 4 files changed, 35 insertions(+), 40 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 4aabf1ebc..c01893041 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -128,7 +128,8 @@ The search without its UI (engines, lookup, use cases, `SearchResultViewModel`/` `searchkit` (JVM, Android 10+, iOS), shared by the desktop and the mobile apps; see `searchkit/README.md`. Lucene runs on a real JVM only (ART can't keep up with Lucene's Java versions), through Nucleus JVM Access: SeforimLibrary's `search-facade` (Kotlin/JVM) is the façade, its common code uses the `LuceneIndex`/`IndexQuery`... NJA generates; the -desktop's own JVM, HotSpot in the app's process on Android, Zero linked into the app on iOS (jdk-mobile's OpenJDK 27); +desktop's own JVM, HotSpot in the app's process on Android (jdk-mobile's OpenJDK 27), a GraalVM native image linked +into the app on iOS (graalvm-mobile); see `SeforimLibrary/JVM_RUNTIME.md`. The view models reach the desktop shell only through the ports of `SearchHost.kt` (`SearchTabs`, `SearchSessionStore`, `SearchSettings`, `SearchHistory`); keep `java.*`, AWT and desktop types out of `searchkit/src/commonMain`. diff --git a/SeforimLibrary b/SeforimLibrary index d313b0dad..6f7ee1288 160000 --- a/SeforimLibrary +++ b/SeforimLibrary @@ -1 +1 @@ -Subproject commit d313b0dad5f271cf428193eedf3d195806dc331b +Subproject commit 6f7ee128826ff088b07de4e7a7ccc1af201ef72a diff --git a/searchkit/README.md b/searchkit/README.md index a235436d2..b38210ee8 100644 --- a/searchkit/README.md +++ b/searchkit/README.md @@ -12,15 +12,16 @@ same text on the JVM, in the iOS simulator and on Android 11 and 15 emulators (` | | Desktop (JVM) | Android 10+ | iOS | |---|---|---|---| -| Lucene (full-text and vector index) | in the app's JVM | on HotSpot in the app's process (NJA) | on Zero linked into the app (NJA) | +| Lucene (full-text and vector index) | in the app's JVM | on HotSpot in the app's process (NJA) | compiled into a GraalVM native image, linked into the app (NJA) | | Database (`SeforimRepository`) | the app's pooled JDBC driver | `openSeforimRepository(context, path)` | `openSeforimRepository(path)` | | Semantic model (official builds) | ONNX Runtime Java, model in the jar | ONNX Runtime Android | ONNX Runtime C API (`onnxruntime-c` 1.20.0) | | Model files | bundled | downloaded with the data | downloaded with the data | -Lucene runs on a real JVM only: ART can't keep up with the Java versions Lucene requires (21 for Lucene 10, 25 for 11). -SeforimLibrary's search reaches it through [Nucleus JVM Access](https://github.com/NucleusFramework/NucleusJvmAccess): -its façade (`search-facade`) runs on the JVM, its common code calls the proxies NJA generates; the mobile JVMs are -jdk-mobile's OpenJDK 27 (see [JVM_RUNTIME.md](../SeforimLibrary/JVM_RUNTIME.md)). +Lucene runs on Java only, never on ART: ART can't keep up with the Java versions Lucene requires (21 for Lucene 10, 25 +for 11). SeforimLibrary's search reaches it through [Nucleus JVM Access](https://github.com/NucleusFramework/NucleusJvmAccess): +its façade (`search-facade`) runs on the JVM, or compiled ahead of time on iOS, and its common code calls the proxies +NJA generates. Android's JVM is jdk-mobile's OpenJDK 27; iOS's native image is GraalVM CE's, with graalvm-mobile's +libraries (see [JVM_RUNTIME.md](../SeforimLibrary/JVM_RUNTIME.md)). Everything else (query building, dictionary, ranking, snippets, highlights, RRF fusion, tokenizer, use cases, view models) is common code. @@ -40,9 +41,8 @@ On Android there is nothing to configure: the JVM (`libnja.so`, arm64-v8a and x8 SeforimLibrary's search library (its native libraries and assets), and start on the first search. `searchkit-check` is such an app (no UI). -On iOS, `./gradlew :searchkit:assembleIosJavaRuntimeDevice` assembles `static/` (what the app links: the options are in -JVM_RUNTIME.md) and `bundle/` (copy its content into `App.app` in an Xcode build phase: the VM's `lib/` and NJA's -`nja/`). +On iOS, `./gradlew :searchkit:assembleIosJavaRuntimeDevice` assembles `static/`: what the app links, the search's +native image (`nja.o`) and graalvm-mobile's libraries (the options are in JVM_RUNTIME.md). Nothing goes into `App.app`. ### The view models diff --git a/searchkit/build.gradle.kts b/searchkit/build.gradle.kts index f9f15ea52..2698834fb 100644 --- a/searchkit/build.gradle.kts +++ b/searchkit/build.gradle.kts @@ -78,8 +78,8 @@ kotlin { // The Java runtime of the mobile apps comes with SeforimLibrary's search, through Nucleus JVM Access (see its // JVM_RUNTIME.md): Android gets it from search's library (its native libraries and assets); an iOS binary links its -// static libraries (search/build/nja/jdkMobile/, which NJA downloads) and carries NJA's bundle -// (search/build/nja/ios/) +// GraalVM native image (search/build/nja/nativeImage//nja.o, which NJA builds) with graalvm-mobile's static +// libraries (search/build/nja/graalvmMobile//lib, which NJA downloads): nothing to bundle val seforimLibrary = gradle.includedBuild("SeforimLibrary") val searchBuild = seforimLibrary.projectDir.resolve("search/build") @@ -87,21 +87,23 @@ fun searchTask(name: String) = seforimLibrary.task(":search:$name") // ONNX Runtime's xcframework, for the semantic search in the iOS tests: -Pseforim.ortXcframework= val ortXcframework: File? = providers.gradleProperty("seforim.ortXcframework").orNull?.let(::file) -val jdkStaticLibs = listOf("jvm", "java", "nio", "net", "zip", "jimage", "verify", "extnet", "fallbackLinker", "syslookup") +// Substrate VM's and the JDK's natives, from graalvm-mobile's kit +val nativeImageLibs = listOf("jvm", "libchelper", "darwin", "java", "nio", "net", "zip", "extnet", "jimage", "verify", "management_ext", "management", "prefs") -// What an iOS binary links for the search: the Java runtime (force-loaded: it finds its natives by name), its libffi, -// and ONNX Runtime for the semantic search +fun searchStage(runtime: String) = runtime.split("-").joinToString("") { it.replaceFirstChar(Char::uppercase) } + +// What an iOS binary links for the search: its native image (entry points exported: NJA's client finds them at run +// time), the kit's libraries, and ONNX Runtime for the semantic search fun org.jetbrains.kotlin.gradle.plugin.mpp.NativeBinary.linkJavaRuntime( runtime: String, ortSlice: File?, ) { - val static = searchBuild.resolve("nja/jdkMobile/$runtime/static") - linkerOpts(jdkStaticLibs.flatMap { listOf("-force_load", static.resolve("lib$it.a").path) }) - linkerOpts(static.resolve("libffi.a").path, "-export_dynamic", "-lz", "-lc++") - listOf("Foundation", "CoreFoundation", "Security", "SystemConfiguration").forEach { linkerOpts("-framework", it) } + val lib = searchBuild.resolve("nja/graalvmMobile/$runtime/lib") + linkerOpts(searchBuild.resolve("nja/nativeImage/$runtime/nja.o").path) + linkerOpts(nativeImageLibs.map { lib.resolve("lib$it.a").path }) + linkerOpts("-export_dynamic", "-lz", "-lc++", "-framework", "Foundation", "-framework", "CoreFoundation") if (ortSlice != null) linkerOpts("-F", ortSlice.path, "-framework", "onnxruntime") - val stage = runtime.split("-").joinToString("") { it.replaceFirstChar(Char::uppercase) } - linkTaskProvider.configure { dependsOn(searchTask("njaJdkMobile$stage")) } + linkTaskProvider.configure { dependsOn(searchTask("njaNativeImage${searchStage(runtime)}")) } } kotlin { @@ -129,34 +131,26 @@ tasks.named("jvmTest") { doFirst { out.parentFile.mkdirs() } } -// For the Xcode project: build/iosJavaRuntime/: static/ (what the app links, see linkJavaRuntime) and bundle/ -// (lib/, the VM's java.home, and nja/: the façades' class path and the VM's options), copied into App.app -listOf("simulator" to "IosSimulatorArm64", "device" to "IosArm64").forEach { (kind, konan) -> +// For the Xcode project: build/iosJavaRuntime//static: what the app links (see linkJavaRuntime), the native +// image (nja.o) and the kit's libraries; nothing to copy into App.app +listOf("simulator", "device").forEach { kind -> tasks.register("assembleIosJavaRuntime${kind.replaceFirstChar { it.uppercase() }}") { group = "build" - description = "Assembles what the iOS app links and bundles for the search's Java runtime ($kind)." - dependsOn(searchTask("njaJdkMobile${if (kind == "simulator") "IosSimulatorAarch64" else "IosDeviceAarch64"}")) - dependsOn(searchTask("njaIosBundle$konan")) - from(searchBuild.resolve("nja/jdkMobile/ios-$kind-aarch64/static")) { into("static") } - from(searchBuild.resolve("nja/ios/${konan.replaceFirstChar(Char::lowercase)}")) { into("bundle") } + description = "Assembles what the iOS app links for the search's native image ($kind)." + val runtime = "ios-$kind-aarch64" + dependsOn(searchTask("njaNativeImage${searchStage(runtime)}")) + from(searchBuild.resolve("nja/nativeImage/$runtime/nja.o")) { into("static") } + from(searchBuild.resolve("nja/graalvmMobile/$runtime/lib")) { + include(nativeImageLibs.map { "lib$it.a" }) + into("static") + } into(layout.buildDirectory.dir("iosJavaRuntime/$kind")) filePermissions { user { write = true } } } } run { - val testBinaryDir = layout.buildDirectory.dir("bin/iosSimulatorArm64/debugTest") - - // NJA's bundle next to the test binary: the VM's java.home is /lib - val iosTestRuntime by tasks.registering(Copy::class) { - dependsOn(searchTask("njaIosBundleIosSimulatorArm64")) - from(searchBuild.resolve("nja/ios/iosSimulatorArm64")) - into(testBinaryDir) - filePermissions { user { write = true } } - } - tasks.withType().configureEach { - dependsOn(iosTestRuntime) device.set(providers.gradleProperty("seforim.iosSimulator").orElse("iPhone 17 Pro")) val models = providers.gradleProperty("seforim.semanticModels") val db = providers.gradleProperty("seforimTestDb") From e7cba8354c4b21c8fc79b2c956dd8578ff18d043 Mon Sep 17 00:00:00 2001 From: Elie Gambache Date: Sun, 18 Oct 2026 00:45:59 +0300 Subject: [PATCH 09/12] build: SeforimLibrary's caches on ConcurrentHashMap off iOS Signed-off-by: Elie Gambache --- SeforimLibrary | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SeforimLibrary b/SeforimLibrary index 6f7ee1288..4576d7ee2 160000 --- a/SeforimLibrary +++ b/SeforimLibrary @@ -1 +1 @@ -Subproject commit 6f7ee128826ff088b07de4e7a7ccc1af201ef72a +Subproject commit 4576d7ee273aadade54a80d5da868c10cc000b30 From b3f9beb74ac7c91c9c58df02c93bbcc233b3a587 Mon Sep 17 00:00:00 2001 From: Elie Gambache Date: Sun, 18 Oct 2026 00:56:43 +0300 Subject: [PATCH 10/12] feat(htmlparser): Android and iOS targets, the DOM from jsoup or Ksoup HtmlParser reads a small DOM of its own (HtmlDom.kt), which jsoup builds on the JVM and Android and its Kotlin port Ksoup on iOS: the parser itself is common code, its tests pass on both. Signed-off-by: Elie Gambache --- gradle/libs.versions.toml | 2 ++ htmlparser/build.gradle.kts | 24 ++++++++++++++-- .../seforim/htmlparser/HtmlDom.androidJvm.kt | 26 +++++++++++++++++ .../seforim/htmlparser/HtmlDom.kt | 28 +++++++++++++++++++ .../seforim/htmlparser/HtmlParser.kt | 26 ++++++++--------- .../seforim/htmlparser/HtmlDom.ios.kt | 27 ++++++++++++++++++ 6 files changed, 116 insertions(+), 17 deletions(-) create mode 100644 htmlparser/src/androidJvmMain/kotlin/io/github/kdroidfilter/seforim/htmlparser/HtmlDom.androidJvm.kt create mode 100644 htmlparser/src/commonMain/kotlin/io/github/kdroidfilter/seforim/htmlparser/HtmlDom.kt create mode 100644 htmlparser/src/iosMain/kotlin/io/github/kdroidfilter/seforim/htmlparser/HtmlDom.ios.kt diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index f43367ff9..97d035c66 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -7,6 +7,7 @@ confettikit = "0.9.0" filekitCore = "0.16.0" hebrewNumerals = "0.2.6" jsoup = "1.23.2" +ksoup = "0.2.6" jvmToolchain = "25" nucleus = "3.0.0-dev-202610132318" koalaplotCore = "0.12.1" @@ -76,6 +77,7 @@ filekit-dialogs-compose = { module = "io.github.vinceglb:filekit-dialogs-compose hebrew-numerals = { module = "io.github.kdroidfilter:hebrewnumerals", version.ref = "hebrewNumerals" } jdbc-driver = { module = "app.cash.sqldelight:jdbc-driver", version.ref = "sqlDelight" } jsoup = { module = "org.jsoup:jsoup", version.ref = "jsoup" } +ksoup = { module = "com.fleeksoft.ksoup:ksoup", version.ref = "ksoup" } nucleus-core-runtime = { module = "dev.nucleusframework:nucleus.core-runtime", version.ref = "nucleus" } nucleus-aot-runtime = { module = "dev.nucleusframework:nucleus.aot-runtime", version.ref = "nucleus" } nucleus-darkmode-detector = { module = "dev.nucleusframework:nucleus.darkmode-detector", version.ref = "nucleus" } diff --git a/htmlparser/build.gradle.kts b/htmlparser/build.gradle.kts index 7307ea3ca..01e3d8b9c 100644 --- a/htmlparser/build.gradle.kts +++ b/htmlparser/build.gradle.kts @@ -4,6 +4,7 @@ plugins { alias(libs.plugins.multiplatform) alias(libs.plugins.compose.compiler) alias(libs.plugins.compose) + alias(libs.plugins.android.kmp.library) } kotlin { @@ -14,20 +15,39 @@ kotlin { ) jvm() + android { + namespace = "io.github.kdroidfilter.seforim.htmlparser" + compileSdk = 37 + minSdk = 21 + } + iosArm64() + iosSimulatorArm64() sourceSets { commonMain.dependencies { implementation(libs.compose.runtime) implementation(libs.compose.foundation) - implementation(libs.jsoup) } commonTest.dependencies { implementation(kotlin("test")) } - jvmMain.dependencies { + // The DOM: jsoup on the JVM and Android, its Kotlin port Ksoup on iOS (see HtmlDom.kt) + val androidJvmMain by creating { + dependsOn(commonMain.get()) + dependencies { implementation(libs.jsoup) } + } + jvmMain { dependsOn(androidJvmMain) } + androidMain { dependsOn(androidJvmMain) } + + // The manual source set above turns off the default hierarchy: iOS shares its own one by hand + val iosMain by creating { + dependsOn(commonMain.get()) + dependencies { implementation(libs.ksoup) } } + iosArm64Main { dependsOn(iosMain) } + iosSimulatorArm64Main { dependsOn(iosMain) } } } diff --git a/htmlparser/src/androidJvmMain/kotlin/io/github/kdroidfilter/seforim/htmlparser/HtmlDom.androidJvm.kt b/htmlparser/src/androidJvmMain/kotlin/io/github/kdroidfilter/seforim/htmlparser/HtmlDom.androidJvm.kt new file mode 100644 index 000000000..887a3806f --- /dev/null +++ b/htmlparser/src/androidJvmMain/kotlin/io/github/kdroidfilter/seforim/htmlparser/HtmlDom.androidJvm.kt @@ -0,0 +1,26 @@ +package io.github.kdroidfilter.seforim.htmlparser + +import org.jsoup.Jsoup +import org.jsoup.nodes.Element +import org.jsoup.nodes.Node +import org.jsoup.nodes.TextNode + +internal actual fun parseHtmlBody(html: String): List = + Jsoup + .parse(html) + .body() + .childNodes() + .mapNotNull(::toHtmlNode) + +// Text and elements: comments and the like carry nothing to render +private fun toHtmlNode(node: Node): HtmlNode? = + when (node) { + is TextNode -> HtmlText(node.text()) + is Element -> + HtmlElement( + tag = node.normalName(), + attributes = node.attributes().associate { it.key to it.value }, + children = node.childNodes().mapNotNull(::toHtmlNode), + ) + else -> null + } diff --git a/htmlparser/src/commonMain/kotlin/io/github/kdroidfilter/seforim/htmlparser/HtmlDom.kt b/htmlparser/src/commonMain/kotlin/io/github/kdroidfilter/seforim/htmlparser/HtmlDom.kt new file mode 100644 index 000000000..1b89effe8 --- /dev/null +++ b/htmlparser/src/commonMain/kotlin/io/github/kdroidfilter/seforim/htmlparser/HtmlDom.kt @@ -0,0 +1,28 @@ +package io.github.kdroidfilter.seforim.htmlparser + +/** A node of a parsed HTML body: what [HtmlParser] reads of jsoup's DOM (JVM, Android) or Ksoup's (iOS). */ +internal sealed interface HtmlNode + +internal class HtmlText( + val text: String, +) : HtmlNode + +internal class HtmlElement( + /** The tag's name, lower-case. */ + val tag: String, + private val attributes: Map, + val children: List, +) : HtmlNode { + /** The attribute [name]'s value, empty when absent. */ + fun attr(name: String): String = attributes[name].orEmpty() + + /** Whether the `class` attribute lists [name] (case-insensitive, as jsoup). */ + fun hasClass(name: String): Boolean = attr("class").split(CLASS_SEPARATOR).any { it.equals(name, ignoreCase = true) } + + private companion object { + val CLASS_SEPARATOR = Regex("\\s+") + } +} + +/** The nodes of [html]'s body, parsed as a browser would. */ +internal expect fun parseHtmlBody(html: String): List diff --git a/htmlparser/src/commonMain/kotlin/io/github/kdroidfilter/seforim/htmlparser/HtmlParser.kt b/htmlparser/src/commonMain/kotlin/io/github/kdroidfilter/seforim/htmlparser/HtmlParser.kt index 76cea1477..555037fa9 100644 --- a/htmlparser/src/commonMain/kotlin/io/github/kdroidfilter/seforim/htmlparser/HtmlParser.kt +++ b/htmlparser/src/commonMain/kotlin/io/github/kdroidfilter/seforim/htmlparser/HtmlParser.kt @@ -1,9 +1,5 @@ package io.github.kdroidfilter.seforim.htmlparser -import org.jsoup.Jsoup -import org.jsoup.nodes.Element -import org.jsoup.nodes.Node -import org.jsoup.nodes.TextNode import kotlin.io.encoding.Base64 import kotlin.io.encoding.ExperimentalEncodingApi @@ -36,9 +32,8 @@ private val WHITESPACE_REGEX = Regex("\\s+") class HtmlParser { fun parse(html: String): List { - val doc = Jsoup.parse(html) val out = mutableListOf() - for (child in doc.body().childNodes()) { + for (child in parseHtmlBody(html)) { processNode( node = child, list = out, @@ -62,7 +57,7 @@ class HtmlParser { } private fun processNode( - node: Node, + node: HtmlNode, list: MutableList, isBold: Boolean, isItalic: Boolean, @@ -76,10 +71,10 @@ class HtmlParser { href: String?, ) { when (node) { - is TextNode -> { + is HtmlText -> { appendSegment( list = list, - textRaw = node.text(), + textRaw = node.text, isBold = isBold, isItalic = isItalic, isSmall = isSmall, @@ -92,8 +87,8 @@ class HtmlParser { href = href, ) } - is Element -> { - val tag = node.normalName() + is HtmlElement -> { + val tag = node.tag if (tag == "br") { appendLineBreak(list) @@ -121,10 +116,11 @@ class HtmlParser { val nextFootnoteContent = isFootnoteContent || isFootnoteContentTag val nextHref = if (tag == "a" && node.hasClass("ref")) node.attr("href").ifBlank { null } else href - if (node.childNodeSize() == 1 && node.childNode(0) is TextNode) { + val onlyText = node.children.singleOrNull() as? HtmlText + if (onlyText != null) { appendSegment( list = list, - textRaw = (node.childNode(0) as TextNode).text(), + textRaw = onlyText.text, isBold = nextBold, isItalic = nextItalic, isSmall = nextSmall, @@ -139,7 +135,7 @@ class HtmlParser { return } - for (child in node.childNodes()) { + for (child in node.children) { processNode( node = child, list = list, @@ -273,7 +269,7 @@ class HtmlParser { // Non-data URIs are ignored (no network fetch — images must be pre-embedded in the DB). private fun appendImage( list: MutableList, - node: Element, + node: HtmlElement, ) { val src = node.attr("src").takeIf { it.isNotBlank() } ?: return val decoded = decodeDataUri(src) ?: return diff --git a/htmlparser/src/iosMain/kotlin/io/github/kdroidfilter/seforim/htmlparser/HtmlDom.ios.kt b/htmlparser/src/iosMain/kotlin/io/github/kdroidfilter/seforim/htmlparser/HtmlDom.ios.kt new file mode 100644 index 000000000..371b015f8 --- /dev/null +++ b/htmlparser/src/iosMain/kotlin/io/github/kdroidfilter/seforim/htmlparser/HtmlDom.ios.kt @@ -0,0 +1,27 @@ +package io.github.kdroidfilter.seforim.htmlparser + +import com.fleeksoft.ksoup.Ksoup +import com.fleeksoft.ksoup.nodes.Element +import com.fleeksoft.ksoup.nodes.Node +import com.fleeksoft.ksoup.nodes.TextNode + +// Ksoup is jsoup's Kotlin port: the same DOM as on the JVM and Android +internal actual fun parseHtmlBody(html: String): List = + Ksoup + .parse(html) + .body() + .childNodes() + .mapNotNull(::toHtmlNode) + +// Text and elements: comments and the like carry nothing to render +private fun toHtmlNode(node: Node): HtmlNode? = + when (node) { + is TextNode -> HtmlText(node.text()) + is Element -> + HtmlElement( + tag = node.normalName(), + attributes = node.attributes().associate { it.key to it.value }, + children = node.childNodes().mapNotNull(::toHtmlNode), + ) + else -> null + } From 8b12a650f6f6e2e462fc8e08c0fadd3b471be8c2 Mon Sep 17 00:00:00 2001 From: Elie Gambache Date: Sun, 18 Oct 2026 01:05:31 +0300 Subject: [PATCH 11/12] refactor(searchkit): its caches on SeforimLibrary's :cache, cachemap on iOS only Signed-off-by: Elie Gambache --- SeforimLibrary | 2 +- gradle/libs.versions.toml | 3 +-- searchkit/build.gradle.kts | 19 +++++++++++++++++-- .../features/search/SearchHomeViewModel.kt | 4 ++-- .../search/domain/BookDetailsLoader.kt | 4 ++-- .../framework/search/AcronymFrequencyCache.kt | 8 ++++---- 6 files changed, 27 insertions(+), 13 deletions(-) diff --git a/SeforimLibrary b/SeforimLibrary index 4576d7ee2..3f563d94e 160000 --- a/SeforimLibrary +++ b/SeforimLibrary @@ -1 +1 @@ -Subproject commit 4576d7ee273aadade54a80d5da868c10cc000b30 +Subproject commit 3f563d94ef84793bef2a5547ce5bbc06dd9426e2 diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 97d035c66..6b5cf9f42 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -43,7 +43,6 @@ seforimSemantic = "0.6.0" kermit = "2.1.0" androidx-test-runner = "1.7.0" androidx-test-junit = "1.3.0" -cachemap = "0.3.0" atomicfu = "0.33.0" kotlinx-io = "0.9.1" seforimQuoteLinks = "0.1.0" @@ -156,7 +155,6 @@ seforim-semantic = { module = "io.github.kdroidfilter:seforim-semantic", version kermit = { module = "co.touchlab:kermit", version.ref = "kermit" } androidx-test-runner = { module = "androidx.test:runner", version.ref = "androidx-test-runner" } androidx-test-junit = { module = "androidx.test.ext:junit", version.ref = "androidx-test-junit" } -cachemap = { module = "io.github.charlietap:cachemap", version.ref = "cachemap" } atomicfu = { module = "org.jetbrains.kotlinx:atomicfu", version.ref = "atomicfu" } kotlinx-io-core = { module = "org.jetbrains.kotlinx:kotlinx-io-core", version.ref = "kotlinx-io" } androidx-lifecycle-viewmodel-core = { module = "org.jetbrains.androidx.lifecycle:lifecycle-viewmodel", version.ref = "androidx-lifecycle" } @@ -179,6 +177,7 @@ compose-ui-test = { module = "org.jetbrains.compose.ui:ui-test-junit4", version. # SeforimLibrary composite build seforimlibrary-core = { module = "io.github.kdroidfilter.seforimlibrary:core" } +seforimlibrary-cache = { module = "io.github.kdroidfilter.seforimlibrary:cache" } seforimlibrary-dao = { module = "io.github.kdroidfilter.seforimlibrary:dao" } seforimlibrary-search = { module = "io.github.kdroidfilter.seforimlibrary:search" } seforimlibrary-cli = { module = "io.github.kdroidfilter.seforimlibrary:cli" } diff --git a/searchkit/build.gradle.kts b/searchkit/build.gradle.kts index 2698834fb..0a887ee3e 100644 --- a/searchkit/build.gradle.kts +++ b/searchkit/build.gradle.kts @@ -50,7 +50,7 @@ kotlin { implementation(libs.compose.runtime) implementation(libs.hebrew.numerals) implementation(libs.kermit) - implementation(libs.cachemap) + implementation(libs.seforimlibrary.cache) implementation(libs.atomicfu) if (withSemantic) implementation(libs.seforim.semantic) } @@ -88,7 +88,22 @@ fun searchTask(name: String) = seforimLibrary.task(":search:$name") // ONNX Runtime's xcframework, for the semantic search in the iOS tests: -Pseforim.ortXcframework= val ortXcframework: File? = providers.gradleProperty("seforim.ortXcframework").orNull?.let(::file) // Substrate VM's and the JDK's natives, from graalvm-mobile's kit -val nativeImageLibs = listOf("jvm", "libchelper", "darwin", "java", "nio", "net", "zip", "extnet", "jimage", "verify", "management_ext", "management", "prefs") +val nativeImageLibs = + listOf( + "jvm", + "libchelper", + "darwin", + "java", + "nio", + "net", + "zip", + "extnet", + "jimage", + "verify", + "management_ext", + "management", + "prefs", + ) fun searchStage(runtime: String) = runtime.split("-").joinToString("") { it.replaceFirstChar(Char::uppercase) } diff --git a/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchHomeViewModel.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchHomeViewModel.kt index ec1c7c26c..970c5a057 100644 --- a/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchHomeViewModel.kt +++ b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchHomeViewModel.kt @@ -3,7 +3,6 @@ package io.github.kdroidfilter.seforimapp.features.search import androidx.compose.runtime.Immutable import androidx.lifecycle.ViewModel import androidx.lifecycle.viewModelScope -import io.github.charlietap.cachemap.cacheMapOf import io.github.kdroidfilter.seforim.tabs.SearchScope import io.github.kdroidfilter.seforim.tabs.TabsDestination import io.github.kdroidfilter.seforimapp.core.coroutines.runSuspendCatching @@ -17,6 +16,7 @@ import io.github.kdroidfilter.seforimapp.framework.search.LuceneLookupSearchServ import io.github.kdroidfilter.seforimapp.framework.search.LuceneLookupSearchService.AuthorHit import io.github.kdroidfilter.seforimapp.framework.search.MIN_BOOK_QUERY_LENGTH import io.github.kdroidfilter.seforimapp.framework.session.SearchPersistedState +import io.github.kdroidfilter.seforimlibrary.cache.concurrentCache import io.github.kdroidfilter.seforimlibrary.core.models.Book import io.github.kdroidfilter.seforimlibrary.core.models.TocEntry import io.github.kdroidfilter.seforimlibrary.dao.repository.SeforimRepository @@ -134,7 +134,7 @@ class SearchHomeViewModel( private val referenceResolver = ReferenceResolver(RepositoryReferenceSource(repository)) // Author aliases by id, to tell which one a query was typed as; few authors have any - private val aliasesByAuthor = cacheMapOf>() + private val aliasesByAuthor = concurrentCache>() private suspend fun authorAliases(authorId: Long): List = aliasesByAuthor[authorId] ?: runSuspendCatching { repository.getAuthorDetails(authorId)?.aliases.orEmpty() } diff --git a/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/BookDetailsLoader.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/BookDetailsLoader.kt index dad27197f..2e8fab501 100644 --- a/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/BookDetailsLoader.kt +++ b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/domain/BookDetailsLoader.kt @@ -1,8 +1,8 @@ package io.github.kdroidfilter.seforimapp.features.search.domain import co.touchlab.kermit.Logger -import io.github.charlietap.cachemap.cacheMapOf import io.github.kdroidfilter.seforimapp.core.coroutines.runSuspendCatching +import io.github.kdroidfilter.seforimlibrary.cache.concurrentCache import io.github.kdroidfilter.seforimlibrary.core.models.Book import io.github.kdroidfilter.seforimlibrary.dao.repository.SeforimRepository @@ -13,7 +13,7 @@ import io.github.kdroidfilter.seforimlibrary.dao.repository.SeforimRepository class BookDetailsLoader( private val repository: SeforimRepository, ) { - private val cache = cacheMapOf() + private val cache = concurrentCache() /** A book with its categories, authors and main parts, once per book; null (logged) when it fails. */ suspend fun load(bookId: Long): SearchEntity.BookEntity? = diff --git a/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/AcronymFrequencyCache.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/AcronymFrequencyCache.kt index 29cb5dc4e..a3a8a93b8 100644 --- a/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/AcronymFrequencyCache.kt +++ b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/framework/search/AcronymFrequencyCache.kt @@ -1,7 +1,7 @@ package io.github.kdroidfilter.seforimapp.framework.search import co.touchlab.kermit.Logger -import io.github.charlietap.cachemap.cacheMapOf +import io.github.kdroidfilter.seforimlibrary.cache.concurrentCache import io.github.kdroidfilter.seforimlibrary.search.SqliteReader import io.github.kdroidfilter.seforimlibrary.search.openReadOnlySqlite @@ -23,13 +23,13 @@ class AcronymFrequencyCache( } // Cache: acronym → frequency (number of books using it) - private val frequencyCache = cacheMapOf() + private val frequencyCache = concurrentCache() // Cache: acronym → list of book titles - private val bookTitlesCache = cacheMapOf>() + private val bookTitlesCache = concurrentCache>() // Cache: book title → list of acronyms - private val titleToAcronymsCache = cacheMapOf>() + private val titleToAcronymsCache = concurrentCache>() init { // Pre-load common acronyms into cache From f6c6ae7198150bfe04e5cf8d9ab53d19deebca6c Mon Sep 17 00:00:00 2001 From: Elie Gambache Date: Sun, 18 Oct 2026 01:20:44 +0300 Subject: [PATCH 12/12] feat(searchkit): the reader's find-in-book on every platform, with a multiplatform logger BookFindMatches and the Hebrew text helpers it uses (HebrewSearch, DiacriticsMode) move to searchkit's common code, with their tests. The logger becomes multiplatform: kotlinx-datetime for its timestamps, Sentry KMP for its reports (the same Sentry Java SDK the desktop initializes; Sentry Cocoa, through SwiftPM, on iOS). Signed-off-by: Elie Gambache --- .gitignore | 3 ++ gradle/libs.versions.toml | 3 ++ logger/build.gradle.kts | 17 ++++--- .../kdroidfilter/seforimapp/logger/Logger.kt | 44 ++++++++++++++----- searchkit/build.gradle.kts | 5 +++ .../core/presentation/text/DiacriticsMode.kt | 2 +- .../core/presentation/text/HebrewSearch.kt | 10 ++--- .../bookcontent/usecases/BookFindMatches.kt | 8 ++-- .../presentation/text/DiacriticsModeTest.kt | 0 .../presentation/text/HebrewSearchTest.kt | 0 .../usecases/BookFindMatchesTest.kt | 8 ++-- 11 files changed, 69 insertions(+), 31 deletions(-) rename {SeforimApp/src/jvmMain => searchkit/src/commonMain}/kotlin/io/github/kdroidfilter/seforimapp/core/presentation/text/DiacriticsMode.kt (96%) rename {SeforimApp/src/jvmMain => searchkit/src/commonMain}/kotlin/io/github/kdroidfilter/seforimapp/core/presentation/text/HebrewSearch.kt (96%) rename {SeforimApp/src/jvmMain => searchkit/src/commonMain}/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/usecases/BookFindMatches.kt (96%) rename {SeforimApp/src/jvmTest => searchkit/src/commonTest}/kotlin/io/github/kdroidfilter/seforimapp/core/presentation/text/DiacriticsModeTest.kt (100%) rename {SeforimApp/src/jvmTest => searchkit/src/commonTest}/kotlin/io/github/kdroidfilter/seforimapp/core/presentation/text/HebrewSearchTest.kt (100%) rename {SeforimApp/src/jvmTest => searchkit/src/commonTest}/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/usecases/BookFindMatchesTest.kt (95%) diff --git a/.gitignore b/.gitignore index aba81386d..c6ffd130c 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,6 @@ Pods/ *.gpg *yarn.lock .claude/ + +# Kotlin SwiftPM import (Sentry Cocoa for the iOS binaries) +.swiftpm-locks/ diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 6b5cf9f42..a6cf0f9f2 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -57,6 +57,7 @@ mockk = "1.14.11" kotlinx-coroutines-test = "1.11.0" sentry = "6.23.0" sentrySdk = "8.58.0" +sentryKmp = "0.28.0" graalHotspot = "22.0.0.2" intellijPlatformIcons = "262.10968.63" jbrApi = "1.10.1" @@ -188,6 +189,7 @@ structured-coroutines-annotations = { module = "io.github.santimattius:structure # Sentry sentry-core = { module = "io.sentry:sentry", version.ref = "sentrySdk" } +sentry-kmp = { module = "io.sentry:sentry-kotlin-multiplatform", version.ref = "sentryKmp" } graal-hotspot-library = { module = "org.graalvm.nativeimage:graal-hotspot-library", version.ref = "graalHotspot" } # Testing @@ -221,3 +223,4 @@ structured-coroutines = { id = "io.github.santimattius.structured-coroutines", v kover = { id = "org.jetbrains.kotlinx.kover", version.ref = "kover" } ksp = { id = "com.google.devtools.ksp", version.ref = "ksp" } sentryJvmGradle = { id = "io.sentry.jvm.gradle", version.ref = "sentry" } +sentryKmpGradle = { id = "io.sentry.kotlin.multiplatform.gradle", version.ref = "sentryKmp" } diff --git a/logger/build.gradle.kts b/logger/build.gradle.kts index 510fb2852..bbdd028fb 100644 --- a/logger/build.gradle.kts +++ b/logger/build.gradle.kts @@ -1,11 +1,20 @@ plugins { alias(libs.plugins.multiplatform) alias(libs.plugins.kotlinx.serialization) + alias(libs.plugins.android.kmp.library) } +// The app's logs: the console in development, Sentry once the app initializes it (the desktop: main.kt), on every +// platform. On iOS the app links Sentry's Cocoa SDK, which Sentry KMP calls. kotlin { - jvm() + android { + namespace = "io.github.kdroidfilter.seforimapp.logger" + compileSdk = 37 + minSdk = 21 + } + iosArm64() + iosSimulatorArm64() jvmToolchain( libs.versions.jvmToolchain .get() @@ -14,10 +23,8 @@ kotlin { sourceSets { commonMain.dependencies { - } - - jvmMain.dependencies { - implementation(libs.sentry.core) + implementation(libs.kotlinx.datetime) + implementation(libs.sentry.kmp) } } } diff --git a/logger/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/logger/Logger.kt b/logger/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/logger/Logger.kt index 1130a5408..f9796076a 100644 --- a/logger/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/logger/Logger.kt +++ b/logger/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/logger/Logger.kt @@ -1,9 +1,15 @@ package io.github.kdroidfilter.seforimapp.logger -import io.sentry.Sentry -import io.sentry.SentryLevel -import java.text.SimpleDateFormat -import java.util.Date +import io.sentry.kotlin.multiplatform.Sentry +import io.sentry.kotlin.multiplatform.SentryLevel +import kotlinx.datetime.LocalDate +import kotlinx.datetime.LocalDateTime +import kotlinx.datetime.TimeZone +import kotlinx.datetime.format.char +import kotlinx.datetime.toLocalDateTime +import kotlin.concurrent.Volatile +import kotlin.jvm.JvmField +import kotlin.time.Clock var isDevEnv: Boolean = true var loggingLevel: LoggingLevel = LoggingLevel.VERBOSE @@ -40,9 +46,24 @@ class LoggingLevel( @PublishedApi internal const val COLOR_RESET = "\u001b[0m" -@PublishedApi internal val dateFormat = SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS") +// yyyy-MM-dd HH:mm:ss.SSS +@PublishedApi internal val dateFormat = + LocalDateTime.Format { + date(LocalDate.Formats.ISO) + char(' ') + hour() + char(':') + minute() + char(':') + second() + char('.') + secondFraction(3) + } -@PublishedApi internal fun getCurrentTimestamp(): String = dateFormat.format(Date()) +@PublishedApi internal fun getCurrentTimestamp(): String = + dateFormat.format( + Clock.System.now().toLocalDateTime(TimeZone.currentSystemDefault()), + ) @PublishedApi internal fun shouldLogToConsole(minLevel: LoggingLevel): Boolean = isDevEnv && loggingLevel.priority <= minLevel.priority @@ -79,14 +100,13 @@ class LoggingLevel( if (sendToSentry) { runCatching { val sentryLevel = toSentryLevel(minLevel) - Sentry.withScope { scope -> - scope.level = sentryLevel - if (throwable != null) { + if (throwable != null) { + Sentry.captureException(throwable) { scope -> + scope.level = sentryLevel scope.setExtra("logger.message", renderedMessage) - Sentry.captureException(throwable) - } else { - Sentry.captureMessage(renderedMessage, sentryLevel) } + } else { + Sentry.captureMessage(renderedMessage) { scope -> scope.level = sentryLevel } } } } diff --git a/searchkit/build.gradle.kts b/searchkit/build.gradle.kts index 0a887ee3e..8f1292836 100644 --- a/searchkit/build.gradle.kts +++ b/searchkit/build.gradle.kts @@ -4,6 +4,8 @@ plugins { alias(libs.plugins.kotlinx.serialization) // The view models contribute their factories to the app's graph (the desktop's AppGraph) alias(libs.plugins.metro) + // The logger (find-in-book) reports to Sentry: its Cocoa SDK, which the iOS binaries link, installed through SwiftPM + alias(libs.plugins.sentryKmpGradle) } // The semantic search is in the official builds only (open core): with its package's token or its sources beside @@ -47,6 +49,9 @@ kotlin { implementation(libs.androidx.lifecycle.viewmodel.savedstate) implementation(libs.metrox.viewmodel) api(project(":navigation")) + // The reader's find-in-book (BookFindMatches): lines as plain text, its logs + implementation(project(":htmlparser")) + implementation(project(":logger")) implementation(libs.compose.runtime) implementation(libs.hebrew.numerals) implementation(libs.kermit) diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/core/presentation/text/DiacriticsMode.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/core/presentation/text/DiacriticsMode.kt similarity index 96% rename from SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/core/presentation/text/DiacriticsMode.kt rename to searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/core/presentation/text/DiacriticsMode.kt index c63da053d..0b5236e91 100644 --- a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/core/presentation/text/DiacriticsMode.kt +++ b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/core/presentation/text/DiacriticsMode.kt @@ -35,7 +35,7 @@ private const val METEG = '\u05BD' private fun isTeamim(c: Char): Boolean = c.code in 0x0591..0x05AF /** True for nikud (vowel points, meteg included) and ta'amim (cantillation) characters. */ -internal fun isNikudOrTeamim(c: Char): Boolean = +fun isNikudOrTeamim(c: Char): Boolean = isTeamim(c) || (c.code in 0x05B0..0x05BD) || (c == '\u05C1') || diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/core/presentation/text/HebrewSearch.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/core/presentation/text/HebrewSearch.kt similarity index 96% rename from SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/core/presentation/text/HebrewSearch.kt rename to searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/core/presentation/text/HebrewSearch.kt index 8020849a3..25c6bf311 100644 --- a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/core/presentation/text/HebrewSearch.kt +++ b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/core/presentation/text/HebrewSearch.kt @@ -55,7 +55,7 @@ fun stripNikudTeamimWithMap( return out.toString() to arr } -internal fun replaceFinalsWithBase(text: String): String = +fun replaceFinalsWithBase(text: String): String = text .replace('\u05DA', '\u05DB') // ך -> כ .replace('\u05DD', '\u05DE') // ם -> מ @@ -63,7 +63,7 @@ internal fun replaceFinalsWithBase(text: String): String = .replace('\u05E3', '\u05E4') // ף -> פ .replace('\u05E5', '\u05E6') // ץ -> צ -internal fun normalizeQueryForHebrew(raw: String): String { +fun normalizeQueryForHebrew(raw: String): String { if (raw.isBlank()) return "" var s = raw.trim() // Remove biblical cantillation marks (teamim) U+0591–U+05AF @@ -80,7 +80,7 @@ internal fun normalizeQueryForHebrew(raw: String): String { return s } -internal fun mapToOrigIndex( +fun mapToOrigIndex( mapToOrig: IntArray, plainIndex: Int, ): Int { @@ -132,13 +132,13 @@ fun mapOriginalToStripped( * Find all diacritic-insensitive matches of [query] in [text], returning original * character index ranges [start, end) suitable for highlighting. */ -internal fun findAllMatchesOriginal( +fun findAllMatchesOriginal( text: String, query: String, ): List = findAllMatchesNormalized(text, normalizeQueryForHebrew(query)) /** [findAllMatchesOriginal] for a query already through [normalizeQueryForHebrew]: scanning many lines normalizes it once. */ -internal fun findAllMatchesNormalized( +fun findAllMatchesNormalized( text: String, q: String, ): List { diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/usecases/BookFindMatches.kt b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/usecases/BookFindMatches.kt similarity index 96% rename from SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/usecases/BookFindMatches.kt rename to searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/usecases/BookFindMatches.kt index 146e092ac..00309754f 100644 --- a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/usecases/BookFindMatches.kt +++ b/searchkit/src/commonMain/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/usecases/BookFindMatches.kt @@ -10,7 +10,7 @@ import io.github.kdroidfilter.seforimlibrary.search.SearchEngine import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.ensureActive import kotlinx.coroutines.withContext -import java.io.IOException +import kotlinx.io.IOException /** * The matches of the find-in-page query in a book, in reading order. Only each matching line's @@ -41,7 +41,7 @@ class BookFindMatches internal constructor( current: Match, forward: Boolean, ): Match? { - val i = lineIndices.binarySearch(current.lineIndex) + val i = lineIndices.asList().binarySearch(current.lineIndex) if (i < 0) return nextAfterLine(current.lineIndex, forward) val ordinal = current.ordinal + if (forward) 1 else -1 return when { @@ -60,7 +60,7 @@ class BookFindMatches internal constructor( forward: Boolean, ): Match? { if (lineIndices.isEmpty()) return null - val pos = lineIndices.binarySearch(fromLineIndex) + val pos = lineIndices.asList().binarySearch(fromLineIndex) return if (forward) { matchAt((if (pos >= 0) pos + 1 else -pos - 1).takeIf { it < lineIndices.size } ?: 0, 0) } else { @@ -71,7 +71,7 @@ class BookFindMatches internal constructor( /** Whether [match] is one of these matches: a match kept from older results may no longer be. */ operator fun contains(match: Match): Boolean { - val i = lineIndices.binarySearch(match.lineIndex) + val i = lineIndices.asList().binarySearch(match.lineIndex) return i >= 0 && lineIds[i] == match.lineId && match.ordinal in 0 until countIn(i) && diff --git a/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/core/presentation/text/DiacriticsModeTest.kt b/searchkit/src/commonTest/kotlin/io/github/kdroidfilter/seforimapp/core/presentation/text/DiacriticsModeTest.kt similarity index 100% rename from SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/core/presentation/text/DiacriticsModeTest.kt rename to searchkit/src/commonTest/kotlin/io/github/kdroidfilter/seforimapp/core/presentation/text/DiacriticsModeTest.kt diff --git a/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/core/presentation/text/HebrewSearchTest.kt b/searchkit/src/commonTest/kotlin/io/github/kdroidfilter/seforimapp/core/presentation/text/HebrewSearchTest.kt similarity index 100% rename from SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/core/presentation/text/HebrewSearchTest.kt rename to searchkit/src/commonTest/kotlin/io/github/kdroidfilter/seforimapp/core/presentation/text/HebrewSearchTest.kt diff --git a/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/usecases/BookFindMatchesTest.kt b/searchkit/src/commonTest/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/usecases/BookFindMatchesTest.kt similarity index 95% rename from SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/usecases/BookFindMatchesTest.kt rename to searchkit/src/commonTest/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/usecases/BookFindMatchesTest.kt index 8da662cb5..cb780a93d 100644 --- a/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/usecases/BookFindMatchesTest.kt +++ b/searchkit/src/commonTest/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/usecases/BookFindMatchesTest.kt @@ -107,7 +107,7 @@ class BookFindMatchesTest { } @Test - fun `confirms candidates literally, nikud ignored, in reading order`() = + fun `confirms candidates literally - nikud ignored - in reading order`() = runTest { val matches = find("ברא") // Twice in line 0 (inside בראשית, then ברא), twice in line 2 @@ -117,14 +117,14 @@ class BookFindMatchesTest { } @Test - fun `matches against the displayed text, not the html`() = + fun `matches against the displayed text - not the html`() = runTest { assertEquals(0, find("").occurrences) assertEquals(1, find("והארץ").occurrences) } @Test - fun `steps through every match, line after line, wrapping both ways`() = + fun `steps through every match - line after line - wrapping both ways`() = runTest { val matches = find("ברא") val first = matches.nextAfterLine(-1, forward = true)!! @@ -137,7 +137,7 @@ class BookFindMatchesTest { } @Test - fun `whole-line matches, one occurrence each`() { + fun `whole-line matches - one occurrence each`() { val matches = BookFindMatches.ofLines(lines.reversed()) assertEquals(3, matches.occurrences) assertEquals(BookFindMatches.Match(1, 11, ordinal = 0, position = 2), matches.nextAfterLine(0, forward = true))