diff --git a/build.gradle.kts b/build.gradle.kts index bfebff0a..b585548f 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -63,6 +63,7 @@ tasks.register("generateSeforimDb") { dependsOn(":sefariasqlite:synthesizeSimanNamesAltToc") dependsOn(":sefariasqlite:inheritChaptersAltToc") dependsOn(":sefariasqlite:buildInlineMarkerAltToc") + dependsOn(":sefariasqlite:buildGemaraDafAltToc") dependsOn(":packaging:writeReleaseInfo") dependsOn(":packaging:downloadLexicalDb") // Stamps schema_meta.db_version into the produced seforim.db so the @@ -89,6 +90,7 @@ val lineContentWriters = listOf( ":sefariasqlite:synthesizeSimanNamesAltToc", ":sefariasqlite:inheritChaptersAltToc", ":sefariasqlite:buildInlineMarkerAltToc", + ":sefariasqlite:buildGemaraDafAltToc", ":generator-common:buildLineRefIndex", ":generator-common:buildLineDhIndex", ":sefariasqlite:buildLineDivineNameIndex", @@ -205,6 +207,7 @@ project(":generator-common").tasks.matching { it.name == "buildLineRefIndex" }.c mustRunAfter(":sefariasqlite:synthesizeSimanNamesAltToc") mustRunAfter(":sefariasqlite:inheritChaptersAltToc") mustRunAfter(":sefariasqlite:buildInlineMarkerAltToc") + mustRunAfter(":sefariasqlite:buildGemaraDafAltToc") } // line_dh is derived from line.content, so it must be rebuilt after every @@ -237,6 +240,7 @@ project(":sefariasqlite").tasks.matching { it.name == "buildLineDivineNameIndex" mustRunAfter(":sefariasqlite:synthesizeSimanNamesAltToc") mustRunAfter(":sefariasqlite:inheritChaptersAltToc") mustRunAfter(":sefariasqlite:buildInlineMarkerAltToc") + mustRunAfter(":sefariasqlite:buildGemaraDafAltToc") mustRunAfter(":generator-common:buildLineRefIndex") mustRunAfter(":generator-common:buildLineDhIndex") } @@ -337,6 +341,9 @@ project(":sefariasqlite").tasks.matching { it.name == "buildInlineMarkerAltToc" mustRunAfter(":sefariasqlite:synthesizeSimanNamesAltToc") mustRunAfter(":sefariasqlite:inheritChaptersAltToc") } +project(":sefariasqlite").tasks.matching { it.name == "buildGemaraDafAltToc" }.configureEach { + mustRunAfter(":sefariasqlite:buildInlineMarkerAltToc") +} project(":catalog").tasks.matching { it.name == "buildCatalog" }.configureEach { mustRunAfter(":generator-common:buildLineNoteIndex") mustRunAfter(":otzariasqlite:generateHavroutaLinks") @@ -347,6 +354,7 @@ project(":catalog").tasks.matching { it.name == "buildCatalog" }.configureEach { mustRunAfter(":sefariasqlite:synthesizeSimanNamesAltToc") mustRunAfter(":sefariasqlite:inheritChaptersAltToc") mustRunAfter(":sefariasqlite:buildInlineMarkerAltToc") + mustRunAfter(":sefariasqlite:buildGemaraDafAltToc") mustRunAfter(":generator-common:buildLineDhIndex") mustRunAfter(":sefariasqlite:buildLineDivineNameIndex") } diff --git a/generator/sefariasqlite/build.gradle.kts b/generator/sefariasqlite/build.gradle.kts index 078c2982..510ce9a1 100644 --- a/generator/sefariasqlite/build.gradle.kts +++ b/generator/sefariasqlite/build.gradle.kts @@ -480,6 +480,34 @@ tasks.register("buildInlineMarkerAltToc") { jvmArgs = listOf("-Xmx1g") } +// Alt-TOC by the Gemara daf cited inline (`(דף ח:)`), for the books in ForDB +// gemara_daf_alt_toc.csv. Runs right after buildInlineMarkerAltToc. +// Usage: +// ./gradlew :sefariasqlite:buildGemaraDafAltToc -PseforimDb=/path/to/seforim.db +tasks.register("buildGemaraDafAltToc") { + group = "application" + description = "Build Gemara-daf alt-TOCs for the books in ForDB gemara_daf_alt_toc.csv." + + dependsOn("jvmJar") + mainClass.set("io.github.kdroidfilter.seforimlibrary.sefariasqlite.GemaraDafAltTocPostProcessKt") + mustRunAfter(tasks.matching { it.name == "buildInlineMarkerAltToc" }) + classpath = files(tasks.named("jvmJar")) + configurations.getByName("jvmRuntimeClasspath") + + if (project.hasProperty("seforimDb")) { + systemProperty("seforimDb", project.property("seforimDb") as String) + } else if (System.getenv("SEFORIM_DB") != null) { + systemProperty("seforimDb", System.getenv("SEFORIM_DB")) + } else { + val defaultDbPath = rootProject.layout.buildDirectory.file("seforim.db").get().asFile.absolutePath + systemProperty("seforimDb", defaultDbPath) + } + if (project.hasProperty("buildStatePath")) { + systemProperty("buildStatePath", project.property("buildStatePath") as String) + } + + jvmArgs = listOf("-Xmx1g") +} + // Each ForDB consumer runs as its own JVM. When the release build pins the // ForDB archive (-PforDbArchive + -PforDbSha256), forward both properties to all // of them so they read byte-identical inputs instead of resolving the repository @@ -498,6 +526,7 @@ tasks.matching { "seedAllMetadata", "seedBookNotices", "buildInlineMarkerAltToc", + "buildGemaraDafAltToc", "buildLineDivineNameIndex", "reportLineDivineNames", "validateForDbInputs", diff --git a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/GemaraDafAltTocPostProcess.kt b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/GemaraDafAltTocPostProcess.kt new file mode 100644 index 00000000..727feabe --- /dev/null +++ b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/GemaraDafAltTocPostProcess.kt @@ -0,0 +1,185 @@ +package io.github.kdroidfilter.seforimlibrary.sefariasqlite + +import co.touchlab.kermit.Logger +import io.github.kdroidfilter.seforimlibrary.common.ids.altTocChildPath +import java.sql.Connection + +/** + * Alt-TOC by the Gemara daf a commentary cites inline — `(דף ח:)` or `(דף יז ע"א)` in the + * middle of a Rif paragraph — for the books listed in ForDB [GEMARA_DAF_ALT_TOC_FILE]. + * + * Entries follow the Gemara's order, one per amud; an amud the text reaches from more + * than one main-TOC heading (Rif page) gets a child per heading, in reading order. + * Each line belongs to the entry of the last marker at or before it. + * Like [INLINE_MARKER_ALT_TOC_FILE], the stage owns the rule's structure key for the + * listed books, and ids come from the build-state allocator keyed on ancestor paths. + * + * Usage: + * ./gradlew :sefariasqlite:buildGemaraDafAltToc -PseforimDb=/path/to/seforim.db + */ +fun main(args: Array) = + runMarkerAltTocStage(args, "GemaraDafAltToc", GEMARA_DAF_ALT_TOC_FILE, ::parseGemaraDafRules) { conn, rules, ids -> + writeGemaraDafAltTocs(conn, rules, ids) + } + +internal const val GEMARA_DAF_ALT_TOC_FILE = "gemara_daf_alt_toc.csv" +private const val GEMARA_DAF_HEADER = "bookName,structureKey,structureHeTitle" + +internal data class GemaraDafRule(val bookTitle: String, val structureKey: String, val structureHeTitle: String) + +/** Fails on the first malformed row: the build must not silently drop a rule. */ +internal fun parseGemaraDafRules(lines: List): List { + val rows = forDbText(lines).lines().map { it.trimEnd('\r') }.filter { it.isNotBlank() } + if (rows.isEmpty()) return emptyList() + require(rows.first().trim() == GEMARA_DAF_HEADER) { + "$GEMARA_DAF_ALT_TOC_FILE: header must be '$GEMARA_DAF_HEADER'" + } + val rules = rows.drop(1).mapIndexed { i, row -> + val cells = row.split(',').map { it.trim() } + val where = "$GEMARA_DAF_ALT_TOC_FILE row ${i + 2}" + require(cells.size == 3) { "$where: expected 3 cells, got ${cells.size}" } + val (book, key, heTitle) = cells + require(book.isNotEmpty() && heTitle.isNotEmpty()) { "$where: bookName and structureHeTitle are required" } + require(STRUCTURE_KEY_REGEX.matches(key) && key !in RESERVED_STRUCTURE_KEYS) { + "$where: structureKey '$key' must be an English identifier outside $RESERVED_STRUCTURE_KEYS" + } + GemaraDafRule(book, key, heTitle) + } + val duplicate = rules.groupBy { it.bookTitle }.filterValues { it.size > 1 }.keys + require(duplicate.isEmpty()) { "$GEMARA_DAF_ALT_TOC_FILE: more than one rule for $duplicate" } + return rules +} + +/** One amud; [daf] keeps the letters as written, [value] orders and identifies it. */ +internal data class GemaraAmud(val daf: String, val value: Int, val side: Char) { + /** The app finds the entry's line by searching for `(text)` or its ע"א/ע"ב form. */ + val text get() = "דף $daf$side" + val key get() = value * 2 + if (side == '.') 0 else 1 +} + +// Nothing but the daf and amud inside round brackets: `(דף ח: ע"ש)` and `(דף לעיל מח:)` +// are references. A stray extra `(` is common in the text, so `((דף` counts too. +private val GEMARA_DAF_MARKER = Regex("""(?]*>") +private val WHITESPACE = Regex("""\s+""") +private val REFERENCE_WORD = Regex("""ו?(?:ד|כד)?(?:לקמן|לעיל|בפרק|בפירקא)""") + +/** `לקמן ((דף יט ע"ב)`: one of the three words before a `((` marker makes it a reference. */ +private fun followsReferenceWord(textBefore: String): Boolean = + textBefore.replace(HTML_TAG, " ").split(WHITESPACE).filter { it.isNotEmpty() }.takeLast(3) + .any { REFERENCE_WORD.matches(it.trim('(', ')', '\'', '"', ',', '.', ':')) } + +/** The Gemara-daf markers of [content], in order. */ +internal fun findGemaraDafMarkers(content: String): List = + GEMARA_DAF_MARKER.findAll(content).mapNotNull { match -> + val (extraParen, daf, side, amud) = match.destructured + if (extraParen.isNotEmpty() && followsReferenceWord(content.substring(0, match.range.first))) return@mapNotNull null + val value = gematriaValue(daf) ?: return@mapNotNull null + GemaraAmud(daf, value, side.firstOrNull() ?: if (amud == "א") '.' else ':') + }.toList() + +internal data class GemaraDafPlan( + val entries: List, + val ownerPathByLineIndex: Map, + /** Markers past [planGemaraDafEntries]'s lastDaf, with their line index. */ + val dropped: List> = emptyList(), +) + +/** + * Plans the structure; markers whose daf is past [lastDaf] (the tractate's end) cite + * another tractate and are dropped. + */ +internal fun planGemaraDafEntries(lines: List, toc: List, lastDaf: Int? = null): GemaraDafPlan { + val tocById = toc.associateBy { it.id } + val soleRoot = toc.filter { it.parentId == null }.singleOrNull()?.id + // Several markers of one amud on one line are one occurrence. + val occurrences = LinkedHashMap>>() + val lastKeyByLineIndex = HashMap() + val dropped = mutableListOf>() + for (line in lines) { + val (markers, beyond) = findGemaraDafMarkers(line.content).partition { lastDaf == null || it.value <= lastDaf } + beyond.mapTo(dropped) { it to line.index } + for (amud in markers) { + val list = occurrences.getOrPut(amud.key) { mutableListOf() } + if (list.lastOrNull()?.second?.index != line.index) list += amud to line + } + markers.lastOrNull()?.let { lastKeyByLineIndex[line.index] = it.key } + } + + val entries = mutableListOf() + val pathByOccurrence = HashMap, String>() + occurrences.entries.sortedBy { it.key }.forEachIndexed { i, (key, list) -> + val path = altTocChildPath("", i + 1) + // One child per Rif page; lines without a page heading stay separate. + val byPage = list.groupBy { (_, line) -> line.tocEntryId?.takeIf { it != soleRoot && it in tocById } ?: -1L - line.index } + .values.toList() + val (amud, first) = byPage.first().first() + entries += PlannedMarkerEntry(path, null, 0, amud.text, first.id, first.index, isMarker = byPage.size == 1) + if (byPage.size == 1) { + for ((_, line) in byPage.single()) pathByOccurrence[key to line.index] = path + return@forEachIndexed + } + byPage.forEachIndexed { j, group -> + val (own, line) = group.first() + val heading = line.tocEntryId?.takeIf { it != soleRoot }?.let(tocById::get)?.text?.trimHeadingStart()?.trim() + val childPath = altTocChildPath(path, j + 1) + val text = "${own.text} – ${heading?.let { "רי\"ף $it" } ?: "${j + 1}"}" + entries += PlannedMarkerEntry(childPath, path, 1, text, line.id, line.index, isMarker = true) + for ((_, occurrence) in group) pathByOccurrence[key to occurrence.index] = childPath + } + } + val owners = lastKeyByLineIndex.mapValues { (index, key) -> pathByOccurrence.getValue(key to index) } + return GemaraDafPlan(entries, owners, dropped) +} + +/** + * Last daf of the Bavli tractate a `רי"ף X` book follows: the highest `דף` heading of + * the one book titled X that has them; null when there is no single such book. + */ +internal fun bavliLastDaf(conn: Connection, rifTitle: String): Int? { + val tractate = rifTitle.removePrefix(RIF_TITLE_PREFIX).takeIf { it != rifTitle } ?: return null + val lastByBook = conn.prepareStatement( + """ + SELECT t.bookId, tt.text FROM tocEntry t JOIN tocText tt ON tt.id = t.textId + WHERE t.bookId IN (SELECT id FROM book WHERE title = ?) + """.trimIndent(), + ).use { st -> + st.setString(1, tractate) + st.executeQuery().use { rs -> + val last = HashMap() + while (rs.next()) { + val daf = GEMARA_DAF_HEADING.matchEntire(rs.getString(2).trimHeadingStart().trim())?.groupValues?.get(1) + val value = daf?.let(::gematriaValue) ?: continue + last.merge(rs.getLong(1), value, ::maxOf) + } + last + } + } + return lastByBook.values.singleOrNull() +} + +private const val RIF_TITLE_PREFIX = "רי\"ף " + +/** Atomically replaces each rule's structure; returns entry counts by book title. */ +internal fun writeGemaraDafAltTocs( + conn: Connection, + rules: List, + stableIds: AttachedBuildStateIds? = null, +): Map = writeEachBookAtomically(conn, GEMARA_DAF_ALT_TOC_FILE, rules.map { it.bookTitle to it }) { bookId, rule -> + val logger = Logger.withTag("GemaraDafAltToc") + val lastDaf = bavliLastDaf(conn, rule.bookTitle) + if (lastDaf == null) logger.w { "${rule.bookTitle}: no single Bavli tractate found; markers are not limited to its dapim" } + val lines = readMarkerLines(conn, bookId) + val plan = planGemaraDafEntries(lines, readMarkerToc(conn, bookId), lastDaf) + if (plan.dropped.isNotEmpty()) { + logger.i { + "${rule.bookTitle}: dropped ${plan.dropped.size} markers past the tractate's end: " + + plan.dropped.joinToString { (amud, index) -> "${amud.text} (line $index)" } + } + } + check(plan.entries.isNotEmpty()) { "$GEMARA_DAF_ALT_TOC_FILE: no '(דף …)' marker in '${rule.bookTitle}'" } + writeMarkerStructure( + conn, bookId, rule.structureKey, rule.structureHeTitle, plan.entries, lines, stableIds, plan.ownerPathByLineIndex, + ) +} diff --git a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/InlineMarkerAltTocPostProcess.kt b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/InlineMarkerAltTocPostProcess.kt index f23f53a0..929c1166 100644 --- a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/InlineMarkerAltTocPostProcess.kt +++ b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/InlineMarkerAltTocPostProcess.kt @@ -26,9 +26,21 @@ import kotlin.system.exitProcess * Usage: * ./gradlew :sefariasqlite:buildInlineMarkerAltToc -PseforimDb=/path/to/seforim.db */ -fun main(args: Array) { +fun main(args: Array) = + runMarkerAltTocStage(args, "InlineMarkerAltToc", INLINE_MARKER_ALT_TOC_FILE, ::parseInlineMarkerRules) { conn, rules, ids -> + writeInlineMarkerAltTocs(conn, rules, ids) + } + +/** Runs a ForDB-driven alt-TOC stage against the DB and its build state; exits 1 on failure. */ +internal fun runMarkerAltTocStage( + args: Array, + tag: String, + forDbFile: String, + parse: (List) -> List, + write: (Connection, List, AttachedBuildStateIds) -> Map, +) { Logger.setMinSeverity(Severity.Info) - val logger = Logger.withTag("InlineMarkerAltToc") + val logger = Logger.withTag(tag) val dbPath = resolveSeforimDbPath(args) if (!dbPath.exists()) { @@ -37,10 +49,9 @@ fun main(args: Array) { } try { // An archive published before the file existed has no rules. - val rules = downloadOptionalForDbFile(INLINE_MARKER_ALT_TOC_FILE, logger) - ?.let(::parseInlineMarkerRules).orEmpty() + val rules = downloadOptionalForDbFile(forDbFile, logger)?.let(parse).orEmpty() if (rules.isEmpty()) { - logger.i { "No $INLINE_MARKER_ALT_TOC_FILE rules" } + logger.i { "No $forDbFile rules" } return } val buildStatePath = resolveSeifimBuildStatePath(dbPath) @@ -52,12 +63,12 @@ fun main(args: Array) { st.setString(1, buildStatePath.toAbsolutePath().toString()) st.execute() } - val written = writeInlineMarkerAltTocs(conn, rules, AttachedBuildStateIds(conn)) + val written = write(conn, rules, AttachedBuildStateIds(conn)) for ((title, count) in written) logger.i { " $title: $count entries" } } BuildStateVerifier.verifyFreshSnapshot(buildStatePath, dbPath, emptyMap()) } catch (e: Exception) { - logger.e(e) { "Failed to build inline-marker alt-TOCs; aborting" } + logger.e(e) { "Failed to build the $forDbFile alt-TOCs; aborting" } exitProcess(1) } } @@ -66,8 +77,8 @@ internal const val INLINE_MARKER_ALT_TOC_FILE = "inline_marker_alt_toc.csv" private const val INLINE_MARKER_HEADER = "bookName,marker,structureKey,structureHeTitle,groupBy" /** Keys the app gives other meanings (label or heading injection, sidebar order). */ -private val RESERVED_STRUCTURE_KEYS = setOf("Simanim", "Seifim", "Topic", "Parasha", "SimanNames") -private val STRUCTURE_KEY_REGEX = Regex("[A-Za-z][A-Za-z0-9]*") +internal val RESERVED_STRUCTURE_KEYS = setOf("Simanim", "Seifim", "Topic", "Parasha", "SimanNames") +internal val STRUCTURE_KEY_REGEX = Regex("[A-Za-z][A-Za-z0-9]*") private val MARKER_WORD_REGEX = Regex("[א-ת]+") internal data class InlineMarkerRule( @@ -172,19 +183,29 @@ internal fun writeInlineMarkerAltTocs( conn: Connection, rules: List, stableIds: AttachedBuildStateIds? = null, +): Map = writeEachBookAtomically(conn, INLINE_MARKER_ALT_TOC_FILE, rules.map { it.bookTitle to it }) { bookId, rule -> + val lines = readMarkerLines(conn, bookId) + val planned = planInlineMarkerEntries(rule, lines, readMarkerToc(conn, bookId)) + check(planned.any { it.isMarker }) { "$INLINE_MARKER_ALT_TOC_FILE: no '[${rule.marker} …]' marker in '${rule.bookTitle}'" } + writeMarkerStructure(conn, bookId, rule.structureKey, rule.structureHeTitle, planned, lines, stableIds) +} + +/** Runs [write] for each book title in one transaction; returns its counts by title. */ +internal fun writeEachBookAtomically( + conn: Connection, + forDbFile: String, + rulesByTitle: List>, + write: (bookId: Long, rule: R) -> Int, ): Map { - check(conn.autoCommit) { "writeInlineMarkerAltTocs requires an unowned JDBC connection" } + check(conn.autoCommit) { "$forDbFile stage requires an unowned JDBC connection" } conn.autoCommit = false return try { - val written = rules.associate { rule -> + val written = rulesByTitle.associate { (title, rule) -> val bookId = conn.prepareStatement("SELECT id FROM book WHERE title = ?").use { st -> - st.setString(1, rule.bookTitle) + st.setString(1, title) st.executeQuery().use { rs -> if (rs.next()) rs.getLong(1) else null } - } ?: error("$INLINE_MARKER_ALT_TOC_FILE: no book titled '${rule.bookTitle}'") - val lines = readMarkerLines(conn, bookId) - val planned = planInlineMarkerEntries(rule, lines, readMarkerToc(conn, bookId)) - check(planned.any { it.isMarker }) { "$INLINE_MARKER_ALT_TOC_FILE: no '[${rule.marker} …]' marker in '${rule.bookTitle}'" } - rule.bookTitle to writeStructure(conn, bookId, rule, planned, lines, stableIds) + } ?: error("$forDbFile: no book titled '$title'") + title to write(bookId, rule) } conn.commit() written @@ -196,7 +217,7 @@ internal fun writeInlineMarkerAltTocs( } } -private fun readMarkerLines(conn: Connection, bookId: Long): List = +internal fun readMarkerLines(conn: Connection, bookId: Long): List = conn.prepareStatement( """ SELECT l.id, l.lineIndex, l.content, lt.tocEntryId @@ -214,7 +235,7 @@ private fun readMarkerLines(conn: Connection, bookId: Long): List } } -private fun readMarkerToc(conn: Connection, bookId: Long): List = +internal fun readMarkerToc(conn: Connection, bookId: Long): List = conn.prepareStatement( """ SELECT t.id, t.parentId, tt.text, t.lineId, l.lineIndex @@ -235,13 +256,20 @@ private fun readMarkerToc(conn: Connection, bookId: Long): List = } } -private fun writeStructure( +/** + * Replaces the book's structure of [structureKey] with [planned]. Each line belongs to + * the entry of [ownerPathByLineIndex] at or before it. + */ +internal fun writeMarkerStructure( conn: Connection, bookId: Long, - rule: InlineMarkerRule, + structureKey: String, + structureHeTitle: String, planned: List, lines: List, stableIds: AttachedBuildStateIds?, + // One line with several markers maps to its last (the app keeps a clicked earlier one selected). + ownerPathByLineIndex: Map = planned.filter { it.isMarker }.associate { it.lineIndex to it.path }, ): Int { for (sql in listOf( "DELETE FROM line_alt_toc WHERE structureId IN (SELECT id FROM alt_toc_structure WHERE bookId = ? AND key = ?)", @@ -250,20 +278,20 @@ private fun writeStructure( )) { conn.prepareStatement(sql).use { st -> st.setLong(1, bookId) - st.setString(2, rule.structureKey) + st.setString(2, structureKey) st.executeUpdate() } } - val structureId = stableIds?.altTocStructureId(bookId, rule.structureKey) + val structureId = stableIds?.altTocStructureId(bookId, structureKey) ?: (queryMaxId(conn, "alt_toc_structure") + 1) conn.prepareStatement( "INSERT INTO alt_toc_structure (id, bookId, key, title, heTitle) VALUES (?, ?, ?, ?, ?)", ).use { st -> st.setLong(1, structureId) st.setLong(2, bookId) - st.setString(3, rule.structureKey) - st.setString(4, rule.structureKey) - st.setString(5, rule.structureHeTitle) + st.setString(3, structureKey) + st.setString(4, structureKey) + st.setString(5, structureHeTitle) st.executeUpdate() } @@ -291,9 +319,7 @@ private fun writeStructure( st.executeBatch() } - // Each line belongs to the last marker at or before it; one line with several - // markers maps to its last (the app keeps a clicked earlier one selected). - val ownerByLineIndex = planned.filter { it.isMarker }.associate { it.lineIndex to idByPath.getValue(it.path) } + val ownerByLineIndex = ownerPathByLineIndex.mapValues { (_, path) -> idByPath.getValue(path) } conn.prepareStatement("INSERT INTO line_alt_toc (lineId, structureId, altTocEntryId) VALUES (?, ?, ?)").use { st -> var owner: Long? = null for (line in lines) { diff --git a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/ValidateForDbInputs.kt b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/ValidateForDbInputs.kt index 13fa29e0..6fa6ef47 100644 --- a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/ValidateForDbInputs.kt +++ b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/ValidateForDbInputs.kt @@ -155,6 +155,8 @@ fun main(args: Array) { val bookNotices = loadBookNoticeRows(logger) val inlineMarkerRules = downloadOptionalForDbFile(INLINE_MARKER_ALT_TOC_FILE, logger) ?.let(::parseInlineMarkerRules).orEmpty() + val gemaraDafRules = downloadOptionalForDbFile(GEMARA_DAF_ALT_TOC_FILE, logger) + ?.let(::parseGemaraDafRules).orEmpty() DriverManager.getConnection("jdbc:sqlite:$dbPath").use { conn -> conn.autoCommit = false @@ -236,6 +238,9 @@ fun main(args: Array) { for (rule in inlineMarkerRules.filter { it.bookTitle !in titles }) { logger.w { "ForDB $INLINE_MARKER_ALT_TOC_FILE: no book titled '${rule.bookTitle}' in this DB" } } + for (rule in gemaraDafRules.filter { it.bookTitle !in titles }) { + logger.w { "ForDB $GEMARA_DAF_ALT_TOC_FILE: no book titled '${rule.bookTitle}' in this DB" } + } } // Fails like the build on a missing list or an unreadable row; unmatched refs only warn. val divineNameOverrides = DriverManager.getConnection("jdbc:sqlite:$tempDb").use { @@ -250,7 +255,7 @@ fun main(args: Array) { "${categoryDescriptionOverrides.size} category-description overrides, " + "${authorCanonicalNames.size} author renames, ${bookNotices.banners.size} banner rows, " + "${bookNotices.protections.size} protection rows, $divineNameOverrides divine-name override rows, " + - "${inlineMarkerRules.size} inline-marker alt-TOC rules. " + + "${inlineMarkerRules.size} inline-marker alt-TOC rules, ${gemaraDafRules.size} Gemara-daf alt-TOC rules. " + "Original DB/buildstate untouched." } } finally { diff --git a/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/GemaraDafAltTocTest.kt b/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/GemaraDafAltTocTest.kt new file mode 100644 index 00000000..a48c43f1 --- /dev/null +++ b/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/GemaraDafAltTocTest.kt @@ -0,0 +1,255 @@ +package io.github.kdroidfilter.seforimlibrary.sefariasqlite + +import app.cash.sqldelight.driver.jdbc.sqlite.JdbcSqliteDriver +import io.github.kdroidfilter.seforimlibrary.core.models.AltTocEntry +import io.github.kdroidfilter.seforimlibrary.core.models.AltTocStructure +import io.github.kdroidfilter.seforimlibrary.core.models.Book +import io.github.kdroidfilter.seforimlibrary.core.models.Category +import io.github.kdroidfilter.seforimlibrary.core.models.Line +import io.github.kdroidfilter.seforimlibrary.core.models.TocEntry +import io.github.kdroidfilter.seforimlibrary.dao.repository.SeforimRepository +import io.github.kdroidfilter.seforimlibrary.db.SeforimDb +import kotlinx.coroutines.runBlocking +import java.nio.file.Files +import java.sql.DriverManager +import kotlin.test.AfterTest +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith + +class GemaraDafAltTocTest { + + private val header = "bookName,structureKey,structureHeTitle" + private val rule = GemaraDafRule("רי\"ף ברכות", "GemaraDapim", "דפי הגמרא") + + @Test + fun `parses rules, names with gershayim need no quoting`() { + assertEquals(listOf(rule), parseGemaraDafRules(listOf("$header", "רי\"ף ברכות,GemaraDapim,דפי הגמרא"))) + assertEquals(emptyList(), parseGemaraDafRules(listOf(header))) + assertFailsWith { parseGemaraDafRules(listOf(header, "רי\"ף ברכות,GemaraDapim")) } + assertFailsWith { parseGemaraDafRules(listOf(header, "רי\"ף ברכות,Topic,דפי הגמרא")) } + assertFailsWith { parseGemaraDafRules(listOf(header, "רי\"ף ברכות,דפים,דפי הגמרא")) } + assertFailsWith { + parseGemaraDafRules(listOf(header, "רי\"ף ברכות,GemaraDapim,דפי הגמרא", "רי\"ף ברכות,Dapim,דפים")) + } + assertFailsWith { parseGemaraDafRules(listOf("bookName,marker", "רי\"ף ברכות,x")) } + } + + @Test + fun `accepts only a bare daf and amud`() { + fun texts(content: String) = findGemaraDafMarkers(content).map { it.text } + assertEquals(listOf("דף ב.", "דף ב:"), texts("גמ' (דף ב.) תנו רבנן (דף ב:) עוד")) + assertEquals(listOf("דף קעו."), texts("(דף קעו.)")) + assertEquals(listOf("דף יז.", "דף ג:"), texts("(דף יז ע\"א) ו(דף ג ע\"ב)")) + // A stray `(` before a marker is common; reference words only matter there. + assertEquals(listOf("דף כא:"), texts("משתעי אינשי הכי ((דף כא ע\"ב) מי דמי")) + assertEquals(listOf("דף כז."), texts("גמ' ((דף כז.) אמר")) + assertEquals(listOf("דף ב:"), texts("לקמן מפורש (דף ב:) תנו")) + assertEquals(listOf("דף נו:"), texts("כדלקמן א ב ג ((דף נו:)")) + for (reference in listOf( + "(דף ח: ע\"ש)", "(דף לעיל מח:)", "(דף לקמן יח.)", "(דף יד: טו.)", "(דף ו. דף לג.)", + "(דף כא. ובפ\"א דחלה)", "(דף נ. ע\"ש (ג\"א)", "(דף ס)", "(דף קל)", "(ברכות דף יב.)", "(דף ח.]", + "[דף ח ע\"א]", "([דף ח ע\"א])", "(דף יז ע\"א)", "(((דף ב.)", + "דאמרי' לקמן ((דף י\"ט ע\"ב)", "בפרק בני העיר ((דף ל ע\"א)", "וגרסינן (בה) בפירקא קמא ((דף ג ע\"א)", + "ודלעיל א ((דף ג.)", + )) { + assertEquals(emptyList(), texts(reference), reference) + } + } + + @Test + fun `orders amudim by gematria, amud a first`() { + val keys = listOf("(דף קעו.)", "(דף י.)", "(דף ט:)", "(דף ט.)").map { findGemaraDafMarkers(it).single() } + assertEquals(listOf("דף ט.", "דף ט:", "דף י.", "דף קעו."), keys.sortedBy { it.key }.map { it.text }) + } + + /** Title (sole root) › Rif pages; Gemara amud ח: is reached twice from page א. and once from ג:. */ + private val toc = listOf( + MarkerTocRow(1, null, "רי\"ף ברכות", 100, 0), + MarkerTocRow(2, 1, "דף א.", 101, 1), + MarkerTocRow(3, 1, "דף ג:", 105, 5), + ) + private val lines = listOf( + MarkerLineRow(100, 0, "

רי\"ף ברכות

", 1), + MarkerLineRow(101, 1, "

דף א.

", 2), + MarkerLineRow(102, 2, "(דף ב.) מאימתי (דף ח:) קורין (דף ח:) עוד", 2), + MarkerLineRow(103, 3, "(דף ח:) המשך באותו דף רי\"ף", 2), + MarkerLineRow(104, 4, "(דף ט.) ו(דף ב:)", 2), + MarkerLineRow(105, 5, "

דף ג:

", 3), + MarkerLineRow(106, 6, "גמ' (דף ח:) תנו רבנן", 3), + ) + + @Test + fun `an amud reached from two Rif pages gets one child per page`() { + val plan = planGemaraDafEntries(lines, toc) + assertEquals( + listOf( + PlannedMarkerEntry("1", null, 0, "דף ב.", 102, 2, isMarker = true), + PlannedMarkerEntry("2", null, 0, "דף ב:", 104, 4, isMarker = true), + PlannedMarkerEntry("3", null, 0, "דף ח:", 102, 2, isMarker = false), + PlannedMarkerEntry("3/1", "3", 1, "דף ח: – רי\"ף דף א.", 102, 2, isMarker = true), + PlannedMarkerEntry("3/2", "3", 1, "דף ח: – רי\"ף דף ג:", 106, 6, isMarker = true), + PlannedMarkerEntry("4", null, 0, "דף ט.", 104, 4, isMarker = true), + ), + plan.entries, + ) + assertEquals(mapOf(2 to "3/1", 3 to "3/1", 4 to "2", 6 to "3/2"), plan.ownerPathByLineIndex) + } + + @Test + fun `an amud reached twice from one Rif page is a single leaf`() { + val plan = planGemaraDafEntries(lines.filter { it.index != 6 }, toc) + assertEquals(PlannedMarkerEntry("3", null, 0, "דף ח:", 102, 2, isMarker = true), plan.entries.single { it.text == "דף ח:" }) + assertEquals(mapOf(2 to "3", 3 to "3", 4 to "2"), plan.ownerPathByLineIndex) + } + + @Test + fun `each entry is spelled like the marker it points to`() { + val spelled = listOf( + MarkerLineRow(102, 2, "(דף יט:) א", 2), + MarkerLineRow(106, 6, "(דף י\"ט ע\"ב) ב", 3), + ) + val entries = planGemaraDafEntries(spelled, toc).entries + assertEquals(listOf("דף יט:", "דף יט: – רי\"ף דף א.", "דף י\"ט: – רי\"ף דף ג:"), entries.map { it.text }) + for (entry in entries) { + val marker = entry.text.substringBefore(" – ") + val content = spelled.single { it.index == entry.lineIndex }.content + val amudForm = "(${marker.dropLast(1)} ע\"${if (marker.last() == '.') 'א' else 'ב'})" + assert("($marker)" in content || amudForm in content) { "${entry.text} not found in '$content'" } + } + } + + @Test + fun `markers past the tractate's last daf are dropped`() { + val plan = planGemaraDafEntries(lines, toc, lastDaf = 8) + assertEquals(listOf("דף ב.", "דף ב:", "דף ח:"), plan.entries.filter { it.parentPath == null }.map { it.text }) + assertEquals(listOf("דף ט." to 4), plan.dropped.map { (amud, index) -> amud.text to index }) + assertEquals("2", plan.ownerPathByLineIndex[4]) + } + + @Test + fun `an occurrence without a Rif page heading is numbered`() { + val flat = lines.map { it.copy(tocEntryId = 1) } + val texts = planGemaraDafEntries(flat, toc).entries.map { it.text } + assertEquals(listOf("דף ח: – 1", "דף ח: – 2", "דף ח: – 3"), texts.filter { it.startsWith("דף ח: –") }) + } + + private val dbFile = Files.createTempFile("gemara-daf-alt-toc", ".db") + private val driver = JdbcSqliteDriver(url = "jdbc:sqlite:$dbFile") + private lateinit var repo: SeforimRepository + + @AfterTest + fun tearDown() { + if (::repo.isInitialized) repo.close() + Files.deleteIfExists(dbFile) + } + + @Test + fun `replaces a structure of the same key and maps every line to its occurrence`() = runBlocking { + SeforimDb.Schema.create(driver) + repo = SeforimRepository(dbFile.toString(), driver) + val catId = repo.insertCategory(Category(0, null, "ראשונים", level = 0, order = 1)) + val bookId = repo.insertBook(Book(categoryId = catId, sourceId = repo.insertSource("Sefaria"), title = rule.bookTitle)) + val lineIds = lines.map { repo.insertLine(Line(bookId = bookId, lineIndex = it.index, content = it.content)) } + for (row in toc) { + repo.insertTocEntry( + TocEntry(id = row.id, bookId = bookId, parentId = row.parentId, text = row.text, level = 0, lineId = lineIds[row.lineIndex!!]), + ) + } + for (line in lines) repo.upsertLineToc(lineIds[line.index], line.tocEntryId!!) + val old = repo.upsertAltTocStructure(AltTocStructure(bookId = bookId, key = "GemaraDapim", title = "x", heTitle = "x")) + repo.insertAltTocEntry(AltTocEntry(structureId = old, text = "ישן", level = 0, lineId = lineIds[2])) + + DriverManager.getConnection("jdbc:sqlite:$dbFile").use { conn -> + assertEquals(mapOf(rule.bookTitle to 6), writeGemaraDafAltTocs(conn, listOf(rule))) + val entries = conn.createStatement().use { st -> + st.executeQuery( + """ + SELECT tt.text, e.level, l.lineIndex, e.hasChildren, e.isLastChild + FROM alt_toc_entry e JOIN tocText tt ON tt.id = e.textId JOIN line l ON l.id = e.lineId + ORDER BY e.id + """.trimIndent(), + ).use { rs -> buildList { while (rs.next()) add(listOf(rs.getString(1), rs.getInt(2), rs.getInt(3), rs.getInt(4), rs.getInt(5))) } } + } + assertEquals( + listOf( + listOf("דף ב.", 0, 2, 0, 0), + listOf("דף ב:", 0, 4, 0, 0), + listOf("דף ח:", 0, 2, 1, 0), + listOf("דף ח: – רי\"ף דף א.", 1, 2, 0, 0), + listOf("דף ח: – רי\"ף דף ג:", 1, 6, 0, 1), + listOf("דף ט.", 0, 4, 0, 1), + ), + entries, + ) + val owners = conn.createStatement().use { st -> + st.executeQuery( + """ + SELECT l.lineIndex, tt.text FROM line_alt_toc m + JOIN line l ON l.id = m.lineId JOIN alt_toc_entry e ON e.id = m.altTocEntryId JOIN tocText tt ON tt.id = e.textId + ORDER BY l.lineIndex + """.trimIndent(), + ).use { rs -> buildList { while (rs.next()) add(rs.getInt(1) to rs.getString(2)) } } + } + assertEquals( + listOf( + 2 to "דף ח: – רי\"ף דף א.", 3 to "דף ח: – רי\"ף דף א.", 4 to "דף ב:", 5 to "דף ב:", + 6 to "דף ח: – רי\"ף דף ג:", + ), + owners, + ) + val heTitle = conn.createStatement().use { st -> + st.executeQuery("SELECT group_concat(heTitle) FROM alt_toc_structure").use { rs -> rs.next(); rs.getString(1) } + } + assertEquals("דפי הגמרא", heTitle) + } + } + + @Test + fun `the last daf comes from the one Bavli tractate of the same name`() = runBlocking { + SeforimDb.Schema.create(driver) + repo = SeforimRepository(dbFile.toString(), driver) + val catId = repo.insertCategory(Category(0, null, "תלמוד בבלי", level = 0, order = 1)) + val source = repo.insertSource("Sefaria") + suspend fun addBook(title: String, headings: List) { + val bookId = repo.insertBook(Book(categoryId = catId, sourceId = source, title = title)) + headings.forEachIndexed { i, text -> + val lineId = repo.insertLine(Line(bookId = bookId, lineIndex = i, content = text)) + repo.insertTocEntry(TocEntry(bookId = bookId, text = text, level = 1, lineId = lineId)) + } + } + addBook("ברכות", listOf("ברכות", "דף ב.", "דף סד.", "דף י:")) + addBook("רי\"ף ברכות", listOf("דף קה:")) + DriverManager.getConnection("jdbc:sqlite:$dbFile").use { conn -> + assertEquals(64, bavliLastDaf(conn, "רי\"ף ברכות")) + assertEquals(null, bavliLastDaf(conn, "רי\"ף שבת")) + assertEquals(null, bavliLastDaf(conn, "ברכות")) + // A second book of that title (the repository dedupes titles, so copy the rows). + conn.createStatement().use { st -> + for (sql in listOf( + "CREATE TEMP TABLE b AS SELECT * FROM book WHERE title = 'ברכות'", + "UPDATE b SET id = 999", + "INSERT INTO book SELECT * FROM b", + "CREATE TEMP TABLE t AS SELECT * FROM tocEntry WHERE bookId = (SELECT id FROM book WHERE title = 'ברכות' AND id <> 999)", + "UPDATE t SET id = id + 1000, bookId = 999, parentId = NULL", + "INSERT INTO tocEntry SELECT * FROM t", + )) st.execute(sql) + } + assertEquals(null, bavliLastDaf(conn, "רי\"ף ברכות")) + } + } + + @Test + fun `a listed book without markers fails`() = runBlocking { + SeforimDb.Schema.create(driver) + repo = SeforimRepository(dbFile.toString(), driver) + val catId = repo.insertCategory(Category(0, null, "ראשונים", level = 0, order = 1)) + val bookId = repo.insertBook(Book(categoryId = catId, sourceId = repo.insertSource("Sefaria"), title = rule.bookTitle)) + repo.insertLine(Line(bookId = bookId, lineIndex = 0, content = "(דף ח: ע\"ש)")) + DriverManager.getConnection("jdbc:sqlite:$dbFile").use { conn -> + assertFailsWith { writeGemaraDafAltTocs(conn, listOf(rule)) } + assertFailsWith { writeGemaraDafAltTocs(conn, listOf(rule.copy(bookTitle = "אין"))) } + } + Unit + } +}