From 073c446d7ab7d9c09dd0f124d432058624489cdd Mon Sep 17 00:00:00 2001 From: palmoni5 Date: Thu, 19 Nov 2026 04:00:17 +0200 Subject: [PATCH 1/4] =?UTF-8?q?=D7=AA=D7=95=D7=9B=D7=9F=20=D7=A2=D7=A0?= =?UTF-8?q?=D7=99=D7=99=D7=A0=D7=99=D7=9D=20"=D7=93=D7=A4=D7=99=20=D7=94?= =?UTF-8?q?=D7=92=D7=9E=D7=A8=D7=90"=20=D7=9C=D7=A8=D7=99"=D7=A3,=20=D7=9C?= =?UTF-8?q?=D7=A4=D7=99=20ForDB=20gemara=5Fdaf=5Falt=5Ftoc.csv?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit שלב חדש buildGemaraDafAltToc, שרץ מיד אחרי buildInlineMarkerAltToc, בונה לספרים הרשומים בקובץ מבנה לפי סמני דף הגמרא שבתוך הטקסט: (דף ח:), (דף יז ע"א). הפניות עם מילים נוספות בסוגריים, כמו (דף ח: ע"ש) או (דף לעיל מח:), אינן נספרות. הערכים ממוינים לפי סדר הגמרא, ערך לכל עמוד. עמוד שהרי"ף מגיע אליו יותר מפעם אחת מקבל ילד לכל הופעה לפי סדר הקריאה, בשם דף הרי"ף שבו הוא מופיע. כל שורה משויכת להופעה של הסמן האחרון שלפניה, כדי שהסרגל יסמן את עמוד הגמרא הנוכחי בזמן הגלילה. כתיבת המבנה והרצת השלב משותפות עכשיו לשני השלבים. ValidateForDbInputs מפענח גם את הקובץ החדש ומזהיר על ספר שחסר ב-DB. --- build.gradle.kts | 8 + generator/sefariasqlite/build.gradle.kts | 29 +++ .../GemaraDafAltTocPostProcess.kt | 121 ++++++++++++ .../InlineMarkerAltTocPostProcess.kt | 84 +++++--- .../sefariasqlite/ValidateForDbInputs.kt | 7 +- .../sefariasqlite/GemaraDafAltTocTest.kt | 183 ++++++++++++++++++ 6 files changed, 402 insertions(+), 30 deletions(-) create mode 100644 generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/GemaraDafAltTocPostProcess.kt create mode 100644 generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/GemaraDafAltTocTest.kt diff --git a/build.gradle.kts b/build.gradle.kts index bfebff0a..b585548f 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -63,6 +63,7 @@ tasks.register("generateSeforimDb") { dependsOn(":sefariasqlite:synthesizeSimanNamesAltToc") dependsOn(":sefariasqlite:inheritChaptersAltToc") dependsOn(":sefariasqlite:buildInlineMarkerAltToc") + dependsOn(":sefariasqlite:buildGemaraDafAltToc") dependsOn(":packaging:writeReleaseInfo") dependsOn(":packaging:downloadLexicalDb") // Stamps schema_meta.db_version into the produced seforim.db so the @@ -89,6 +90,7 @@ val lineContentWriters = listOf( ":sefariasqlite:synthesizeSimanNamesAltToc", ":sefariasqlite:inheritChaptersAltToc", ":sefariasqlite:buildInlineMarkerAltToc", + ":sefariasqlite:buildGemaraDafAltToc", ":generator-common:buildLineRefIndex", ":generator-common:buildLineDhIndex", ":sefariasqlite:buildLineDivineNameIndex", @@ -205,6 +207,7 @@ project(":generator-common").tasks.matching { it.name == "buildLineRefIndex" }.c mustRunAfter(":sefariasqlite:synthesizeSimanNamesAltToc") mustRunAfter(":sefariasqlite:inheritChaptersAltToc") mustRunAfter(":sefariasqlite:buildInlineMarkerAltToc") + mustRunAfter(":sefariasqlite:buildGemaraDafAltToc") } // line_dh is derived from line.content, so it must be rebuilt after every @@ -237,6 +240,7 @@ project(":sefariasqlite").tasks.matching { it.name == "buildLineDivineNameIndex" mustRunAfter(":sefariasqlite:synthesizeSimanNamesAltToc") mustRunAfter(":sefariasqlite:inheritChaptersAltToc") mustRunAfter(":sefariasqlite:buildInlineMarkerAltToc") + mustRunAfter(":sefariasqlite:buildGemaraDafAltToc") mustRunAfter(":generator-common:buildLineRefIndex") mustRunAfter(":generator-common:buildLineDhIndex") } @@ -337,6 +341,9 @@ project(":sefariasqlite").tasks.matching { it.name == "buildInlineMarkerAltToc" mustRunAfter(":sefariasqlite:synthesizeSimanNamesAltToc") mustRunAfter(":sefariasqlite:inheritChaptersAltToc") } +project(":sefariasqlite").tasks.matching { it.name == "buildGemaraDafAltToc" }.configureEach { + mustRunAfter(":sefariasqlite:buildInlineMarkerAltToc") +} project(":catalog").tasks.matching { it.name == "buildCatalog" }.configureEach { mustRunAfter(":generator-common:buildLineNoteIndex") mustRunAfter(":otzariasqlite:generateHavroutaLinks") @@ -347,6 +354,7 @@ project(":catalog").tasks.matching { it.name == "buildCatalog" }.configureEach { mustRunAfter(":sefariasqlite:synthesizeSimanNamesAltToc") mustRunAfter(":sefariasqlite:inheritChaptersAltToc") mustRunAfter(":sefariasqlite:buildInlineMarkerAltToc") + mustRunAfter(":sefariasqlite:buildGemaraDafAltToc") mustRunAfter(":generator-common:buildLineDhIndex") mustRunAfter(":sefariasqlite:buildLineDivineNameIndex") } diff --git a/generator/sefariasqlite/build.gradle.kts b/generator/sefariasqlite/build.gradle.kts index 078c2982..510ce9a1 100644 --- a/generator/sefariasqlite/build.gradle.kts +++ b/generator/sefariasqlite/build.gradle.kts @@ -480,6 +480,34 @@ tasks.register("buildInlineMarkerAltToc") { jvmArgs = listOf("-Xmx1g") } +// Alt-TOC by the Gemara daf cited inline (`(דף ח:)`), for the books in ForDB +// gemara_daf_alt_toc.csv. Runs right after buildInlineMarkerAltToc. +// Usage: +// ./gradlew :sefariasqlite:buildGemaraDafAltToc -PseforimDb=/path/to/seforim.db +tasks.register("buildGemaraDafAltToc") { + group = "application" + description = "Build Gemara-daf alt-TOCs for the books in ForDB gemara_daf_alt_toc.csv." + + dependsOn("jvmJar") + mainClass.set("io.github.kdroidfilter.seforimlibrary.sefariasqlite.GemaraDafAltTocPostProcessKt") + mustRunAfter(tasks.matching { it.name == "buildInlineMarkerAltToc" }) + classpath = files(tasks.named("jvmJar")) + configurations.getByName("jvmRuntimeClasspath") + + if (project.hasProperty("seforimDb")) { + systemProperty("seforimDb", project.property("seforimDb") as String) + } else if (System.getenv("SEFORIM_DB") != null) { + systemProperty("seforimDb", System.getenv("SEFORIM_DB")) + } else { + val defaultDbPath = rootProject.layout.buildDirectory.file("seforim.db").get().asFile.absolutePath + systemProperty("seforimDb", defaultDbPath) + } + if (project.hasProperty("buildStatePath")) { + systemProperty("buildStatePath", project.property("buildStatePath") as String) + } + + jvmArgs = listOf("-Xmx1g") +} + // Each ForDB consumer runs as its own JVM. When the release build pins the // ForDB archive (-PforDbArchive + -PforDbSha256), forward both properties to all // of them so they read byte-identical inputs instead of resolving the repository @@ -498,6 +526,7 @@ tasks.matching { "seedAllMetadata", "seedBookNotices", "buildInlineMarkerAltToc", + "buildGemaraDafAltToc", "buildLineDivineNameIndex", "reportLineDivineNames", "validateForDbInputs", diff --git a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/GemaraDafAltTocPostProcess.kt b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/GemaraDafAltTocPostProcess.kt new file mode 100644 index 00000000..e51db484 --- /dev/null +++ b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/GemaraDafAltTocPostProcess.kt @@ -0,0 +1,121 @@ +package io.github.kdroidfilter.seforimlibrary.sefariasqlite + +import io.github.kdroidfilter.seforimlibrary.common.ids.altTocChildPath +import java.sql.Connection + +/** + * Alt-TOC by the Gemara daf a commentary cites inline — `(דף ח:)` or `(דף יז ע"א)` in the + * middle of a Rif paragraph — for the books listed in ForDB [GEMARA_DAF_ALT_TOC_FILE]. + * + * Entries follow the Gemara's order, one per amud; an amud the text reaches more than + * once gets a child per occurrence (in reading order), named by the main-TOC heading + * above it. Each line belongs to the occurrence of the last marker at or before it. + * Like [INLINE_MARKER_ALT_TOC_FILE], the stage owns the rule's structure key for the + * listed books, and ids come from the build-state allocator keyed on ancestor paths. + * + * Usage: + * ./gradlew :sefariasqlite:buildGemaraDafAltToc -PseforimDb=/path/to/seforim.db + */ +fun main(args: Array) = + runMarkerAltTocStage(args, "GemaraDafAltToc", GEMARA_DAF_ALT_TOC_FILE, ::parseGemaraDafRules) { conn, rules, ids -> + writeGemaraDafAltTocs(conn, rules, ids) + } + +internal const val GEMARA_DAF_ALT_TOC_FILE = "gemara_daf_alt_toc.csv" +private const val GEMARA_DAF_HEADER = "bookName,structureKey,structureHeTitle" + +internal data class GemaraDafRule(val bookTitle: String, val structureKey: String, val structureHeTitle: String) + +/** Fails on the first malformed row: the build must not silently drop a rule. */ +internal fun parseGemaraDafRules(lines: List): List { + val rows = forDbText(lines).lines().map { it.trimEnd('\r') }.filter { it.isNotBlank() } + if (rows.isEmpty()) return emptyList() + require(rows.first().trim() == GEMARA_DAF_HEADER) { + "$GEMARA_DAF_ALT_TOC_FILE: header must be '$GEMARA_DAF_HEADER'" + } + val rules = rows.drop(1).mapIndexed { i, row -> + val cells = row.split(',').map { it.trim() } + val where = "$GEMARA_DAF_ALT_TOC_FILE row ${i + 2}" + require(cells.size == 3) { "$where: expected 3 cells, got ${cells.size}" } + val (book, key, heTitle) = cells + require(book.isNotEmpty() && heTitle.isNotEmpty()) { "$where: bookName and structureHeTitle are required" } + require(STRUCTURE_KEY_REGEX.matches(key) && key !in RESERVED_STRUCTURE_KEYS) { + "$where: structureKey '$key' must be an English identifier outside $RESERVED_STRUCTURE_KEYS" + } + GemaraDafRule(book, key, heTitle) + } + val duplicate = rules.groupBy { it.bookTitle }.filterValues { it.size > 1 }.keys + require(duplicate.isEmpty()) { "$GEMARA_DAF_ALT_TOC_FILE: more than one rule for $duplicate" } + return rules +} + +/** One amud; [daf] keeps the letters as written, [value] orders and identifies it. */ +internal data class GemaraAmud(val daf: String, val value: Int, val side: Char) { + val text get() = "דף $daf$side" + val key get() = value * 2 + if (side == '.') 0 else 1 +} + +// Nothing but the daf and amud inside the brackets: `(דף ח: ע"ש)`, `(דף לעיל מח:)` are references. +private val GEMARA_DAF_MARKER = Regex("""([(\[])דף ([א-ת"'׳״]{1,5})(?:([.:])|\s+ע["״]([אב]))([)\]])""") + +/** The Gemara-daf markers of [content], in order. */ +internal fun findGemaraDafMarkers(content: String): List = + GEMARA_DAF_MARKER.findAll(content).mapNotNull { match -> + val (open, daf, side, amud, close) = match.destructured + if ((open == "(") != (close == ")")) return@mapNotNull null + val value = gematriaValue(daf) ?: return@mapNotNull null + GemaraAmud(daf, value, side.firstOrNull() ?: if (amud == "א") '.' else ':') + }.toList() + +internal data class GemaraDafPlan(val entries: List, val ownerPathByLineIndex: Map) + +internal fun planGemaraDafEntries(lines: List, toc: List): GemaraDafPlan { + val tocById = toc.associateBy { it.id } + val soleRoot = toc.filter { it.parentId == null }.singleOrNull()?.id + // Several markers of one amud on one line are one occurrence. + val occurrences = LinkedHashMap>>() + val lastKeyByLineIndex = HashMap() + for (line in lines) { + val markers = findGemaraDafMarkers(line.content) + for (amud in markers) { + val list = occurrences.getOrPut(amud.key) { mutableListOf() } + if (list.lastOrNull()?.second?.index != line.index) list += amud to line + } + markers.lastOrNull()?.let { lastKeyByLineIndex[line.index] = it.key } + } + + val entries = mutableListOf() + val pathByOccurrence = HashMap, String>() + occurrences.entries.sortedBy { it.key }.forEachIndexed { i, (key, list) -> + val path = altTocChildPath("", i + 1) + val (amud, first) = list.first() + entries += PlannedMarkerEntry(path, null, 0, amud.text, first.id, first.index, isMarker = list.size == 1) + if (list.size == 1) { + pathByOccurrence[key to first.index] = path + return@forEachIndexed + } + list.forEachIndexed { j, (_, line) -> + val heading = line.tocEntryId?.takeIf { it != soleRoot }?.let(tocById::get)?.text?.trimHeadingStart()?.trim() + val childPath = altTocChildPath(path, j + 1) + val text = "${amud.text} – ${heading?.let { "רי\"ף $it" } ?: "${j + 1}"}" + entries += PlannedMarkerEntry(childPath, path, 1, text, line.id, line.index, isMarker = true) + pathByOccurrence[key to line.index] = childPath + } + } + val owners = lastKeyByLineIndex.mapValues { (index, key) -> pathByOccurrence.getValue(key to index) } + return GemaraDafPlan(entries, owners) +} + +/** Atomically replaces each rule's structure; returns entry counts by book title. */ +internal fun writeGemaraDafAltTocs( + conn: Connection, + rules: List, + stableIds: AttachedBuildStateIds? = null, +): Map = writeEachBookAtomically(conn, GEMARA_DAF_ALT_TOC_FILE, rules.map { it.bookTitle to it }) { bookId, rule -> + val lines = readMarkerLines(conn, bookId) + val plan = planGemaraDafEntries(lines, readMarkerToc(conn, bookId)) + check(plan.entries.isNotEmpty()) { "$GEMARA_DAF_ALT_TOC_FILE: no '(דף …)' marker in '${rule.bookTitle}'" } + writeMarkerStructure( + conn, bookId, rule.structureKey, rule.structureHeTitle, plan.entries, lines, stableIds, plan.ownerPathByLineIndex, + ) +} diff --git a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/InlineMarkerAltTocPostProcess.kt b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/InlineMarkerAltTocPostProcess.kt index f23f53a0..929c1166 100644 --- a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/InlineMarkerAltTocPostProcess.kt +++ b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/InlineMarkerAltTocPostProcess.kt @@ -26,9 +26,21 @@ import kotlin.system.exitProcess * Usage: * ./gradlew :sefariasqlite:buildInlineMarkerAltToc -PseforimDb=/path/to/seforim.db */ -fun main(args: Array) { +fun main(args: Array) = + runMarkerAltTocStage(args, "InlineMarkerAltToc", INLINE_MARKER_ALT_TOC_FILE, ::parseInlineMarkerRules) { conn, rules, ids -> + writeInlineMarkerAltTocs(conn, rules, ids) + } + +/** Runs a ForDB-driven alt-TOC stage against the DB and its build state; exits 1 on failure. */ +internal fun runMarkerAltTocStage( + args: Array, + tag: String, + forDbFile: String, + parse: (List) -> List, + write: (Connection, List, AttachedBuildStateIds) -> Map, +) { Logger.setMinSeverity(Severity.Info) - val logger = Logger.withTag("InlineMarkerAltToc") + val logger = Logger.withTag(tag) val dbPath = resolveSeforimDbPath(args) if (!dbPath.exists()) { @@ -37,10 +49,9 @@ fun main(args: Array) { } try { // An archive published before the file existed has no rules. - val rules = downloadOptionalForDbFile(INLINE_MARKER_ALT_TOC_FILE, logger) - ?.let(::parseInlineMarkerRules).orEmpty() + val rules = downloadOptionalForDbFile(forDbFile, logger)?.let(parse).orEmpty() if (rules.isEmpty()) { - logger.i { "No $INLINE_MARKER_ALT_TOC_FILE rules" } + logger.i { "No $forDbFile rules" } return } val buildStatePath = resolveSeifimBuildStatePath(dbPath) @@ -52,12 +63,12 @@ fun main(args: Array) { st.setString(1, buildStatePath.toAbsolutePath().toString()) st.execute() } - val written = writeInlineMarkerAltTocs(conn, rules, AttachedBuildStateIds(conn)) + val written = write(conn, rules, AttachedBuildStateIds(conn)) for ((title, count) in written) logger.i { " $title: $count entries" } } BuildStateVerifier.verifyFreshSnapshot(buildStatePath, dbPath, emptyMap()) } catch (e: Exception) { - logger.e(e) { "Failed to build inline-marker alt-TOCs; aborting" } + logger.e(e) { "Failed to build the $forDbFile alt-TOCs; aborting" } exitProcess(1) } } @@ -66,8 +77,8 @@ internal const val INLINE_MARKER_ALT_TOC_FILE = "inline_marker_alt_toc.csv" private const val INLINE_MARKER_HEADER = "bookName,marker,structureKey,structureHeTitle,groupBy" /** Keys the app gives other meanings (label or heading injection, sidebar order). */ -private val RESERVED_STRUCTURE_KEYS = setOf("Simanim", "Seifim", "Topic", "Parasha", "SimanNames") -private val STRUCTURE_KEY_REGEX = Regex("[A-Za-z][A-Za-z0-9]*") +internal val RESERVED_STRUCTURE_KEYS = setOf("Simanim", "Seifim", "Topic", "Parasha", "SimanNames") +internal val STRUCTURE_KEY_REGEX = Regex("[A-Za-z][A-Za-z0-9]*") private val MARKER_WORD_REGEX = Regex("[א-ת]+") internal data class InlineMarkerRule( @@ -172,19 +183,29 @@ internal fun writeInlineMarkerAltTocs( conn: Connection, rules: List, stableIds: AttachedBuildStateIds? = null, +): Map = writeEachBookAtomically(conn, INLINE_MARKER_ALT_TOC_FILE, rules.map { it.bookTitle to it }) { bookId, rule -> + val lines = readMarkerLines(conn, bookId) + val planned = planInlineMarkerEntries(rule, lines, readMarkerToc(conn, bookId)) + check(planned.any { it.isMarker }) { "$INLINE_MARKER_ALT_TOC_FILE: no '[${rule.marker} …]' marker in '${rule.bookTitle}'" } + writeMarkerStructure(conn, bookId, rule.structureKey, rule.structureHeTitle, planned, lines, stableIds) +} + +/** Runs [write] for each book title in one transaction; returns its counts by title. */ +internal fun writeEachBookAtomically( + conn: Connection, + forDbFile: String, + rulesByTitle: List>, + write: (bookId: Long, rule: R) -> Int, ): Map { - check(conn.autoCommit) { "writeInlineMarkerAltTocs requires an unowned JDBC connection" } + check(conn.autoCommit) { "$forDbFile stage requires an unowned JDBC connection" } conn.autoCommit = false return try { - val written = rules.associate { rule -> + val written = rulesByTitle.associate { (title, rule) -> val bookId = conn.prepareStatement("SELECT id FROM book WHERE title = ?").use { st -> - st.setString(1, rule.bookTitle) + st.setString(1, title) st.executeQuery().use { rs -> if (rs.next()) rs.getLong(1) else null } - } ?: error("$INLINE_MARKER_ALT_TOC_FILE: no book titled '${rule.bookTitle}'") - val lines = readMarkerLines(conn, bookId) - val planned = planInlineMarkerEntries(rule, lines, readMarkerToc(conn, bookId)) - check(planned.any { it.isMarker }) { "$INLINE_MARKER_ALT_TOC_FILE: no '[${rule.marker} …]' marker in '${rule.bookTitle}'" } - rule.bookTitle to writeStructure(conn, bookId, rule, planned, lines, stableIds) + } ?: error("$forDbFile: no book titled '$title'") + title to write(bookId, rule) } conn.commit() written @@ -196,7 +217,7 @@ internal fun writeInlineMarkerAltTocs( } } -private fun readMarkerLines(conn: Connection, bookId: Long): List = +internal fun readMarkerLines(conn: Connection, bookId: Long): List = conn.prepareStatement( """ SELECT l.id, l.lineIndex, l.content, lt.tocEntryId @@ -214,7 +235,7 @@ private fun readMarkerLines(conn: Connection, bookId: Long): List } } -private fun readMarkerToc(conn: Connection, bookId: Long): List = +internal fun readMarkerToc(conn: Connection, bookId: Long): List = conn.prepareStatement( """ SELECT t.id, t.parentId, tt.text, t.lineId, l.lineIndex @@ -235,13 +256,20 @@ private fun readMarkerToc(conn: Connection, bookId: Long): List = } } -private fun writeStructure( +/** + * Replaces the book's structure of [structureKey] with [planned]. Each line belongs to + * the entry of [ownerPathByLineIndex] at or before it. + */ +internal fun writeMarkerStructure( conn: Connection, bookId: Long, - rule: InlineMarkerRule, + structureKey: String, + structureHeTitle: String, planned: List, lines: List, stableIds: AttachedBuildStateIds?, + // One line with several markers maps to its last (the app keeps a clicked earlier one selected). + ownerPathByLineIndex: Map = planned.filter { it.isMarker }.associate { it.lineIndex to it.path }, ): Int { for (sql in listOf( "DELETE FROM line_alt_toc WHERE structureId IN (SELECT id FROM alt_toc_structure WHERE bookId = ? AND key = ?)", @@ -250,20 +278,20 @@ private fun writeStructure( )) { conn.prepareStatement(sql).use { st -> st.setLong(1, bookId) - st.setString(2, rule.structureKey) + st.setString(2, structureKey) st.executeUpdate() } } - val structureId = stableIds?.altTocStructureId(bookId, rule.structureKey) + val structureId = stableIds?.altTocStructureId(bookId, structureKey) ?: (queryMaxId(conn, "alt_toc_structure") + 1) conn.prepareStatement( "INSERT INTO alt_toc_structure (id, bookId, key, title, heTitle) VALUES (?, ?, ?, ?, ?)", ).use { st -> st.setLong(1, structureId) st.setLong(2, bookId) - st.setString(3, rule.structureKey) - st.setString(4, rule.structureKey) - st.setString(5, rule.structureHeTitle) + st.setString(3, structureKey) + st.setString(4, structureKey) + st.setString(5, structureHeTitle) st.executeUpdate() } @@ -291,9 +319,7 @@ private fun writeStructure( st.executeBatch() } - // Each line belongs to the last marker at or before it; one line with several - // markers maps to its last (the app keeps a clicked earlier one selected). - val ownerByLineIndex = planned.filter { it.isMarker }.associate { it.lineIndex to idByPath.getValue(it.path) } + val ownerByLineIndex = ownerPathByLineIndex.mapValues { (_, path) -> idByPath.getValue(path) } conn.prepareStatement("INSERT INTO line_alt_toc (lineId, structureId, altTocEntryId) VALUES (?, ?, ?)").use { st -> var owner: Long? = null for (line in lines) { diff --git a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/ValidateForDbInputs.kt b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/ValidateForDbInputs.kt index 13fa29e0..6fa6ef47 100644 --- a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/ValidateForDbInputs.kt +++ b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/ValidateForDbInputs.kt @@ -155,6 +155,8 @@ fun main(args: Array) { val bookNotices = loadBookNoticeRows(logger) val inlineMarkerRules = downloadOptionalForDbFile(INLINE_MARKER_ALT_TOC_FILE, logger) ?.let(::parseInlineMarkerRules).orEmpty() + val gemaraDafRules = downloadOptionalForDbFile(GEMARA_DAF_ALT_TOC_FILE, logger) + ?.let(::parseGemaraDafRules).orEmpty() DriverManager.getConnection("jdbc:sqlite:$dbPath").use { conn -> conn.autoCommit = false @@ -236,6 +238,9 @@ fun main(args: Array) { for (rule in inlineMarkerRules.filter { it.bookTitle !in titles }) { logger.w { "ForDB $INLINE_MARKER_ALT_TOC_FILE: no book titled '${rule.bookTitle}' in this DB" } } + for (rule in gemaraDafRules.filter { it.bookTitle !in titles }) { + logger.w { "ForDB $GEMARA_DAF_ALT_TOC_FILE: no book titled '${rule.bookTitle}' in this DB" } + } } // Fails like the build on a missing list or an unreadable row; unmatched refs only warn. val divineNameOverrides = DriverManager.getConnection("jdbc:sqlite:$tempDb").use { @@ -250,7 +255,7 @@ fun main(args: Array) { "${categoryDescriptionOverrides.size} category-description overrides, " + "${authorCanonicalNames.size} author renames, ${bookNotices.banners.size} banner rows, " + "${bookNotices.protections.size} protection rows, $divineNameOverrides divine-name override rows, " + - "${inlineMarkerRules.size} inline-marker alt-TOC rules. " + + "${inlineMarkerRules.size} inline-marker alt-TOC rules, ${gemaraDafRules.size} Gemara-daf alt-TOC rules. " + "Original DB/buildstate untouched." } } finally { diff --git a/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/GemaraDafAltTocTest.kt b/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/GemaraDafAltTocTest.kt new file mode 100644 index 00000000..912741df --- /dev/null +++ b/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/GemaraDafAltTocTest.kt @@ -0,0 +1,183 @@ +package io.github.kdroidfilter.seforimlibrary.sefariasqlite + +import app.cash.sqldelight.driver.jdbc.sqlite.JdbcSqliteDriver +import io.github.kdroidfilter.seforimlibrary.core.models.AltTocEntry +import io.github.kdroidfilter.seforimlibrary.core.models.AltTocStructure +import io.github.kdroidfilter.seforimlibrary.core.models.Book +import io.github.kdroidfilter.seforimlibrary.core.models.Category +import io.github.kdroidfilter.seforimlibrary.core.models.Line +import io.github.kdroidfilter.seforimlibrary.core.models.TocEntry +import io.github.kdroidfilter.seforimlibrary.dao.repository.SeforimRepository +import io.github.kdroidfilter.seforimlibrary.db.SeforimDb +import kotlinx.coroutines.runBlocking +import java.nio.file.Files +import java.sql.DriverManager +import kotlin.test.AfterTest +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith + +class GemaraDafAltTocTest { + + private val header = "bookName,structureKey,structureHeTitle" + private val rule = GemaraDafRule("רי\"ף ברכות", "GemaraDapim", "דפי הגמרא") + + @Test + fun `parses rules, names with gershayim need no quoting`() { + assertEquals(listOf(rule), parseGemaraDafRules(listOf("$header", "רי\"ף ברכות,GemaraDapim,דפי הגמרא"))) + assertEquals(emptyList(), parseGemaraDafRules(listOf(header))) + assertFailsWith { parseGemaraDafRules(listOf(header, "רי\"ף ברכות,GemaraDapim")) } + assertFailsWith { parseGemaraDafRules(listOf(header, "רי\"ף ברכות,Topic,דפי הגמרא")) } + assertFailsWith { parseGemaraDafRules(listOf(header, "רי\"ף ברכות,דפים,דפי הגמרא")) } + assertFailsWith { + parseGemaraDafRules(listOf(header, "רי\"ף ברכות,GemaraDapim,דפי הגמרא", "רי\"ף ברכות,Dapim,דפים")) + } + assertFailsWith { parseGemaraDafRules(listOf("bookName,marker", "רי\"ף ברכות,x")) } + } + + @Test + fun `accepts only a bare daf and amud`() { + fun texts(content: String) = findGemaraDafMarkers(content).map { it.text } + assertEquals(listOf("דף ב.", "דף ב:"), texts("גמ' (דף ב.) תנו רבנן (דף ב:) עוד")) + assertEquals(listOf("דף קעו."), texts("(דף קעו.)")) + assertEquals(listOf("דף יז.", "דף ג:"), texts("(דף יז ע\"א) ו(דף ג ע\"ב)")) + assertEquals(listOf("דף ח."), texts("[דף ח ע\"א]")) + for (reference in listOf( + "(דף ח: ע\"ש)", "(דף לעיל מח:)", "(דף לקמן יח.)", "(דף יד: טו.)", "(דף ו. דף לג.)", + "(דף כא. ובפ\"א דחלה)", "(דף נ. ע\"ש (ג\"א)", "(דף ס)", "(דף קל)", "(ברכות דף יב.)", "(דף ח.]", + )) { + assertEquals(emptyList(), texts(reference), reference) + } + } + + @Test + fun `orders amudim by gematria, amud a first`() { + val keys = listOf("(דף קעו.)", "(דף י.)", "(דף ט:)", "(דף ט.)").map { findGemaraDafMarkers(it).single() } + assertEquals(listOf("דף ט.", "דף ט:", "דף י.", "דף קעו."), keys.sortedBy { it.key }.map { it.text }) + } + + /** Title (sole root) › Rif pages; Gemara amud ח: is reached twice, once from each Rif page. */ + private val toc = listOf( + MarkerTocRow(1, null, "רי\"ף ברכות", 100, 0), + MarkerTocRow(2, 1, "דף א.", 101, 1), + MarkerTocRow(3, 1, "דף ג:", 105, 5), + ) + private val lines = listOf( + MarkerLineRow(100, 0, "

רי\"ף ברכות

", 1), + MarkerLineRow(101, 1, "

דף א.

", 2), + MarkerLineRow(102, 2, "(דף ב.) מאימתי (דף ח:) קורין (דף ח:) עוד", 2), + MarkerLineRow(103, 3, "המשך", 2), + MarkerLineRow(104, 4, "(דף ט.) ו(דף ב:)", 2), + MarkerLineRow(105, 5, "

דף ג:

", 3), + MarkerLineRow(106, 6, "גמ' (דף ח:) תנו רבנן", 3), + ) + + @Test + fun `an amud reached twice gets a child per occurrence named by its Rif page`() { + val plan = planGemaraDafEntries(lines, toc) + assertEquals( + listOf( + PlannedMarkerEntry("1", null, 0, "דף ב.", 102, 2, isMarker = true), + PlannedMarkerEntry("2", null, 0, "דף ב:", 104, 4, isMarker = true), + PlannedMarkerEntry("3", null, 0, "דף ח:", 102, 2, isMarker = false), + PlannedMarkerEntry("3/1", "3", 1, "דף ח: – רי\"ף דף א.", 102, 2, isMarker = true), + PlannedMarkerEntry("3/2", "3", 1, "דף ח: – רי\"ף דף ג:", 106, 6, isMarker = true), + PlannedMarkerEntry("4", null, 0, "דף ט.", 104, 4, isMarker = true), + ), + plan.entries, + ) + assertEquals(mapOf(2 to "3/1", 4 to "2", 6 to "3/2"), plan.ownerPathByLineIndex) + } + + @Test + fun `an occurrence without a Rif page heading is numbered`() { + val flat = lines.map { it.copy(tocEntryId = 1) } + val texts = planGemaraDafEntries(flat, toc).entries.map { it.text } + assertEquals(listOf("דף ח: – 1", "דף ח: – 2"), texts.filter { it.startsWith("דף ח: –") }) + } + + private val dbFile = Files.createTempFile("gemara-daf-alt-toc", ".db") + private val driver = JdbcSqliteDriver(url = "jdbc:sqlite:$dbFile") + private lateinit var repo: SeforimRepository + + @AfterTest + fun tearDown() { + if (::repo.isInitialized) repo.close() + Files.deleteIfExists(dbFile) + } + + @Test + fun `replaces a structure of the same key and maps every line to its occurrence`() = runBlocking { + SeforimDb.Schema.create(driver) + repo = SeforimRepository(dbFile.toString(), driver) + val catId = repo.insertCategory(Category(0, null, "ראשונים", level = 0, order = 1)) + val bookId = repo.insertBook(Book(categoryId = catId, sourceId = repo.insertSource("Sefaria"), title = rule.bookTitle)) + val lineIds = lines.map { repo.insertLine(Line(bookId = bookId, lineIndex = it.index, content = it.content)) } + for (row in toc) { + repo.insertTocEntry( + TocEntry(id = row.id, bookId = bookId, parentId = row.parentId, text = row.text, level = 0, lineId = lineIds[row.lineIndex!!]), + ) + } + for (line in lines) repo.upsertLineToc(lineIds[line.index], line.tocEntryId!!) + val old = repo.upsertAltTocStructure(AltTocStructure(bookId = bookId, key = "GemaraDapim", title = "x", heTitle = "x")) + repo.insertAltTocEntry(AltTocEntry(structureId = old, text = "ישן", level = 0, lineId = lineIds[2])) + + DriverManager.getConnection("jdbc:sqlite:$dbFile").use { conn -> + assertEquals(mapOf(rule.bookTitle to 6), writeGemaraDafAltTocs(conn, listOf(rule))) + val entries = conn.createStatement().use { st -> + st.executeQuery( + """ + SELECT tt.text, e.level, l.lineIndex, e.hasChildren, e.isLastChild + FROM alt_toc_entry e JOIN tocText tt ON tt.id = e.textId JOIN line l ON l.id = e.lineId + ORDER BY e.id + """.trimIndent(), + ).use { rs -> buildList { while (rs.next()) add(listOf(rs.getString(1), rs.getInt(2), rs.getInt(3), rs.getInt(4), rs.getInt(5))) } } + } + assertEquals( + listOf( + listOf("דף ב.", 0, 2, 0, 0), + listOf("דף ב:", 0, 4, 0, 0), + listOf("דף ח:", 0, 2, 1, 0), + listOf("דף ח: – רי\"ף דף א.", 1, 2, 0, 0), + listOf("דף ח: – רי\"ף דף ג:", 1, 6, 0, 1), + listOf("דף ט.", 0, 4, 0, 1), + ), + entries, + ) + val owners = conn.createStatement().use { st -> + st.executeQuery( + """ + SELECT l.lineIndex, tt.text FROM line_alt_toc m + JOIN line l ON l.id = m.lineId JOIN alt_toc_entry e ON e.id = m.altTocEntryId JOIN tocText tt ON tt.id = e.textId + ORDER BY l.lineIndex + """.trimIndent(), + ).use { rs -> buildList { while (rs.next()) add(rs.getInt(1) to rs.getString(2)) } } + } + assertEquals( + listOf( + 2 to "דף ח: – רי\"ף דף א.", 3 to "דף ח: – רי\"ף דף א.", 4 to "דף ב:", 5 to "דף ב:", + 6 to "דף ח: – רי\"ף דף ג:", + ), + owners, + ) + val heTitle = conn.createStatement().use { st -> + st.executeQuery("SELECT group_concat(heTitle) FROM alt_toc_structure").use { rs -> rs.next(); rs.getString(1) } + } + assertEquals("דפי הגמרא", heTitle) + } + } + + @Test + fun `a listed book without markers fails`() = runBlocking { + SeforimDb.Schema.create(driver) + repo = SeforimRepository(dbFile.toString(), driver) + val catId = repo.insertCategory(Category(0, null, "ראשונים", level = 0, order = 1)) + val bookId = repo.insertBook(Book(categoryId = catId, sourceId = repo.insertSource("Sefaria"), title = rule.bookTitle)) + repo.insertLine(Line(bookId = bookId, lineIndex = 0, content = "(דף ח: ע\"ש)")) + DriverManager.getConnection("jdbc:sqlite:$dbFile").use { conn -> + assertFailsWith { writeGemaraDafAltTocs(conn, listOf(rule)) } + assertFailsWith { writeGemaraDafAltTocs(conn, listOf(rule.copy(bookTitle = "אין"))) } + } + Unit + } +} From 1d0a0ea94d0cc16f98ef196dd094615e4cac08f7 Mon Sep 17 00:00:00 2001 From: palmoni5 Date: Thu, 19 Nov 2026 04:01:55 +0200 Subject: [PATCH 2/4] =?UTF-8?q?=D7=93=D7=A4=D7=99=20=D7=94=D7=92=D7=9E?= =?UTF-8?q?=D7=A8=D7=90:=20=D7=99=D7=9C=D7=93=20=D7=90=D7=97=D7=93=20?= =?UTF-8?q?=D7=9C=D7=9B=D7=9C=20=D7=93=D7=A3=20=D7=A8=D7=99"=D7=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit הופעות של אותו עמוד גמרא תחת אותו דף רי"ף מתאחדות לילד אחד, שמצביע על ההופעה הראשונה, וכל השורות שאחרי ההופעות הבאות משויכות אליו. עמוד שכל הופעותיו תחת דף רי"ף אחד הוא ערך יחיד בלי ילדים. הופעה בלי כותרת דף נשארת נפרדת וממוספרת. --- .../GemaraDafAltTocPostProcess.kt | 23 +++++++++++-------- .../sefariasqlite/GemaraDafAltTocTest.kt | 17 ++++++++++---- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/GemaraDafAltTocPostProcess.kt b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/GemaraDafAltTocPostProcess.kt index e51db484..d7bd185f 100644 --- a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/GemaraDafAltTocPostProcess.kt +++ b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/GemaraDafAltTocPostProcess.kt @@ -7,9 +7,9 @@ import java.sql.Connection * Alt-TOC by the Gemara daf a commentary cites inline — `(דף ח:)` or `(דף יז ע"א)` in the * middle of a Rif paragraph — for the books listed in ForDB [GEMARA_DAF_ALT_TOC_FILE]. * - * Entries follow the Gemara's order, one per amud; an amud the text reaches more than - * once gets a child per occurrence (in reading order), named by the main-TOC heading - * above it. Each line belongs to the occurrence of the last marker at or before it. + * Entries follow the Gemara's order, one per amud; an amud the text reaches from more + * than one main-TOC heading (Rif page) gets a child per heading, in reading order. + * Each line belongs to the entry of the last marker at or before it. * Like [INLINE_MARKER_ALT_TOC_FILE], the stage owns the rule's structure key for the * listed books, and ids come from the build-state allocator keyed on ancestor paths. * @@ -88,18 +88,23 @@ internal fun planGemaraDafEntries(lines: List, toc: List, String>() occurrences.entries.sortedBy { it.key }.forEachIndexed { i, (key, list) -> val path = altTocChildPath("", i + 1) - val (amud, first) = list.first() - entries += PlannedMarkerEntry(path, null, 0, amud.text, first.id, first.index, isMarker = list.size == 1) - if (list.size == 1) { - pathByOccurrence[key to first.index] = path + val amud = list.first().first + // One child per Rif page; lines without a page heading stay separate. + val byPage = list.groupBy { (_, line) -> line.tocEntryId?.takeIf { it != soleRoot && it in tocById } ?: -1L - line.index } + .values.map { group -> group.map { it.second } } + val first = byPage.first().first() + entries += PlannedMarkerEntry(path, null, 0, amud.text, first.id, first.index, isMarker = byPage.size == 1) + if (byPage.size == 1) { + for (line in byPage.single()) pathByOccurrence[key to line.index] = path return@forEachIndexed } - list.forEachIndexed { j, (_, line) -> + byPage.forEachIndexed { j, group -> + val line = group.first() val heading = line.tocEntryId?.takeIf { it != soleRoot }?.let(tocById::get)?.text?.trimHeadingStart()?.trim() val childPath = altTocChildPath(path, j + 1) val text = "${amud.text} – ${heading?.let { "רי\"ף $it" } ?: "${j + 1}"}" entries += PlannedMarkerEntry(childPath, path, 1, text, line.id, line.index, isMarker = true) - pathByOccurrence[key to line.index] = childPath + for (occurrence in group) pathByOccurrence[key to occurrence.index] = childPath } } val owners = lastKeyByLineIndex.mapValues { (index, key) -> pathByOccurrence.getValue(key to index) } diff --git a/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/GemaraDafAltTocTest.kt b/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/GemaraDafAltTocTest.kt index 912741df..158e406e 100644 --- a/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/GemaraDafAltTocTest.kt +++ b/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/GemaraDafAltTocTest.kt @@ -56,7 +56,7 @@ class GemaraDafAltTocTest { assertEquals(listOf("דף ט.", "דף ט:", "דף י.", "דף קעו."), keys.sortedBy { it.key }.map { it.text }) } - /** Title (sole root) › Rif pages; Gemara amud ח: is reached twice, once from each Rif page. */ + /** Title (sole root) › Rif pages; Gemara amud ח: is reached twice from page א. and once from ג:. */ private val toc = listOf( MarkerTocRow(1, null, "רי\"ף ברכות", 100, 0), MarkerTocRow(2, 1, "דף א.", 101, 1), @@ -66,14 +66,14 @@ class GemaraDafAltTocTest { MarkerLineRow(100, 0, "

רי\"ף ברכות

", 1), MarkerLineRow(101, 1, "

דף א.

", 2), MarkerLineRow(102, 2, "(דף ב.) מאימתי (דף ח:) קורין (דף ח:) עוד", 2), - MarkerLineRow(103, 3, "המשך", 2), + MarkerLineRow(103, 3, "(דף ח:) המשך באותו דף רי\"ף", 2), MarkerLineRow(104, 4, "(דף ט.) ו(דף ב:)", 2), MarkerLineRow(105, 5, "

דף ג:

", 3), MarkerLineRow(106, 6, "גמ' (דף ח:) תנו רבנן", 3), ) @Test - fun `an amud reached twice gets a child per occurrence named by its Rif page`() { + fun `an amud reached from two Rif pages gets one child per page`() { val plan = planGemaraDafEntries(lines, toc) assertEquals( listOf( @@ -86,14 +86,21 @@ class GemaraDafAltTocTest { ), plan.entries, ) - assertEquals(mapOf(2 to "3/1", 4 to "2", 6 to "3/2"), plan.ownerPathByLineIndex) + assertEquals(mapOf(2 to "3/1", 3 to "3/1", 4 to "2", 6 to "3/2"), plan.ownerPathByLineIndex) + } + + @Test + fun `an amud reached twice from one Rif page is a single leaf`() { + val plan = planGemaraDafEntries(lines.filter { it.index != 6 }, toc) + assertEquals(PlannedMarkerEntry("3", null, 0, "דף ח:", 102, 2, isMarker = true), plan.entries.single { it.text == "דף ח:" }) + assertEquals(mapOf(2 to "3", 3 to "3", 4 to "2"), plan.ownerPathByLineIndex) } @Test fun `an occurrence without a Rif page heading is numbered`() { val flat = lines.map { it.copy(tocEntryId = 1) } val texts = planGemaraDafEntries(flat, toc).entries.map { it.text } - assertEquals(listOf("דף ח: – 1", "דף ח: – 2"), texts.filter { it.startsWith("דף ח: –") }) + assertEquals(listOf("דף ח: – 1", "דף ח: – 2", "דף ח: – 3"), texts.filter { it.startsWith("דף ח: –") }) } private val dbFile = Files.createTempFile("gemara-daf-alt-toc", ".db") From 7184f71e6e1140408bd23b43ff49e35d3730d95a Mon Sep 17 00:00:00 2001 From: palmoni5 Date: Thu, 19 Nov 2026 04:21:41 +0200 Subject: [PATCH 3/4] =?UTF-8?q?=D7=93=D7=A4=D7=99=20=D7=94=D7=92=D7=9E?= =?UTF-8?q?=D7=A8=D7=90:=20=D7=A8=D7=A7=20=D7=A1=D7=9E=D7=A0=D7=99=D7=9D?= =?UTF-8?q?=20=D7=A9=D7=94=D7=90=D7=A4=D7=9C=D7=99=D7=A7=D7=A6=D7=99=D7=94?= =?UTF-8?q?=20=D7=9E=D7=95=D7=A6=D7=90=D7=AA,=20=D7=95=D7=91=D7=AA=D7=97?= =?UTF-8?q?=D7=95=D7=9D=20=D7=94=D7=9E=D7=A1=D7=9B=D7=AA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit סמן נספר רק בסוגריים עגולים אחד בלבד, עם רווח יחיד לפני ע"א/ע"ב: (( לפני הסמן או סוגריים מרובעים אינם נספרים. שם כל ערך נכתב לפי איות ההופעה שהוא מצביע עליה, כדי שהאפליקציה תמצא את הסמן בשורה. סמן שהדף שלו אחרי הדף האחרון של מסכת הבבלי באותו שם הוא הפניה למסכת אחרת ואינו נספר. הדף האחרון נלקח מכותרות "דף" של הספר היחיד שכותרתו כשם הרי"ף בלי "רי"ף "; אם אין ספר כזה, לא מסננים ורושמים אזהרה. הסמנים שסוננו נרשמים ביומן לכל ספר. --- .../GemaraDafAltTocPostProcess.kt | 82 +++++++++++++++---- .../sefariasqlite/GemaraDafAltTocTest.kt | 60 +++++++++++++- 2 files changed, 125 insertions(+), 17 deletions(-) diff --git a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/GemaraDafAltTocPostProcess.kt b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/GemaraDafAltTocPostProcess.kt index d7bd185f..735f9663 100644 --- a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/GemaraDafAltTocPostProcess.kt +++ b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/GemaraDafAltTocPostProcess.kt @@ -1,5 +1,6 @@ package io.github.kdroidfilter.seforimlibrary.sefariasqlite +import co.touchlab.kermit.Logger import io.github.kdroidfilter.seforimlibrary.common.ids.altTocChildPath import java.sql.Connection @@ -51,32 +52,45 @@ internal fun parseGemaraDafRules(lines: List): List { /** One amud; [daf] keeps the letters as written, [value] orders and identifies it. */ internal data class GemaraAmud(val daf: String, val value: Int, val side: Char) { + /** The app finds the entry's line by searching for `(text)` or its ע"א/ע"ב form. */ val text get() = "דף $daf$side" val key get() = value * 2 + if (side == '.') 0 else 1 } -// Nothing but the daf and amud inside the brackets: `(דף ח: ע"ש)`, `(דף לעיל מח:)` are references. -private val GEMARA_DAF_MARKER = Regex("""([(\[])דף ([א-ת"'׳״]{1,5})(?:([.:])|\s+ע["״]([אב]))([)\]])""") +// Nothing but the daf and amud inside one pair of round brackets: `(דף ח: ע"ש)`, +// `(דף לעיל מח:)` and `((דף יט ע"ב)` are references. +private val GEMARA_DAF_MARKER = Regex("""(? = GEMARA_DAF_MARKER.findAll(content).mapNotNull { match -> - val (open, daf, side, amud, close) = match.destructured - if ((open == "(") != (close == ")")) return@mapNotNull null + val (daf, side, amud) = match.destructured val value = gematriaValue(daf) ?: return@mapNotNull null GemaraAmud(daf, value, side.firstOrNull() ?: if (amud == "א") '.' else ':') }.toList() -internal data class GemaraDafPlan(val entries: List, val ownerPathByLineIndex: Map) +internal data class GemaraDafPlan( + val entries: List, + val ownerPathByLineIndex: Map, + /** Markers past [planGemaraDafEntries]'s lastDaf, with their line index. */ + val dropped: List> = emptyList(), +) -internal fun planGemaraDafEntries(lines: List, toc: List): GemaraDafPlan { +/** + * Plans the structure; markers whose daf is past [lastDaf] (the tractate's end) cite + * another tractate and are dropped. + */ +internal fun planGemaraDafEntries(lines: List, toc: List, lastDaf: Int? = null): GemaraDafPlan { val tocById = toc.associateBy { it.id } val soleRoot = toc.filter { it.parentId == null }.singleOrNull()?.id // Several markers of one amud on one line are one occurrence. val occurrences = LinkedHashMap>>() val lastKeyByLineIndex = HashMap() + val dropped = mutableListOf>() for (line in lines) { - val markers = findGemaraDafMarkers(line.content) + val (markers, beyond) = findGemaraDafMarkers(line.content).partition { lastDaf == null || it.value <= lastDaf } + beyond.mapTo(dropped) { it to line.index } for (amud in markers) { val list = occurrences.getOrPut(amud.key) { mutableListOf() } if (list.lastOrNull()?.second?.index != line.index) list += amud to line @@ -88,37 +102,73 @@ internal fun planGemaraDafEntries(lines: List, toc: List, String>() occurrences.entries.sortedBy { it.key }.forEachIndexed { i, (key, list) -> val path = altTocChildPath("", i + 1) - val amud = list.first().first // One child per Rif page; lines without a page heading stay separate. val byPage = list.groupBy { (_, line) -> line.tocEntryId?.takeIf { it != soleRoot && it in tocById } ?: -1L - line.index } - .values.map { group -> group.map { it.second } } - val first = byPage.first().first() + .values.toList() + val (amud, first) = byPage.first().first() entries += PlannedMarkerEntry(path, null, 0, amud.text, first.id, first.index, isMarker = byPage.size == 1) if (byPage.size == 1) { - for (line in byPage.single()) pathByOccurrence[key to line.index] = path + for ((_, line) in byPage.single()) pathByOccurrence[key to line.index] = path return@forEachIndexed } byPage.forEachIndexed { j, group -> - val line = group.first() + val (own, line) = group.first() val heading = line.tocEntryId?.takeIf { it != soleRoot }?.let(tocById::get)?.text?.trimHeadingStart()?.trim() val childPath = altTocChildPath(path, j + 1) - val text = "${amud.text} – ${heading?.let { "רי\"ף $it" } ?: "${j + 1}"}" + val text = "${own.text} – ${heading?.let { "רי\"ף $it" } ?: "${j + 1}"}" entries += PlannedMarkerEntry(childPath, path, 1, text, line.id, line.index, isMarker = true) - for (occurrence in group) pathByOccurrence[key to occurrence.index] = childPath + for ((_, occurrence) in group) pathByOccurrence[key to occurrence.index] = childPath } } val owners = lastKeyByLineIndex.mapValues { (index, key) -> pathByOccurrence.getValue(key to index) } - return GemaraDafPlan(entries, owners) + return GemaraDafPlan(entries, owners, dropped) } +/** + * Last daf of the Bavli tractate a `רי"ף X` book follows: the highest `דף` heading of + * the one book titled X that has them; null when there is no single such book. + */ +internal fun bavliLastDaf(conn: Connection, rifTitle: String): Int? { + val tractate = rifTitle.removePrefix(RIF_TITLE_PREFIX).takeIf { it != rifTitle } ?: return null + val lastByBook = conn.prepareStatement( + """ + SELECT t.bookId, tt.text FROM tocEntry t JOIN tocText tt ON tt.id = t.textId + WHERE t.bookId IN (SELECT id FROM book WHERE title = ?) + """.trimIndent(), + ).use { st -> + st.setString(1, tractate) + st.executeQuery().use { rs -> + val last = HashMap() + while (rs.next()) { + val daf = GEMARA_DAF_HEADING.matchEntire(rs.getString(2).trimHeadingStart().trim())?.groupValues?.get(1) + val value = daf?.let(::gematriaValue) ?: continue + last.merge(rs.getLong(1), value, ::maxOf) + } + last + } + } + return lastByBook.values.singleOrNull() +} + +private const val RIF_TITLE_PREFIX = "רי\"ף " + /** Atomically replaces each rule's structure; returns entry counts by book title. */ internal fun writeGemaraDafAltTocs( conn: Connection, rules: List, stableIds: AttachedBuildStateIds? = null, ): Map = writeEachBookAtomically(conn, GEMARA_DAF_ALT_TOC_FILE, rules.map { it.bookTitle to it }) { bookId, rule -> + val logger = Logger.withTag("GemaraDafAltToc") + val lastDaf = bavliLastDaf(conn, rule.bookTitle) + if (lastDaf == null) logger.w { "${rule.bookTitle}: no single Bavli tractate found; markers are not limited to its dapim" } val lines = readMarkerLines(conn, bookId) - val plan = planGemaraDafEntries(lines, readMarkerToc(conn, bookId)) + val plan = planGemaraDafEntries(lines, readMarkerToc(conn, bookId), lastDaf) + if (plan.dropped.isNotEmpty()) { + logger.i { + "${rule.bookTitle}: dropped ${plan.dropped.size} markers past the tractate's end: " + + plan.dropped.joinToString { (amud, index) -> "${amud.text} (line $index)" } + } + } check(plan.entries.isNotEmpty()) { "$GEMARA_DAF_ALT_TOC_FILE: no '(דף …)' marker in '${rule.bookTitle}'" } writeMarkerStructure( conn, bookId, rule.structureKey, rule.structureHeTitle, plan.entries, lines, stableIds, plan.ownerPathByLineIndex, diff --git a/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/GemaraDafAltTocTest.kt b/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/GemaraDafAltTocTest.kt index 158e406e..3614d959 100644 --- a/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/GemaraDafAltTocTest.kt +++ b/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/GemaraDafAltTocTest.kt @@ -41,10 +41,10 @@ class GemaraDafAltTocTest { assertEquals(listOf("דף ב.", "דף ב:"), texts("גמ' (דף ב.) תנו רבנן (דף ב:) עוד")) assertEquals(listOf("דף קעו."), texts("(דף קעו.)")) assertEquals(listOf("דף יז.", "דף ג:"), texts("(דף יז ע\"א) ו(דף ג ע\"ב)")) - assertEquals(listOf("דף ח."), texts("[דף ח ע\"א]")) for (reference in listOf( "(דף ח: ע\"ש)", "(דף לעיל מח:)", "(דף לקמן יח.)", "(דף יד: טו.)", "(דף ו. דף לג.)", "(דף כא. ובפ\"א דחלה)", "(דף נ. ע\"ש (ג\"א)", "(דף ס)", "(דף קל)", "(ברכות דף יב.)", "(דף ח.]", + "[דף ח ע\"א]", "([דף ח ע\"א])", "לקמן ((דף י\"ט ע\"ב)", "(דף יז ע\"א)", )) { assertEquals(emptyList(), texts(reference), reference) } @@ -96,6 +96,30 @@ class GemaraDafAltTocTest { assertEquals(mapOf(2 to "3", 3 to "3", 4 to "2"), plan.ownerPathByLineIndex) } + @Test + fun `each entry is spelled like the marker it points to`() { + val spelled = listOf( + MarkerLineRow(102, 2, "(דף יט:) א", 2), + MarkerLineRow(106, 6, "(דף י\"ט ע\"ב) ב", 3), + ) + val entries = planGemaraDafEntries(spelled, toc).entries + assertEquals(listOf("דף יט:", "דף יט: – רי\"ף דף א.", "דף י\"ט: – רי\"ף דף ג:"), entries.map { it.text }) + for (entry in entries) { + val marker = entry.text.substringBefore(" – ") + val content = spelled.single { it.index == entry.lineIndex }.content + val amudForm = "(${marker.dropLast(1)} ע\"${if (marker.last() == '.') 'א' else 'ב'})" + assert("($marker)" in content || amudForm in content) { "${entry.text} not found in '$content'" } + } + } + + @Test + fun `markers past the tractate's last daf are dropped`() { + val plan = planGemaraDafEntries(lines, toc, lastDaf = 8) + assertEquals(listOf("דף ב.", "דף ב:", "דף ח:"), plan.entries.filter { it.parentPath == null }.map { it.text }) + assertEquals(listOf("דף ט." to 4), plan.dropped.map { (amud, index) -> amud.text to index }) + assertEquals("2", plan.ownerPathByLineIndex[4]) + } + @Test fun `an occurrence without a Rif page heading is numbered`() { val flat = lines.map { it.copy(tocEntryId = 1) } @@ -174,6 +198,40 @@ class GemaraDafAltTocTest { } } + @Test + fun `the last daf comes from the one Bavli tractate of the same name`() = runBlocking { + SeforimDb.Schema.create(driver) + repo = SeforimRepository(dbFile.toString(), driver) + val catId = repo.insertCategory(Category(0, null, "תלמוד בבלי", level = 0, order = 1)) + val source = repo.insertSource("Sefaria") + suspend fun addBook(title: String, headings: List) { + val bookId = repo.insertBook(Book(categoryId = catId, sourceId = source, title = title)) + headings.forEachIndexed { i, text -> + val lineId = repo.insertLine(Line(bookId = bookId, lineIndex = i, content = text)) + repo.insertTocEntry(TocEntry(bookId = bookId, text = text, level = 1, lineId = lineId)) + } + } + addBook("ברכות", listOf("ברכות", "דף ב.", "דף סד.", "דף י:")) + addBook("רי\"ף ברכות", listOf("דף קה:")) + DriverManager.getConnection("jdbc:sqlite:$dbFile").use { conn -> + assertEquals(64, bavliLastDaf(conn, "רי\"ף ברכות")) + assertEquals(null, bavliLastDaf(conn, "רי\"ף שבת")) + assertEquals(null, bavliLastDaf(conn, "ברכות")) + // A second book of that title (the repository dedupes titles, so copy the rows). + conn.createStatement().use { st -> + for (sql in listOf( + "CREATE TEMP TABLE b AS SELECT * FROM book WHERE title = 'ברכות'", + "UPDATE b SET id = 999", + "INSERT INTO book SELECT * FROM b", + "CREATE TEMP TABLE t AS SELECT * FROM tocEntry WHERE bookId = (SELECT id FROM book WHERE title = 'ברכות' AND id <> 999)", + "UPDATE t SET id = id + 1000, bookId = 999, parentId = NULL", + "INSERT INTO tocEntry SELECT * FROM t", + )) st.execute(sql) + } + assertEquals(null, bavliLastDaf(conn, "רי\"ף ברכות")) + } + } + @Test fun `a listed book without markers fails`() = runBlocking { SeforimDb.Schema.create(driver) From bd37a0600023898308052d8aa9a3b9039206bed6 Mon Sep 17 00:00:00 2001 From: palmoni5 Date: Thu, 19 Nov 2026 04:24:02 +0200 Subject: [PATCH 4/4] =?UTF-8?q?=D7=93=D7=A4=D7=99=20=D7=94=D7=92=D7=9E?= =?UTF-8?q?=D7=A8=D7=90:=20=D7=A1=D7=9E=D7=9F=20=D7=90=D7=97=D7=A8=D7=99?= =?UTF-8?q?=20"(("=20=D7=A0=D7=A1=D7=A4=D7=A8,=20=D7=97=D7=95=D7=A5=20?= =?UTF-8?q?=D7=9E=D7=94=D7=A4=D7=A0=D7=99=D7=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit סוגר מיותר לפני סמן נפוץ בטקסט, ולכן ((דף כא ע"ב) נספר שוב. סמן כזה נדחה רק כשאחת משלוש המילים שלפניו היא לקמן, לעיל, בפרק או בפירקא, אולי עם ו/ד/כד בתחילתה. סמן בסוגריים יחידים אינו נבדק כך. --- .../sefariasqlite/GemaraDafAltTocPostProcess.kt | 17 +++++++++++++---- .../sefariasqlite/GemaraDafAltTocTest.kt | 9 ++++++++- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/GemaraDafAltTocPostProcess.kt b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/GemaraDafAltTocPostProcess.kt index 735f9663..727feabe 100644 --- a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/GemaraDafAltTocPostProcess.kt +++ b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/GemaraDafAltTocPostProcess.kt @@ -57,15 +57,24 @@ internal data class GemaraAmud(val daf: String, val value: Int, val side: Char) val key get() = value * 2 + if (side == '.') 0 else 1 } -// Nothing but the daf and amud inside one pair of round brackets: `(דף ח: ע"ש)`, -// `(דף לעיל מח:)` and `((דף יט ע"ב)` are references. -private val GEMARA_DAF_MARKER = Regex("""(?]*>") +private val WHITESPACE = Regex("""\s+""") +private val REFERENCE_WORD = Regex("""ו?(?:ד|כד)?(?:לקמן|לעיל|בפרק|בפירקא)""") + +/** `לקמן ((דף יט ע"ב)`: one of the three words before a `((` marker makes it a reference. */ +private fun followsReferenceWord(textBefore: String): Boolean = + textBefore.replace(HTML_TAG, " ").split(WHITESPACE).filter { it.isNotEmpty() }.takeLast(3) + .any { REFERENCE_WORD.matches(it.trim('(', ')', '\'', '"', ',', '.', ':')) } /** The Gemara-daf markers of [content], in order. */ internal fun findGemaraDafMarkers(content: String): List = GEMARA_DAF_MARKER.findAll(content).mapNotNull { match -> - val (daf, side, amud) = match.destructured + val (extraParen, daf, side, amud) = match.destructured + if (extraParen.isNotEmpty() && followsReferenceWord(content.substring(0, match.range.first))) return@mapNotNull null val value = gematriaValue(daf) ?: return@mapNotNull null GemaraAmud(daf, value, side.firstOrNull() ?: if (amud == "א") '.' else ':') }.toList() diff --git a/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/GemaraDafAltTocTest.kt b/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/GemaraDafAltTocTest.kt index 3614d959..a48c43f1 100644 --- a/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/GemaraDafAltTocTest.kt +++ b/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/GemaraDafAltTocTest.kt @@ -41,10 +41,17 @@ class GemaraDafAltTocTest { assertEquals(listOf("דף ב.", "דף ב:"), texts("גמ' (דף ב.) תנו רבנן (דף ב:) עוד")) assertEquals(listOf("דף קעו."), texts("(דף קעו.)")) assertEquals(listOf("דף יז.", "דף ג:"), texts("(דף יז ע\"א) ו(דף ג ע\"ב)")) + // A stray `(` before a marker is common; reference words only matter there. + assertEquals(listOf("דף כא:"), texts("משתעי אינשי הכי ((דף כא ע\"ב) מי דמי")) + assertEquals(listOf("דף כז."), texts("גמ' ((דף כז.) אמר")) + assertEquals(listOf("דף ב:"), texts("לקמן מפורש (דף ב:) תנו")) + assertEquals(listOf("דף נו:"), texts("כדלקמן א ב ג ((דף נו:)")) for (reference in listOf( "(דף ח: ע\"ש)", "(דף לעיל מח:)", "(דף לקמן יח.)", "(דף יד: טו.)", "(דף ו. דף לג.)", "(דף כא. ובפ\"א דחלה)", "(דף נ. ע\"ש (ג\"א)", "(דף ס)", "(דף קל)", "(ברכות דף יב.)", "(דף ח.]", - "[דף ח ע\"א]", "([דף ח ע\"א])", "לקמן ((דף י\"ט ע\"ב)", "(דף יז ע\"א)", + "[דף ח ע\"א]", "([דף ח ע\"א])", "(דף יז ע\"א)", "(((דף ב.)", + "דאמרי' לקמן ((דף י\"ט ע\"ב)", "בפרק בני העיר ((דף ל ע\"א)", "וגרסינן (בה) בפירקא קמא ((דף ג ע\"א)", + "ודלעיל א ((דף ג.)", )) { assertEquals(emptyList(), texts(reference), reference) }