Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14,224 changes: 14,224 additions & 0 deletions generator/data/ref-arbitration.jsonl

Large diffs are not rendered by default.

82 changes: 82 additions & 0 deletions generator/reflinks/build.gradle.kts
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
plugins {
alias(libs.plugins.multiplatform)
alias(libs.plugins.kotlinx.serialization)
}

kotlin {
jvmToolchain(libs.versions.jvmToolchain.get().toInt())

jvm()

sourceSets {
commonMain.dependencies {
implementation(libs.kotlinx.coroutines.core)
implementation(libs.kotlinx.serialization.json)
implementation(libs.kermit)
}

commonTest.dependencies {
implementation(kotlin("test"))
}

jvmMain.dependencies {
implementation(libs.sqlite.jdbc)
implementation(libs.slf4j.simple)
}

jvmTest.dependencies {
implementation(libs.kotlinx.coroutines.test)
}
}
}

val defaultDb: String = rootProject.layout.buildDirectory.file("seforim.db").get().asFile.absolutePath
val decisionsFile: String = layout.projectDirectory.file("../data/ref-arbitration.jsonl").asFile.absolutePath

/**
* Fills the LLM decision cache for the ambiguous citations of the corpus. The only task that calls the network.
* Usage:
* ./gradlew :reflinks:arbitrateRefs -PseforimDb=… -PllmKeyFile=~/key.txt -PmaxSpend=10 [-PmaxCases=5000] [-Pconfirm]
* Without -Pconfirm it only prints the estimate (cache misses, cost, remaining credit) and exits.
*/
tasks.register<JavaExec>("arbitrateRefs") {
group = "application"
description = "Resolve ambiguous citations with the LLM and append the decisions to generator/data/ref-arbitration.jsonl."
dependsOn("jvmJar")
mainClass.set("io.github.kdroidfilter.seforimlibrary.reflinks.ArbitrateRefsKt")
classpath = files(tasks.named("jvmJar")) + configurations.getByName("jvmRuntimeClasspath")
jvmArgs = listOf("-Xmx6g", "--enable-native-access=ALL-UNNAMED")
args = buildList {
add("--db=" + ((project.findProperty("seforimDb") as String?) ?: defaultDb))
add("--decisions=$decisionsFile")
add("--sefaria-schemas=" + rootProject.layout.projectDirectory.dir("generator/sefariasqlite/build/sefaria/export/schemas").asFile.absolutePath)
(project.findProperty("llmKeyFile") as String?)?.let { add("--key-file=$it") }
(project.findProperty("llmModel") as String?)?.let { add("--model=$it") }
(project.findProperty("maxSpend") as String?)?.let { add("--max-spend=$it") }
(project.findProperty("maxCases") as String?)?.let { add("--max-cases=$it") }
(project.findProperty("workers") as String?)?.let { add("--workers=$it") }
(project.findProperty("classes") as String?)?.let { add("--classes=$it") }
if (project.hasProperty("confirm")) add("--confirm")
if (project.hasProperty("retryInvalid")) add("--retry-invalid")
}
}

/**
* Writes the citation links into the text of a copy of the database (no network: deterministic links + cached LLM
* decisions). Usage: ./gradlew :reflinks:generateRefLinks [-PseforimDb=…] [-PrefLinksOut=…]
*/
tasks.register<JavaExec>("generateRefLinks") {
group = "application"
description = "Write <a class=\"ref\" href=\"seforim://line/<id>\"> links into a copy of seforim.db."
dependsOn("jvmJar")
mainClass.set("io.github.kdroidfilter.seforimlibrary.reflinks.GenerateRefLinksKt")
classpath = files(tasks.named("jvmJar")) + configurations.getByName("jvmRuntimeClasspath")
jvmArgs = listOf("-Xmx8g", "--enable-native-access=ALL-UNNAMED")
args = listOf(
"--db=" + ((project.findProperty("seforimDb") as String?) ?: defaultDb),
"--out=" + ((project.findProperty("refLinksOut") as String?)
?: rootProject.layout.buildDirectory.file("reflinks/seforim.db").get().asFile.absolutePath),
"--decisions=$decisionsFile",
"--sefaria-schemas=" + rootProject.layout.projectDirectory.dir("generator/sefariasqlite/build/sefaria/export/schemas").asFile.absolutePath,
)
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
package io.github.kdroidfilter.seforimlibrary.reflinks

/**
* Surface forms of the books the parser recognizes, mapped to their `book.title`.
* Explicit spelling variants only: normalizing yod/vav globally caused false hits ("הובא למעלה" → "מעילה").
*/
internal object BookAliases {

val tanakh: Map<String, List<String>> = mapOf(
"בראשית" to listOf("בראשית", "בר'", "ברא'"),
"שמות" to listOf("שמות"),
"ויקרא" to listOf("ויקרא", "ויק'"),
"במדבר" to listOf("במדבר", "במד'"),
"דברים" to listOf("דברים", "דב'"),
"יהושע" to listOf("יהושע", "יהו'"),
"שופטים" to listOf("שופטים", "שופ'"),
"שמואל א" to listOf("שמואל א", "ש\"א", "שמ\"א", "שמואל א'", "שמואל ראשון"),
"שמואל ב" to listOf("שמואל ב", "ש\"ב", "שמ\"ב", "שמואל ב'", "שמואל שני"),
"מלכים א" to listOf("מלכים א", "מ\"א", "מל\"א", "מלכים א'"),
"מלכים ב" to listOf("מלכים ב", "מ\"ב", "מל\"ב", "מלכים ב'"),
"ישעיהו" to listOf("ישעיהו", "ישעיה", "ישעי'"),
"ירמיהו" to listOf("ירמיהו", "ירמיה", "ירמי'"),
"יחזקאל" to listOf("יחזקאל", "יחזק'", "יחז'"),
"הושע" to listOf("הושע"), "יואל" to listOf("יואל"), "עמוס" to listOf("עמוס"),
"עובדיה" to listOf("עובדיה", "עבדיה"), "יונה" to listOf("יונה"), "מיכה" to listOf("מיכה"),
"נחום" to listOf("נחום"), "חבקוק" to listOf("חבקוק"), "צפניה" to listOf("צפניה"), "חגי" to listOf("חגי"),
"זכריה" to listOf("זכריה", "זכרי'"), "מלאכי" to listOf("מלאכי"),
"תהילים" to listOf("תהלים", "תהילים", "תהל'", "תהי'"),
"משלי" to listOf("משלי"), "איוב" to listOf("איוב"),
"שיר השירים" to listOf("שיר השירים", "שה\"ש", "שיר השיר'"),
"רות" to listOf("רות"), "איכה" to listOf("איכה"), "קהלת" to listOf("קהלת", "קוהלת"),
"אסתר" to listOf("אסתר"), "דניאל" to listOf("דניאל"), "עזרא" to listOf("עזרא"), "נחמיה" to listOf("נחמיה"),
"דברי הימים א" to listOf("דברי הימים א", "דה\"א", "דהי\"א", "דברי הימים א'"),
"דברי הימים ב" to listOf("דברי הימים ב", "דה\"ב", "דהי\"ב", "דברי הימים ב'"),
)

/** Bavli tractates come first: [bavli] is the subset that has a Babylonian Talmud. */
val tractates: Map<String, List<String>> = linkedMapOf(
"ברכות" to listOf("ברכות"), "שבת" to listOf("שבת"), "עירובין" to listOf("עירובין", "ערובין"),
"פסחים" to listOf("פסחים"), "יומא" to listOf("יומא"), "סוכה" to listOf("סוכה"), "ביצה" to listOf("ביצה"),
"ראש השנה" to listOf("ראש השנה", "ר\"ה"), "תענית" to listOf("תענית"), "מגילה" to listOf("מגילה"),
"מועד קטן" to listOf("מועד קטן", "מו\"ק"), "חגיגה" to listOf("חגיגה"), "יבמות" to listOf("יבמות"),
"כתובות" to listOf("כתובות"), "נדרים" to listOf("נדרים"), "נזיר" to listOf("נזיר"), "סוטה" to listOf("סוטה"),
"גיטין" to listOf("גיטין"), "קידושין" to listOf("קידושין", "קדושין"),
"בבא קמא" to listOf("בבא קמא", "ב\"ק"), "בבא מציעא" to listOf("בבא מציעא", "ב\"מ"),
"בבא בתרא" to listOf("בבא בתרא", "ב\"ב"), "סנהדרין" to listOf("סנהדרין"), "מכות" to listOf("מכות"),
"שבועות" to listOf("שבועות"), "עבודה זרה" to listOf("עבודה זרה", "ע\"ז"), "הוריות" to listOf("הוריות"),
"זבחים" to listOf("זבחים"), "מנחות" to listOf("מנחות"), "חולין" to listOf("חולין"), "בכורות" to listOf("בכורות"),
"ערכין" to listOf("ערכין"), "תמורה" to listOf("תמורה"), "כריתות" to listOf("כריתות"), "מעילה" to listOf("מעילה"),
"תמיד" to listOf("תמיד"), "נדה" to listOf("נדה", "נידה"),
"פאה" to listOf("פאה"), "דמאי" to listOf("דמאי"), "כלאים" to listOf("כלאים"), "שביעית" to listOf("שביעית"),
"תרומות" to listOf("תרומות"), "מעשרות" to listOf("מעשרות"), "מעשר שני" to listOf("מעשר שני"),
"חלה" to listOf("חלה"), "ערלה" to listOf("ערלה"), "ביכורים" to listOf("ביכורים", "בכורים"),
"שקלים" to listOf("שקלים"), "עדויות" to listOf("עדויות", "עדיות"), "אבות" to listOf("אבות"),
"מדות" to listOf("מדות", "מידות"), "קינים" to listOf("קינים", "קנים"), "כלים" to listOf("כלים"),
"אהלות" to listOf("אהלות"), "נגעים" to listOf("נגעים"), "פרה" to listOf("פרה"), "טהרות" to listOf("טהרות"),
"מקואות" to listOf("מקואות", "מקוואות"), "מכשירין" to listOf("מכשירין"), "זבים" to listOf("זבים"),
"טבול יום" to listOf("טבול יום"), "ידים" to listOf("ידים", "ידיים"), "עוקצים" to listOf("עוקצים", "עוקצין"),
)

val bavli: Set<String> = tractates.keys.take(37).toSet()

val rabba: Map<String, List<String>> = mapOf(
"בראשית רבה" to listOf("בראשית רבה", "ב\"ר", "בר\"ר", "בראשית רבא"),
"שמות רבה" to listOf("שמות רבה", "שמו\"ר", "ש\"ר", "שמ\"ר"),
"ויקרא רבה" to listOf("ויקרא רבה", "ויק\"ר", "וי\"ר"),
"במדבר רבה" to listOf("במדבר רבה", "במד\"ר", "במ\"ר"),
"דברים רבה" to listOf("דברים רבה", "דב\"ר"),
"שיר השירים רבה" to listOf("שיר השירים רבה", "שהש\"ר", "שה\"ש רבה"),
"רות רבה" to listOf("רות רבה", "רו\"ר"),
"איכה רבה" to listOf("איכה רבה", "איכ\"ר", "איכה רבתי"),
"קוהלת רבה" to listOf("קהלת רבה", "קה\"ר", "קוהלת רבה"),
"אסתר רבה" to listOf("אסתר רבה", "אסת\"ר", "אס\"ר"),
)

/** The four parts of Tur / Shulchan Arukh. */
val shulchanArukhParts: Map<String, List<String>> = mapOf(
"אורח חיים" to listOf("או\"ח", "אורח חיים"),
"יורה דעה" to listOf("יו\"ד", "יורה דעה"),
"אבן העזר" to listOf("אה\"ע", "אבה\"ע", "אבן העזר"),
"חושן משפט" to listOf("חו\"מ", "חושן משפט", "חשן משפט"),
)

val misc: Map<String, List<String>> = mapOf(
"משנה ברורה" to listOf("משנה ברורה", "משנ\"ב"),
"מדרש תנחומא" to listOf("תנחומא", "מדרש תנחומא"),
"ספרא" to listOf("ספרא", "תורת כהנים"),
"ספרי במדבר" to listOf("ספרי במדבר"),
"ספרי דברים" to listOf("ספרי דברים"),
"ספר החינוך" to listOf("ספר החינוך", "החינוך"),
"ילקוט שמעוני על התורה" to listOf("ילקוט שמעוני", "ילק\"ש"),
)

/** Short Rambam hilchot names that differ from the title's own wording. */
val rambamExtra: Map<String, List<String>> = mapOf(
"קריאת שמע" to listOf("ק\"ש"),
"יסודי התורה" to listOf("יסוה\"ת", "יסודי תורה"),
"מאכלות אסורות" to listOf("מאכ\"א"),
"איסורי ביאה" to listOf("איסו\"ב"),
"תפילה וברכת כהנים" to listOf("תפלה", "תפילה"),
"עבודה זרה וחוקות הגויים" to listOf("ע\"ז", "עכו\"ם", "עבודה זרה"),
"תלמוד תורה" to listOf("ת\"ת"),
"שביתת יום טוב" to listOf("יום טוב", "יו\"ט"),
"מלוה ולוה" to listOf("מלוה"),
"גזילה ואבידה" to listOf("גזילה", "אבידה"),
"סנהדרין והעונשין המסורין להם" to listOf("סנהדרין"),
"מלכים ומלחמות" to listOf("מלכים"),
"רוצח ושמירת נפש" to listOf("רוצח"),
"זכייה ומתנה" to listOf("זכיה ומתנה"),
"ערכים וחרמין" to listOf("ערכין"),
"שמיטה ויובל" to listOf("שמיטה"),
"קידוש החודש" to listOf("קדוש החדש"),
"יבום וחליצה" to listOf("יבום", "חליצה"),
"אבל" to listOf("אבלות"),
)

/** Words that introduce a citation without being part of it ("ועיין", "ראה", …), longest first. */
val introWords: List<String> = listOf(
"ועיין עוד", "עיין עוד", "ועיין", "עיין", "ועי'", "עי'", "וע\"ע", "ע\"ע", "וע'", "ע'", "וראה", "ראה", "עיי'",
"וכמ\"ש", "כמ\"ש", "כדאיתא", "דאיתא", "והובא", "הובא", "מובא", "ע\"פ", "על פי", "עפ\"י", "וכן", "יעויין",
"ויעויין", "וראו", "ראו",
)
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
package io.github.kdroidfilter.seforimlibrary.reflinks

/** One level of a heRef address ("בראשית א, ד" = [Num(1), Num(4)], "שבת כא:" = [Daf(21, B)]). */
sealed interface Section {
data class Num(val value: Int) : Section
data class Daf(val folio: Int, val side: Side) : Section
data class Name(val text: String) : Section

enum class Side { A, B }
}

enum class Family {
TANAKH, BAVLI, MISHNA, YERUSHALMI, TOSEFTA, RASHI_BAVLI, TOSAFOT,
RABBA, SHULCHAN_ARUKH, TUR, RAMBAM, MISC, ZOHAR_DAF,
}

/** How precise the parsed location is. BOOK means the name was recognized but no usable location. */
enum class Precision { BOOK, CHAPTER, FULL }

/**
* A recognized citation: [title] is the target book title as stored in `book.title`, [sections] its heRef address.
* [trailingTokens] counts the words after the location that were not part of the citation (e.g. "ד\"ה …").
*/
data class Citation(
val family: Family,
val title: String,
val sections: List<Section>,
val precision: Precision,
val alias: String,
val trailingTokens: Int = 0,
val rangeEnd: Int? = null,
/** Zohar only: volume (1-3) and folio, resolved through the alt TOC rather than heRef. */
val zoharVolume: Int? = null,
val zoharDaf: Section.Daf? = null,
)
Loading
Loading