Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -3,61 +3,23 @@ package io.github.kdroidfilter.seforimlibrary.cli
import io.github.kdroidfilter.seforimlibrary.dao.repository.SeforimRepository
import io.github.kdroidfilter.seforimlibrary.search.LineSnippetInfo
import io.github.kdroidfilter.seforimlibrary.search.SnippetProvider
import io.github.kdroidfilter.seforimlibrary.search.SnippetSource
import io.github.kdroidfilter.seforimlibrary.search.SnippetSources
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
import org.jsoup.Jsoup
import org.jsoup.safety.Safelist

/**
* CLI implementation of [SnippetProvider] that fetches line content from the database.
*/
private const val SNIPPET_NEIGHBOR_WINDOW = 4
private const val SNIPPET_MIN_LENGTH = 280

class CliSnippetProvider(
private val repository: SeforimRepository,
) : SnippetProvider {
override suspend fun getSnippetSources(lines: List<LineSnippetInfo>): Map<Long, String> {
override suspend fun getSnippetSources(lines: List<LineSnippetInfo>): Map<Long, SnippetSource> {
if (lines.isEmpty()) return emptyMap()

return withContext(Dispatchers.IO) {
val byBook = lines.groupBy { it.bookId }
val result = mutableMapOf<Long, String>()

for ((bookId, bookLines) in byBook) {
val minIdx = bookLines.minOf { it.lineIndex }
val maxIdx = bookLines.maxOf { it.lineIndex }
val rangeStart = (minIdx - SNIPPET_NEIGHBOR_WINDOW).coerceAtLeast(0)
val rangeEnd = maxIdx + SNIPPET_NEIGHBOR_WINDOW

val allLines = repository.getLines(bookId, rangeStart, rangeEnd)

val plainByIndex = allLines.associate { line ->
line.lineIndex to cleanHtml(line.content)
}

for (info in bookLines) {
val basePlain = plainByIndex[info.lineIndex].orEmpty()
val snippetSource = if (basePlain.length >= SNIPPET_MIN_LENGTH) {
basePlain
} else {
val start = (info.lineIndex - SNIPPET_NEIGHBOR_WINDOW).coerceAtLeast(0)
val end = info.lineIndex + SNIPPET_NEIGHBOR_WINDOW
(start..end)
.mapNotNull { plainByIndex[it] }
.joinToString(" ")
}
result[info.lineId] = snippetSource
}
SnippetSources.build(lines) { bookId, from, to ->
repository.getLines(bookId, from, to).associate { it.lineIndex to it.content }
}

result
}
}

private fun cleanHtml(content: String): String =
Jsoup
.clean(content, Safelist.none())
.replace("\\s+".toRegex(), " ")
.trim()
}
2 changes: 1 addition & 1 deletion gradle/libs.versions.toml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ kotlin = "2.3.20"
agp = "9.1.1"
jvmToolchain = "25"
lucene = "10.5.2"
seforimSemantic = "0.4.0"
seforimSemantic = "0.4.1"
onnxruntime = "1.30.0"
maven-publish = "0.36.0"
kotlinx-coroutines = "1.10.2"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -54,12 +54,22 @@ object HebrewTextUtils {
* @param text The input text
* @return Text with final letters replaced
*/
fun replaceFinalsWithBase(text: String): String = text
.replace('\u05DA', '\u05DB') // ך -> כ
.replace('\u05DD', '\u05DE') // ם -> מ
.replace('\u05DF', '\u05E0') // ן -> נ
.replace('\u05E3', '\u05E4') // ף -> פ
.replace('\u05E5', '\u05E6') // ץ -> צ
fun replaceFinalsWithBase(text: String): String {
if (text.none { it != baseLetter(it) }) return text
val sb = StringBuilder(text.length)
for (c in text) sb.append(baseLetter(c))
return sb.toString()
}

/** [c] with a final letter (sofit) as its base form: ך כ, ם מ, ן נ, ף פ, ץ צ. */
fun baseLetter(c: Char): Char = when (c) {
'\u05DA' -> '\u05DB'
'\u05DD' -> '\u05DE'
'\u05DF' -> '\u05E0'
'\u05E3' -> '\u05E4'
'\u05E5' -> '\u05E6'
else -> c
}

/**
* Checks if a character is a Hebrew diacritic (nikud or teamim).
Expand All @@ -71,7 +81,10 @@ object HebrewTextUtils {
val code = c.code
return (code in 0x0591..0x05AF) || // teamim
(code in 0x05B0..0x05BD) || // nikud + meteg
(c == '\u05C1') || (c == '\u05C2') || (c == '\u05C7')
(c == '\u05BF') || // rafe
(c == '\u05C1') || (c == '\u05C2') || // shin and sin dots
(c == '\u05C4') || (c == '\u05C5') || // upper and lower dots
(c == '\u05C7')
}

/**
Expand Down
Loading
Loading